Skip to content

Commit 2be384d

Browse files
VanillaSpoonVanillaSpoon
committed
add: mutating webhook
1 parent c37c4b2 commit 2be384d

13 files changed

+279
-3
lines changed

‎PROJECT

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,5 +15,9 @@ resources:
1515
domain: codeflare.dev
1616
group: ray
1717
kind: RayCluster
18+
path: github.com/project-codeflare/codeflare-operator/pkg/controllers
1819
version: v1
20+
webhooks:
21+
defaulting: true
22+
webhookVersion: v1
1923
version: "3"

‎config/certmanager/certificate.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# The following manifests contain a self-signed issuer CR and a certificate CR.
2+
# More document can be found at https://docs.cert-manager.io
3+
# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes.

‎config/certmanager/kustomization.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
resources:
2+
- certificate.yaml
3+
4+
configurations:
5+
- kustomizeconfig.yaml

‎config/certmanager/kustomizeconfig.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# This configuration is for teaching kustomize how to update name ref and var substitution
2+
nameReference:
3+
- kind: Issuer
4+
group: cert-manager.io
5+
fieldSpecs:
6+
- kind: Certificate
7+
group: cert-manager.io
8+
path: spec/issuerRef/name
9+
10+
varReference:
11+
- kind: Certificate
12+
group: cert-manager.io
13+
path: spec/commonName
14+
- kind: Certificate
15+
group: cert-manager.io
16+
path: spec/dnsNames

‎config/default/kustomization.yaml

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,15 @@ commonLabels:
1414
app.kubernetes.io/part-of: codeflare
1515

1616
bases:
17-
- ../rbac
18-
- ../manager
17+
- ../rbac
18+
- ../manager
19+
- ../webhook
1920
# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
2021
# - ../prometheus
2122

2223
resources:
23-
- metrics_service.yaml
24+
- metrics_service.yaml
25+
26+
patches:
27+
- path: manager_webhook_patch.yaml
28+
- path: webhookcainjection_patch.yaml

‎config/default/manager_webhook_patch.yaml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: manager
5+
namespace: system
6+
spec:
7+
template:
8+
spec:
9+
containers:
10+
- name: manager
11+
ports:
12+
- containerPort: 9443
13+
name: webhook-server
14+
protocol: TCP
15+
volumeMounts:
16+
- mountPath: /tmp/k8s-webhook-server/serving-certs
17+
name: cert
18+
readOnly: true
19+
volumes:
20+
- name: cert
21+
secret:
22+
defaultMode: 420
23+
secretName: codeflare-operator-raycluster-webhook-cert

‎config/default/webhookcainjection_patch.yaml

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# This patch add annotation to admission webhook config
2+
apiVersion: admissionregistration.k8s.io/v1
3+
kind: MutatingWebhookConfiguration
4+
metadata:
5+
labels:
6+
app.kubernetes.io/name: mutatingwebhookconfiguration
7+
app.kubernetes.io/instance: mutating-webhook-configuration
8+
app.kubernetes.io/component: webhook
9+
app.kubernetes.io/created-by: codeflare-operator
10+
app.kubernetes.io/part-of: codeflare-operator
11+
app.kubernetes.io/managed-by: kustomize
12+
name: mutating-webhook-configuration
13+
annotations:
14+
service.beta.openshift.io/inject-cabundle: "true"
15+
service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert

‎config/webhook/kustomization.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
resources:
2+
- manifests.yaml
3+
- service.yaml
4+
5+
configurations:
6+
- kustomizeconfig.yaml

‎config/webhook/kustomizeconfig.yaml

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# the following config is for teaching kustomize where to look at when substituting vars.
2+
# It requires kustomize v2.1.0 or newer to work properly.
3+
nameReference:
4+
- kind: Service
5+
version: v1
6+
fieldSpecs:
7+
- kind: MutatingWebhookConfiguration
8+
group: admissionregistration.k8s.io
9+
path: webhooks/clientConfig/service/name
10+
- kind: ValidatingWebhookConfiguration
11+
group: admissionregistration.k8s.io
12+
path: webhooks/clientConfig/service/name
13+
14+
namespace:
15+
- kind: MutatingWebhookConfiguration
16+
group: admissionregistration.k8s.io
17+
path: webhooks/clientConfig/service/namespace
18+
create: true
19+
- kind: ValidatingWebhookConfiguration
20+
group: admissionregistration.k8s.io
21+
path: webhooks/clientConfig/service/namespace
22+
create: true
23+
24+
varReference:
25+
- path: metadata/annotations

‎config/webhook/manifests.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
---
2+
apiVersion: admissionregistration.k8s.io/v1
3+
kind: MutatingWebhookConfiguration
4+
metadata:
5+
creationTimestamp: null
6+
name: mutating-webhook-configuration
7+
webhooks:
8+
- admissionReviewVersions:
9+
- v1
10+
clientConfig:
11+
service:
12+
name: webhook-service
13+
namespace: system
14+
path: /mutate-ray-io-v1-raycluster
15+
failurePolicy: Fail
16+
name: mraycluster.kb.io
17+
rules:
18+
- apiGroups:
19+
- ray.io
20+
apiVersions:
21+
- v1
22+
operations:
23+
- CREATE
24+
- UPDATE
25+
resources:
26+
- rayclusters
27+
sideEffects: None

‎config/webhook/service.yaml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
apiVersion: v1
2+
kind: Service
3+
metadata:
4+
labels:
5+
app.kubernetes.io/name: service
6+
app.kubernetes.io/part-of: codeflare-operator
7+
app.kubernetes.io/instance: webhook-service
8+
app.kubernetes.io/component: webhook
9+
app.kubernetes.io/created-by: codeflare-operator
10+
app.kubernetes.io/managed-by: kustomize
11+
name: webhook-service
12+
namespace: openshift-operators
13+
annotations:
14+
service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert
15+
spec:
16+
ports:
17+
- port: 443
18+
protocol: TCP
19+
targetPort: 9443
20+
selector:
21+
app.kubernetes.io/part-of: codeflare
22+
app.kubernetes.io/name: codeflare-operator

‎main.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -147,6 +147,9 @@ func main() {
147147
})
148148
exitOnError(err, "unable to start manager")
149149

150+
rayClusterDefaulter := &controllers.RayClusterDefaulter{}
151+
exitOnError(rayClusterDefaulter.SetupWebhookWithManager(mgr), "error setting up webhook")
152+
150153
ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster"))
151154
if ok {
152155
rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay}

‎pkg/controllers/raycluster_webhook.go

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/*
2+
Copyright 2023.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package controllers
18+
19+
import (
20+
"context"
21+
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
22+
corev1 "k8s.io/api/core/v1"
23+
"k8s.io/apimachinery/pkg/runtime"
24+
ctrl "sigs.k8s.io/controller-runtime"
25+
logf "sigs.k8s.io/controller-runtime/pkg/log"
26+
"sigs.k8s.io/controller-runtime/pkg/webhook"
27+
)
28+
29+
// log is for logging in this package.
30+
var rayclusterlog = logf.Log.WithName("raycluster-resource")
31+
32+
func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error {
33+
return ctrl.NewWebhookManagedBy(mgr).
34+
For(&rayv1.RayCluster{}).
35+
WithDefaulter(r).
36+
Complete()
37+
}
38+
39+
//+kubebuilder:webhook:path=/mutate-ray-io-v1-raycluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=ray.io,resources=rayclusters,verbs=create;update,versions=v1,name=mraycluster.kb.io,admissionReviewVersions=v1
40+
41+
type RayClusterDefaulter struct{}
42+
43+
var _ webhook.CustomDefaulter = &RayClusterDefaulter{}
44+
45+
// Default implements webhook.Defaulter so a webhook will be registered for the type
46+
func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error {
47+
raycluster := obj.(*rayv1.RayCluster)
48+
49+
rayclusterlog.Info("default", "name", raycluster.Name)
50+
// Check and add OAuth proxy if it does not exist.
51+
alreadyExists := false
52+
for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers {
53+
if container.Name == "oauth-proxy" {
54+
rayclusterlog.Info("OAuth sidecar already exists, no patch needed")
55+
alreadyExists = true
56+
break // exits the for loop
57+
}
58+
}
59+
60+
if !alreadyExists {
61+
rayclusterlog.Info("Adding OAuth sidecar container")
62+
// definition of the new container
63+
newOAuthSidecar := corev1.Container{
64+
Name: "oauth-proxy",
65+
Image: "registry.redhat.io/openshift4/ose-oauth-proxy@sha256:1ea6a01bf3e63cdcf125c6064cbd4a4a270deaf0f157b3eabb78f60556840366",
66+
Ports: []corev1.ContainerPort{
67+
{ContainerPort: 8443, Name: "oauth-proxy"},
68+
},
69+
Args: []string{
70+
"--https-address=:8443",
71+
"--provider=openshift",
72+
"--openshift-service-account=" + raycluster.Name + "-oauth-proxy",
73+
"--upstream=http://localhost:8265",
74+
"--tls-cert=/etc/tls/private/tls.crt",
75+
"--tls-key=/etc/tls/private/tls.key",
76+
"--cookie-secret=$(COOKIE_SECRET)",
77+
"--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}",
78+
"--added-label=True",
79+
},
80+
Env: []corev1.EnvVar{
81+
{
82+
Name: "COOKIE_SECRET",
83+
ValueFrom: &corev1.EnvVarSource{
84+
SecretKeyRef: &corev1.SecretKeySelector{
85+
LocalObjectReference: corev1.LocalObjectReference{
86+
Name: raycluster.Name + "-oauth-config",
87+
},
88+
Key: "cookie_secret",
89+
},
90+
},
91+
},
92+
},
93+
VolumeMounts: []corev1.VolumeMount{
94+
{
95+
Name: "proxy-tls-secret",
96+
MountPath: "/etc/tls/private",
97+
ReadOnly: true,
98+
},
99+
},
100+
}
101+
102+
// Adding the new OAuth sidecar container
103+
raycluster.Spec.HeadGroupSpec.Template.Spec.Containers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers, newOAuthSidecar)
104+
105+
tlsSecretVolume := corev1.Volume{
106+
Name: "proxy-tls-secret",
107+
VolumeSource: corev1.VolumeSource{
108+
Secret: &corev1.SecretVolumeSource{
109+
SecretName: raycluster.Name + "-proxy-tls-secret",
110+
},
111+
},
112+
}
113+
114+
raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, tlsSecretVolume)
115+
116+
// Ensure the service account is set
117+
if raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName == "" {
118+
raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy"
119+
}
120+
}
121+
return nil
122+
}

0 commit comments

Comments
 (0)