Skip to content

Commit 37629d8

Browse files
Bobbins228Bobbins228
committed
Created InitContainer & resources for mtls patch
1 parent 2be384d commit 37629d8

File tree

3 files changed

+171
-11
lines changed

3 files changed

+171
-11
lines changed

‎config/default/webhookcainjection_patch.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,5 +11,5 @@ metadata:
1111
app.kubernetes.io/managed-by: kustomize
1212
name: mutating-webhook-configuration
1313
annotations:
14-
service.beta.openshift.io/inject-cabundle: "true"
15-
service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert
14+
service.beta.openshift.io/inject-cabundle: "true"
15+
service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert

‎config/webhook/service.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,20 @@ kind: Service
33
metadata:
44
labels:
55
app.kubernetes.io/name: service
6-
app.kubernetes.io/part-of: codeflare-operator
6+
app.kubernetes.io/part-of: codeflare-operator
77
app.kubernetes.io/instance: webhook-service
88
app.kubernetes.io/component: webhook
99
app.kubernetes.io/created-by: codeflare-operator
1010
app.kubernetes.io/managed-by: kustomize
1111
name: webhook-service
1212
namespace: openshift-operators
1313
annotations:
14-
service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert
14+
service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert
1515
spec:
1616
ports:
1717
- port: 443
1818
protocol: TCP
1919
targetPort: 9443
2020
selector:
21-
app.kubernetes.io/part-of: codeflare
21+
app.kubernetes.io/part-of: codeflare
2222
app.kubernetes.io/name: codeflare-operator

‎pkg/controllers/raycluster_webhook.go

Lines changed: 166 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package controllers
1818

1919
import (
2020
"context"
21+
2122
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
2223
corev1 "k8s.io/api/core/v1"
2324
"k8s.io/apimachinery/pkg/runtime"
@@ -26,8 +27,12 @@ import (
2627
"sigs.k8s.io/controller-runtime/pkg/webhook"
2728
)
2829

29-
// log is for logging in this package.
30-
var rayclusterlog = logf.Log.WithName("raycluster-resource")
30+
var (
31+
NameConsoleLink string = "console"
32+
NamespaceConsoleLink string = "openshift-console"
33+
// log is for logging in this package.
34+
rayclusterlog = logf.Log.WithName("raycluster-resource")
35+
)
3136

3237
func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error {
3338
return ctrl.NewWebhookManagedBy(mgr).
@@ -47,17 +52,36 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e
4752
raycluster := obj.(*rayv1.RayCluster)
4853

4954
rayclusterlog.Info("default", "name", raycluster.Name)
55+
oauthExists := false
56+
initHeadExists := false
57+
workerHeadExists := false
5058
// Check and add OAuth proxy if it does not exist.
51-
alreadyExists := false
5259
for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers {
5360
if container.Name == "oauth-proxy" {
5461
rayclusterlog.Info("OAuth sidecar already exists, no patch needed")
55-
alreadyExists = true
62+
oauthExists = true
63+
break // exits the for loop
64+
}
65+
}
66+
67+
// Check for the create-cert Init Containers
68+
for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.InitContainers {
69+
if container.Name == "create-cert" {
70+
rayclusterlog.Info("Head Init Containers already exist, no patch needed")
71+
initHeadExists = true
72+
break // exits the for loop
73+
}
74+
}
75+
// Check fot the create-cert Init Container WorkerGroupSpec
76+
for _, container := range raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers {
77+
if container.Name == "create-cert" {
78+
rayclusterlog.Info("Worker Init Containers already exist, no patch needed")
79+
workerHeadExists = true
5680
break // exits the for loop
5781
}
5882
}
5983

60-
if !alreadyExists {
84+
if !oauthExists {
6185
rayclusterlog.Info("Adding OAuth sidecar container")
6286
// definition of the new container
6387
newOAuthSidecar := corev1.Container{
@@ -75,7 +99,6 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e
7599
"--tls-key=/etc/tls/private/tls.key",
76100
"--cookie-secret=$(COOKIE_SECRET)",
77101
"--openshift-delegate-urls={\"/\":{\"resource\":\"pods\",\"namespace\":\"default\",\"verb\":\"get\"}}",
78-
"--added-label=True",
79102
},
80103
Env: []corev1.EnvVar{
81104
{
@@ -118,5 +141,142 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e
118141
raycluster.Spec.HeadGroupSpec.Template.Spec.ServiceAccountName = raycluster.Name + "-oauth-proxy"
119142
}
120143
}
144+
mtlsPatch(raycluster, initHeadExists, workerHeadExists)
121145
return nil
122146
}
147+
148+
func mtlsPatch(raycluster *rayv1.RayCluster, initHeadExists bool, workerHeadExists bool) {
149+
150+
rayclusterlog.Info("creating json patch for RayCluster initContainers")
151+
152+
// Volume Mounts for the Init Containers
153+
key_volumes := []corev1.VolumeMount{
154+
{
155+
Name: "ca-vol",
156+
MountPath: "/home/ray/workspace/ca",
157+
ReadOnly: true,
158+
},
159+
{
160+
Name: "server-cert",
161+
MountPath: "/home/ray/workspace/tls",
162+
ReadOnly: false,
163+
},
164+
}
165+
166+
// Service name for basic interactive
167+
svcDomain := raycluster.Name + "-head-svc." + raycluster.Namespace + ".svc"
168+
// Ca Secret generated by the SDK
169+
secretName := `ca-secret-` + raycluster.Name
170+
171+
// Env variables for Worker & Head Containers
172+
envList := []corev1.EnvVar{
173+
{
174+
Name: "MY_POD_IP",
175+
ValueFrom: &corev1.EnvVarSource{
176+
FieldRef: &corev1.ObjectFieldSelector{
177+
FieldPath: "status.podIP",
178+
},
179+
},
180+
},
181+
{
182+
Name: "RAY_USE_TLS",
183+
Value: "1",
184+
},
185+
{
186+
Name: "RAY_TLS_SERVER_CERT",
187+
Value: "/home/ray/workspace/tls/server.crt",
188+
},
189+
{
190+
Name: "RAY_TLS_SERVER_KEY",
191+
Value: "/home/ray/workspace/tls/server.key",
192+
},
193+
{
194+
Name: "RAY_TLS_CA_CERT",
195+
Value: "/home/ray/workspace/tls/ca.crt",
196+
},
197+
}
198+
199+
// Volumes for the main container of Head and worker
200+
caVolumes := []corev1.Volume{
201+
{
202+
Name: "ca-vol",
203+
VolumeSource: corev1.VolumeSource{
204+
Secret: &corev1.SecretVolumeSource{
205+
SecretName: secretName,
206+
},
207+
},
208+
},
209+
{
210+
Name: "server-cert",
211+
VolumeSource: corev1.VolumeSource{
212+
EmptyDir: &corev1.EmptyDirVolumeSource{},
213+
},
214+
},
215+
}
216+
217+
if !initHeadExists {
218+
domain := "random.com" // getDomainName()
219+
220+
rayClientRoute := "rayclient-" + raycluster.Name + "-" + raycluster.Namespace + "." + domain
221+
initContainerHead := corev1.Container{
222+
Name: "create-cert",
223+
Image: "quay.io/project-codeflare/ray:latest-py39-cu118",
224+
Command: []string{
225+
"sh",
226+
"-c",
227+
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
228+
},
229+
VolumeMounts: key_volumes,
230+
}
231+
232+
// Append the list of environment variables for the ray-head container
233+
for index, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers {
234+
if container.Name == "ray-head" {
235+
raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[index].Env = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[index].Env, envList...)
236+
}
237+
}
238+
239+
// Append the create-cert Init Container
240+
raycluster.Spec.HeadGroupSpec.Template.Spec.InitContainers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, initContainerHead)
241+
242+
// Append the CA volumes
243+
raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, caVolumes...)
244+
}
245+
246+
if !workerHeadExists {
247+
initContainerWorker := corev1.Container{
248+
Name: "create-cert",
249+
Image: "quay.io/project-codeflare/ray:latest-py39-cu118",
250+
Command: []string{
251+
"sh",
252+
"-c",
253+
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
254+
},
255+
VolumeMounts: key_volumes,
256+
}
257+
// Append the CA volumes
258+
raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes = append(raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVolumes...)
259+
// Append the create-cert Init Container
260+
raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = append(raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, initContainerWorker)
261+
262+
// Append the list of environment variables for the machine-learning container
263+
for index, container := range raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers {
264+
if container.Name == "machine-learning" {
265+
raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[index].Env = append(raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[index].Env, envList...)
266+
}
267+
}
268+
}
269+
}
270+
271+
/*
272+
func getDomainName() (string, error) {
273+
consoleRoute := &routev1.Route{}
274+
275+
if err := k8Client.Get(context.TODO(), types.NamespacedName{Name: NameConsoleLink, Namespace: NamespaceConsoleLink}, consoleRoute); err != nil {
276+
return "error getting console route URL", err
277+
}
278+
domainIndex := strings.Index(consoleRoute.Spec.Host, ".")
279+
consoleLinkDomain := consoleRoute.Spec.Host[domainIndex+1:]
280+
return consoleLinkDomain, nil
281+
}
282+
*/

0 commit comments

Comments
 (0)