Skip to content

Commit 30ea56a

Browse files
Bobbins228Bobbins228
committed
Created InitContainer & resources for mtls patch
1 parent b32e0a9 commit 30ea56a

File tree

2 files changed

+182
-14
lines changed

2 files changed

+182
-14
lines changed

‎main.go

+30-7
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ import (
4747
"sigs.k8s.io/yaml"
4848

4949
routev1 "github.com/openshift/api/route/v1"
50+
routev1ClientSet "github.com/openshift/client-go/route/clientset/versioned"
5051

5152
"github.com/project-codeflare/codeflare-operator/pkg/config"
5253
"github.com/project-codeflare/codeflare-operator/pkg/controllers"
@@ -56,12 +57,15 @@ import (
5657
)
5758

5859
var (
59-
scheme = runtime.NewScheme()
60-
setupLog = ctrl.Log.WithName("setup")
61-
OperatorVersion = "UNKNOWN"
62-
McadVersion = "UNKNOWN"
63-
InstaScaleVersion = "UNKNOWN"
64-
BuildDate = "UNKNOWN"
60+
scheme = runtime.NewScheme()
61+
setupLog = ctrl.Log.WithName("setup")
62+
OperatorVersion = "UNKNOWN"
63+
McadVersion = "UNKNOWN"
64+
InstaScaleVersion = "UNKNOWN"
65+
BuildDate = "UNKNOWN"
66+
NameConsoleLink string = "console"
67+
NamespaceConsoleLink string = "openshift-console"
68+
domain = ""
6569
)
6670

6771
func init() {
@@ -147,8 +151,17 @@ func main() {
147151
})
148152
exitOnError(err, "unable to start manager")
149153

154+
if cfg.KubeRay.IngressDomain == "" {
155+
routeClient, err := routev1ClientSet.NewForConfig(kubeConfig)
156+
exitOnError(err, "unable to create Route Client Set")
157+
domain, err = getOpenShiftDomainName(ctx, routeClient)
158+
exitOnError(err, domain)
159+
} else {
160+
domain = cfg.KubeRay.IngressDomain
161+
}
162+
150163
rayClusterDefaulter := &controllers.RayClusterDefaulter{}
151-
exitOnError(rayClusterDefaulter.SetupWebhookWithManager(mgr), "error setting up webhook")
164+
exitOnError(rayClusterDefaulter.SetupWebhookWithManager(mgr, domain), "error setting up webhook")
152165

153166
ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster"))
154167
if ok {
@@ -248,3 +261,13 @@ func exitOnError(err error, msg string) {
248261
os.Exit(1)
249262
}
250263
}
264+
265+
func getOpenShiftDomainName(ctx context.Context, routeClient routev1ClientSet.Interface) (string, error) {
266+
route, err := routeClient.RouteV1().Routes(NamespaceConsoleLink).Get(ctx, NameConsoleLink, metav1.GetOptions{})
267+
if err != nil {
268+
return "error getting console route URL", err
269+
}
270+
domainIndex := strings.Index(route.Spec.Host, ".")
271+
consoleLinkDomain := route.Spec.Host[domainIndex+1:]
272+
return consoleLinkDomain, nil
273+
}

‎pkg/controllers/raycluster_webhook.go

+152-7
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,14 @@ import (
3030
"github.com/project-codeflare/codeflare-operator/pkg/config"
3131
)
3232

33-
// log is for logging in this package.
34-
var rayclusterlog = logf.Log.WithName("raycluster-resource")
33+
var (
34+
// log is for logging in this package.
35+
rayclusterlog = logf.Log.WithName("raycluster-resource")
36+
baseDomain string = ""
37+
)
3538

36-
func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager) error {
39+
func (r *RayClusterDefaulter) SetupWebhookWithManager(mgr ctrl.Manager, domain string) error {
40+
baseDomain = domain
3741
return ctrl.NewWebhookManagedBy(mgr).
3842
For(&rayv1.RayCluster{}).
3943
WithDefaulter(&RayClusterDefaulter{
@@ -55,20 +59,39 @@ var _ webhook.CustomDefaulter = &RayClusterDefaulter{}
5559
// Default implements webhook.Defaulter so a webhook will be registered for the type
5660
func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) error {
5761
raycluster := obj.(*rayv1.RayCluster)
62+
rayclusterlog.Info("default", "name", raycluster.Name)
63+
oauthExists := false
64+
initHeadExists := false
65+
workerHeadExists := false
66+
67+
// Check for the create-cert Init Containers
68+
for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.InitContainers {
69+
if container.Name == "create-cert" {
70+
rayclusterlog.Info("Head Init Containers already exist, no patch needed")
71+
initHeadExists = true
72+
break // exits the for loop
73+
}
74+
}
75+
// Check fot the create-cert Init Container WorkerGroupSpec
76+
for _, container := range raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers {
77+
if container.Name == "create-cert" {
78+
rayclusterlog.Info("Worker Init Containers already exist, no patch needed")
79+
workerHeadExists = true
80+
break // exits the for loop
81+
}
82+
}
5883

5984
if r.rayDashboardOauthEnabled {
60-
rayclusterlog.Info("default", "name", raycluster.Name)
6185
// Check and add OAuth proxy if it does not exist.
62-
alreadyExists := false
6386
for _, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers {
6487
if container.Name == "oauth-proxy" {
6588
rayclusterlog.Info("OAuth sidecar already exists, no patch needed")
66-
alreadyExists = true
89+
oauthExists = true
6790
break // exits the for loop
6891
}
6992
}
7093

71-
if !alreadyExists {
94+
if !oauthExists {
7295
rayclusterlog.Info("Adding OAuth sidecar container")
7396
// definition of the new container
7497
newOAuthSidecar := corev1.Container{
@@ -133,5 +156,127 @@ func (r *RayClusterDefaulter) Default(ctx context.Context, obj runtime.Object) e
133156
}
134157
}
135158
}
159+
mtlsPatch(raycluster, initHeadExists, workerHeadExists)
136160
return nil
137161
}
162+
163+
func mtlsPatch(raycluster *rayv1.RayCluster, initHeadExists bool, workerHeadExists bool) {
164+
165+
rayclusterlog.Info("creating json patch for RayCluster initContainers")
166+
167+
// Volume Mounts for the Init Containers
168+
key_volumes := []corev1.VolumeMount{
169+
{
170+
Name: "ca-vol",
171+
MountPath: "/home/ray/workspace/ca",
172+
ReadOnly: true,
173+
},
174+
{
175+
Name: "server-cert",
176+
MountPath: "/home/ray/workspace/tls",
177+
ReadOnly: false,
178+
},
179+
}
180+
181+
// Service name for basic interactive
182+
svcDomain := raycluster.Name + "-head-svc." + raycluster.Namespace + ".svc"
183+
// Ca Secret generated by the SDK
184+
secretName := `ca-secret-` + raycluster.Name
185+
186+
// Env variables for Worker & Head Containers
187+
envList := []corev1.EnvVar{
188+
{
189+
Name: "MY_POD_IP",
190+
ValueFrom: &corev1.EnvVarSource{
191+
FieldRef: &corev1.ObjectFieldSelector{
192+
FieldPath: "status.podIP",
193+
},
194+
},
195+
},
196+
{
197+
Name: "RAY_USE_TLS",
198+
Value: "1",
199+
},
200+
{
201+
Name: "RAY_TLS_SERVER_CERT",
202+
Value: "/home/ray/workspace/tls/server.crt",
203+
},
204+
{
205+
Name: "RAY_TLS_SERVER_KEY",
206+
Value: "/home/ray/workspace/tls/server.key",
207+
},
208+
{
209+
Name: "RAY_TLS_CA_CERT",
210+
Value: "/home/ray/workspace/tls/ca.crt",
211+
},
212+
}
213+
214+
// Volumes for the main container of Head and worker
215+
caVolumes := []corev1.Volume{
216+
{
217+
Name: "ca-vol",
218+
VolumeSource: corev1.VolumeSource{
219+
Secret: &corev1.SecretVolumeSource{
220+
SecretName: secretName,
221+
},
222+
},
223+
},
224+
{
225+
Name: "server-cert",
226+
VolumeSource: corev1.VolumeSource{
227+
EmptyDir: &corev1.EmptyDirVolumeSource{},
228+
},
229+
},
230+
}
231+
232+
if !initHeadExists {
233+
rayClientRoute := "rayclient-" + raycluster.Name + "-" + raycluster.Namespace + "." + baseDomain
234+
initContainerHead := corev1.Container{
235+
Name: "create-cert",
236+
Image: "quay.io/project-codeflare/ray:latest-py39-cu118",
237+
Command: []string{
238+
"sh",
239+
"-c",
240+
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)\nDNS.5 = ` + rayClientRoute + `\nDNS.6 = ` + svcDomain + `">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
241+
},
242+
VolumeMounts: key_volumes,
243+
}
244+
245+
// Append the list of environment variables for the ray-head container
246+
for index, container := range raycluster.Spec.HeadGroupSpec.Template.Spec.Containers {
247+
if container.Name == "ray-head" {
248+
raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[index].Env = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Containers[index].Env, envList...)
249+
}
250+
}
251+
252+
// Append the create-cert Init Container
253+
raycluster.Spec.HeadGroupSpec.Template.Spec.InitContainers = append(raycluster.Spec.HeadGroupSpec.Template.Spec.InitContainers, initContainerHead)
254+
255+
// Append the CA volumes
256+
raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes = append(raycluster.Spec.HeadGroupSpec.Template.Spec.Volumes, caVolumes...)
257+
}
258+
259+
if !workerHeadExists {
260+
initContainerWorker := corev1.Container{
261+
Name: "create-cert",
262+
Image: "quay.io/project-codeflare/ray:latest-py39-cu118",
263+
Command: []string{
264+
"sh",
265+
"-c",
266+
`cd /home/ray/workspace/tls && openssl req -nodes -newkey rsa:2048 -keyout server.key -out server.csr -subj '/CN=ray-head' && printf "authorityKeyIdentifier=keyid,issuer\nbasicConstraints=CA:FALSE\nsubjectAltName = @alt_names\n[alt_names]\nDNS.1 = 127.0.0.1\nDNS.2 = localhost\nDNS.3 = ${FQ_RAY_IP}\nDNS.4 = $(awk 'END{print $1}' /etc/hosts)">./domain.ext && cp /home/ray/workspace/ca/* . && openssl x509 -req -CA ca.crt -CAkey ca.key -in server.csr -out server.crt -days 365 -CAcreateserial -extfile domain.ext`,
267+
},
268+
VolumeMounts: key_volumes,
269+
}
270+
// Append the CA volumes
271+
raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes = append(raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Volumes, caVolumes...)
272+
// Append the create-cert Init Container
273+
raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers = append(raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.InitContainers, initContainerWorker)
274+
275+
// Append the list of environment variables for the machine-learning container
276+
for index, container := range raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers {
277+
if container.Name == "machine-learning" {
278+
raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[index].Env = append(raycluster.Spec.WorkerGroupSpecs[0].Template.Spec.Containers[index].Env, envList...)
279+
}
280+
}
281+
}
282+
}

0 commit comments

Comments
 (0)