fix(makefile): standardized image targets

- deleted logs/ directory and added it to .gitignore

- removed RELEASE_PYTHON_VERSION and standardized on PYTHON_VERSION makefile variable

- helper functions to parse makefile target and extract important metadata as makefile variables

- add retries to podman push in build_image makefile function

- dynamically build workbench directory / dockerfile filename based on target

- standardized makefile image targets as <accelerator>-<feature>-<scope>-<os>-<python version>

- single deploy-% target for all images

- single undeploy-% target for all images

- singe test-% target for all images

- new e2e-% target that runs $* + deploy-$* + test-$* + undeploy-$*

- updated/simplified make_test.py in light of Makefile changes

- pass kustomize output to kubectl via stdin to avoid accidental checkin of personal settings

- refactored notebooks/ repo file hierarchy to consistently leverage subfolders for accelerator-specific resources
	- renamed runtimes folder to runtime to match target name
	- jupyter/cuda + jupyter/rocm
	- runtime/cuda + runtime/rocm

- updated kustomize resources for consistency
	- image name used an manifest name prefix
	- -workbench used as manifest name suffix
	- using labels transformer as commonLabels deprecated
	- containerPort named workbench-port
	- removed spec.containers.command from codeserver/rstudio to let server start
	- images.newTag aligned with makefile target
	- added emptyDir volume mount to all workloads
	- added startupProbe to our accelerator images
	- using term "workbench" as opposed to "notebook" consistently throughout manifests

- updated various Dockerfile to match new folder hierarchy where necessary

- refactored test_jupyter_with_papermill to support testing needs of all workbenches + runtimes
	- scripts/makefile_utils directory created
	- numerous usability enhancements to the logic
		- reduce hardcoding of "magic" strings by parsing kustomize output to identify workload names and ports
		- scan for open port and use that when verifying container starts via kubectl port-forward
		- confirms container starts for all workbenches (not just jupyter)
		- confirms required libraries installed in container (now applied to jupyter notebooks as well)
		- moved all validate-xxx target logic into script for better consolidated maintenance
		- relies on makefile to pass metadata parsed from target name to avoid duplicating logic

- TODO:
	- fix any problems in GHA due to above changes
	- add NAMING.md file to explain the "rules" around our makefile target names and all the places in our development flow that is impacted
	- fix openshift/release due to above changes
	- cleanup now-defunct/legacy makefile targets

Related-to: https://issues.redhat.com/browse/RHOAIENG-23291

Author: andyatmiami

.gitignore

@@ -12,6 +12,7 @@ build/
 develop-eggs/
 dist/
 downloads/
+logs/
 eggs/
 .eggs/
 lib/

Makefile

@@ -33,31 +33,8 @@ def main() -> None:
 
 def run_tests(target: str) -> None:
     prefix = target.translate(str.maketrans(".", "-"))
-    # this is a pod name in statefulset, some tests deploy individual unmanaged pods, though
-    pod = prefix + "-notebook-0"  # `$(kubectl get statefulset -o name | head -n 1)` would work too
     namespace = "ns-" + prefix
 
-    if target.startswith("runtime-"):
-        deploy = "deploy9"
-        deploy_target = target.replace("runtime-", "runtimes-")
-    elif target.startswith("rocm-runtime-"):
-        deploy = "deploy9"
-        deploy_target = target.replace("rocm-runtime-", "runtimes-rocm-")
-    elif target.startswith("rocm-jupyter-"):
-        deploy = "deploy9"
-        deploy_target = target.replace("rocm-jupyter-", "jupyter-rocm-")
-    elif target.startswith("cuda-rstudio-"):
-        deploy = "deploy"
-        os = re.match(r"^cuda-rstudio-([^-]+-).*", target)
-        deploy_target = os.group(1) + target.removeprefix("cuda-")
-    elif target.startswith("rstudio-"):
-        deploy = "deploy"
-        os = re.match(r"^rstudio-([^-]+-).*", target)
-        deploy_target = os.group(1) + target
-    else:
-        deploy = "deploy9"
-        deploy_target = target
-
     check_call(f"kubectl create namespace {namespace}", shell=True)
     check_call(f"kubectl config set-context --current --namespace={namespace}", shell=True)
     check_call(f"kubectl label namespace {namespace} fake-scc=fake-restricted-v2", shell=True)
@@ -69,24 +46,10 @@ def run_tests(target: str) -> None:
     # See https://github.com/kubernetes/kubernetes/issues/66689
     check_call("timeout 10s bash -c 'until kubectl get serviceaccount/default; do sleep 1; done'", shell=True)
 
-    check_call(f"make {deploy}-{deploy_target}", shell=True)
-    wait_for_stability(pod)
+    check_call(f"make deploy-{target}", shell=True)
 
     try:
-        if target.startswith("runtime-"):
-            check_call(f"make validate-runtime-image image={target}", shell=True)
-        elif target.startswith("rocm-runtime-"):
-            check_call(
-                f"make validate-runtime-image image={target.replace('rocm-runtime-', 'runtime-rocm-')}", shell=True
-            )
-        elif target.startswith(("rstudio-", "cuda-rstudio-")):
-            check_call(f"make validate-rstudio-image image={target}", shell=True)
-        elif target.startswith("codeserver-"):
-            check_call(f"make validate-codeserver-image image={target}", shell=True)
-        elif target.startswith("rocm-jupyter"):
-            check_call(f"make test-{target.replace('rocm-jupyter-', 'jupyter-rocm-')}", shell=True)
-        else:
-            check_call(f"make test-{target}", shell=True)
+        check_call(f"make test-{target}", shell=True)
     finally:
         # dump a lot of info to the GHA logs
         with gha_log_group("pod and statefulset info"):
@@ -109,7 +72,7 @@ def run_tests(target: str) -> None:
             # regular logs from a running (or finished) pod
             call("kubectl logs --selector=nosuchlabel!=nosuchvalue --all-pods --timestamps", shell=True)
 
-    check_call(f"make un{deploy}-{deploy_target}", shell=True)
+    check_call(f"make undeploy-{target}", shell=True)
 
     print(f"[INFO] Finished testing {target}")
 
@@ -133,22 +96,6 @@ def execute(executor: typing.Callable, args: tuple, kwargs: dict) -> int:
     return result
 
 
-# TODO(jdanek) this is a dumb impl, needs to be improved
-def wait_for_stability(pod: str) -> None:
-    """Waits for the pod to be stable. Often I'm seeing that the probes initially fail.
-    > error: Internal error occurred: error executing command in container: container is not created or running
-    > error: unable to upgrade connection: container not found ("notebook")
-    """
-    timeout = 100
-    for _ in range(3):
-        call(
-            f"timeout {timeout}s bash -c 'until kubectl wait --for=condition=Ready pods --all --timeout 5s; do sleep 1; done'",
-            shell=True,
-        )
-        timeout = 50
-        time.sleep(3)
-
-
 # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#grouping-log-lines
 @contextlib.contextmanager
 def gha_log_group(title):
@@ -170,81 +117,81 @@ def test_make_commands_jupyter(self, mock_execute: unittest.mock.Mock) -> None:
         """Compares the commands with what we had in the openshift/release yaml"""
         run_tests("jupyter-minimal-ubi9-python-3.11")
         commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list]
-        assert "make deploy9-jupyter-minimal-ubi9-python-3.11" in commands
+        assert "make deploy-jupyter-minimal-ubi9-python-3.11" in commands
         assert "make test-jupyter-minimal-ubi9-python-3.11" in commands
-        assert "make undeploy9-jupyter-minimal-ubi9-python-3.11" in commands
+        assert "make undeploy-jupyter-minimal-ubi9-python-3.11" in commands
 
     @unittest.mock.patch("make_test.execute")
     def test_make_commands_jupyter_rocm(self, mock_execute: unittest.mock.Mock) -> None:
         """Compares the commands with what we had in the openshift/release yaml"""
         run_tests("rocm-jupyter-tensorflow-ubi9-python-3.11")
         commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list]
-        assert "make deploy9-jupyter-rocm-tensorflow-ubi9-python-3.11" in commands
-        assert "make test-jupyter-rocm-tensorflow-ubi9-python-3.11" in commands
-        assert "make undeploy9-jupyter-rocm-tensorflow-ubi9-python-3.11" in commands
+        assert "make deploy-rocm-jupyter-tensorflow-ubi9-python-3.11" in commands
+        assert "make test-rocm-jupyter-tensorflow-ubi9-python-3.11" in commands
+        assert "make undeploy-rocm-jupyter-tensorflow-ubi9-python-3.11" in commands
 
     @unittest.mock.patch("make_test.execute")
     def test_make_commands_codeserver(self, mock_execute: unittest.mock.Mock) -> None:
         """Compares the commands with what we had in the openshift/release yaml"""
         run_tests("codeserver-ubi9-python-3.11")
         commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list]
-        assert "make deploy9-codeserver-ubi9-python-3.11" in commands
-        assert "make validate-codeserver-image image=codeserver-ubi9-python-3.11" in commands
-        assert "make undeploy9-codeserver-ubi9-python-3.11" in commands
+        assert "make deploy-codeserver-ubi9-python-3.11" in commands
+        assert "make test-codeserver-ubi9-python-3.11" in commands
+        assert "make undeploy-codeserver-ubi9-python-3.11" in commands
 
     @unittest.mock.patch("make_test.execute")
     def test_make_commands_rstudio(self, mock_execute: unittest.mock.Mock) -> None:
         """Compares the commands with what we had in the openshift/release yaml"""
         run_tests("rstudio-c9s-python-3.11")
         commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list]
-        assert "make deploy-c9s-rstudio-c9s-python-3.11" in commands
-        assert "make validate-rstudio-image image=rstudio-c9s-python-3.11" in commands
-        assert "make undeploy-c9s-rstudio-c9s-python-3.11" in commands
+        assert "make deploy-rstudio-c9s-python-3.11" in commands
+        assert "make test-rstudio-c9s-python-3.11" in commands
+        assert "make undeploy-rstudio-c9s-python-3.11" in commands
 
     @unittest.mock.patch("make_test.execute")
     def test_make_commands_rsudio_rhel(self, mock_execute: unittest.mock.Mock) -> None:
         """Compares the commands with what we had in the openshift/release yaml"""
         run_tests("rstudio-rhel9-python-3.11")
         commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list]
-        assert "make deploy-rhel9-rstudio-rhel9-python-3.11" in commands
-        assert "make validate-rstudio-image image=rstudio-rhel9-python-3.11" in commands
-        assert "make undeploy-rhel9-rstudio-rhel9-python-3.11" in commands
+        assert "make deploy-rstudio-rhel9-python-3.11" in commands
+        assert "make test-rstudio-rhel9-python-3.11" in commands
+        assert "make undeploy-rstudio-rhel9-python-3.11" in commands
 
     @unittest.mock.patch("make_test.execute")
     def test_make_commands_cuda_rstudio(self, mock_execute: unittest.mock.Mock) -> None:
         """Compares the commands with what we had in the openshift/release yaml"""
         run_tests("cuda-rstudio-c9s-python-3.11")
         commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list]
-        assert "make deploy-c9s-rstudio-c9s-python-3.11" in commands
-        assert "make validate-rstudio-image image=cuda-rstudio-c9s-python-3.11" in commands
-        assert "make undeploy-c9s-rstudio-c9s-python-3.11" in commands
+        assert "make deploy-cuda-rstudio-c9s-python-3.11" in commands
+        assert "make test-cuda-rstudio-c9s-python-3.11" in commands
+        assert "make undeploy-cuda-rstudio-c9s-python-3.11" in commands
 
     @unittest.mock.patch("make_test.execute")
     def test_make_commands_cuda_rstudio_rhel(self, mock_execute: unittest.mock.Mock) -> None:
         """Compares the commands with what we had in the openshift/release yaml"""
         run_tests("cuda-rstudio-rhel9-python-3.11")
         commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list]
-        assert "make deploy-rhel9-rstudio-rhel9-python-3.11" in commands
-        assert "make validate-rstudio-image image=cuda-rstudio-rhel9-python-3.11" in commands
-        assert "make undeploy-rhel9-rstudio-rhel9-python-3.11" in commands
+        assert "make deploy-cuda-rstudio-rhel9-python-3.11" in commands
+        assert "make test-cuda-rstudio-rhel9-python-3.11" in commands
+        assert "make undeploy-cuda-rstudio-rhel9-python-3.11" in commands
 
     @unittest.mock.patch("make_test.execute")
     def test_make_commands_runtime(self, mock_execute: unittest.mock.Mock) -> None:
         """Compares the commands with what we had in the openshift/release yaml"""
         run_tests("runtime-datascience-ubi9-python-3.11")
         commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list]
-        assert "make deploy9-runtimes-datascience-ubi9-python-3.11" in commands
-        assert "make validate-runtime-image image=runtime-datascience-ubi9-python-3.11" in commands
-        assert "make undeploy9-runtimes-datascience-ubi9-python-3.11" in commands
+        assert "make deploy-runtime-datascience-ubi9-python-3.11" in commands
+        assert "make test-runtime-datascience-ubi9-python-3.11" in commands
+        assert "make undeploy-runtime-datascience-ubi9-python-3.11" in commands
 
     @unittest.mock.patch("make_test.execute")
     def test_make_commands_rocm_runtime(self, mock_execute: unittest.mock.Mock) -> None:
         """Compares the commands with what we had in the openshift/release yaml"""
         run_tests("rocm-runtime-pytorch-ubi9-python-3.11")
         commands: list[str] = [c[0][1][0] for c in mock_execute.call_args_list]
-        assert "make deploy9-runtimes-rocm-pytorch-ubi9-python-3.11" in commands
-        assert "make validate-runtime-image image=runtime-rocm-pytorch-ubi9-python-3.11" in commands
-        assert "make undeploy9-runtimes-rocm-pytorch-ubi9-python-3.11" in commands
+        assert "make deploy-rocm-runtime-pytorch-ubi9-python-3.11" in commands
+        assert "make test-rocm-runtime-pytorch-ubi9-python-3.11" in commands
+        assert "make undeploy-rocm-runtime-pytorch-ubi9-python-3.11" in commands
 
 
 if __name__ == "__main__":

ci/cached-builds/make_test.py

@@ -1,10 +1,12 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-namePrefix: codeserver-
+namePrefix: codeserver-ubi9-python-3-11-
 resources:
   - pod.yaml
+transformers:
+  - labels.yaml
 images:
-  - name: codeserver-workbench
+  - name: quay.io/opendatahub/workbench-images
     newName: quay.io/opendatahub/workbench-images
     newTag: codeserver-ubi9-python-3.11

codeserver/ubi9-python-3.11/kustomize/base/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: builtin
+kind: LabelTransformer
+metadata:
+  name: add-labels
+labels:
+  app: codeserver-ubi9-python-3-11
+fieldSpecs:
+  - path: metadata/labels
+    create: true
+  - path: spec/template/metadata/labels
+    create: false

codeserver/ubi9-python-3.11/kustomize/base/labels.yaml

@@ -2,17 +2,16 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: pod
-  labels:
-    app: codeserver-image
+  name: workbench
 spec:
   containers:
-    - name: codeserver
-      image: codeserver-workbench
-      command: ["/bin/sh", "-c", "while true ; do date; sleep 5; done;"]
+    - name: workbench
+      image: quay.io/opendatahub/workbench-images
       imagePullPolicy: Always
       ports:
-        - containerPort: 8585
+        - name: workbench-port
+          protocol: TCP
+          containerPort: 8787
       resources:
         limits:
           cpu: 500m

codeserver/ubi9-python-3.11/kustomize/base/pod.yaml

@@ -135,7 +135,7 @@ RUN yum install -y \
     ${NV_CUDNN_PACKAGE_DEV} \
     && yum clean all \
     && rm -rf /var/cache/yum/*
-    
+
 # Set this flag so that libraries can find the location of CUDA
 ENV XLA_FLAGS=--xla_gpu_cuda_data_dir=/usr/local/cuda
 
@@ -146,7 +146,7 @@ WORKDIR /opt/app-root/src
 #########################
 # cuda-jupyter-minimal  #
 #########################
-FROM cuda-base AS cuda-jupyter-minimal  
+FROM cuda-base AS cuda-jupyter-minimal
 
 ARG JUPYTER_REUSABLE_UTILS=jupyter/utils
 ARG MINIMAL_SOURCE_CODE=jupyter/minimal/ubi9-python-3.11
@@ -156,7 +156,7 @@ WORKDIR /opt/app-root/bin
 COPY ${JUPYTER_REUSABLE_UTILS} utils/
 
 COPY ${MINIMAL_SOURCE_CODE}/start-notebook.sh ./
-    
+
 WORKDIR /opt/app-root/src
 
 ENTRYPOINT ["start-notebook.sh"]
@@ -202,7 +202,7 @@ WORKDIR /opt/app-root/src
 FROM cuda-jupyter-datascience AS cuda-jupyter-pytorch
 
 ARG DATASCIENCE_SOURCE_CODE=jupyter/datascience/ubi9-python-3.11
-ARG PYTORCH_SOURCE_CODE=jupyter/pytorch/ubi9-python-3.11
+ARG PYTORCH_SOURCE_CODE=jupyter/cuda/pytorch/ubi9-python-3.11
 
 WORKDIR /opt/app-root/bin
 
@@ -227,11 +227,11 @@ RUN echo "Installing softwares and packages" && \
     # Remove default Elyra runtime-images \
     rm /opt/app-root/share/jupyter/metadata/runtime-images/*.json && \
     # Replace Notebook's launcher, "(ipykernel)" with Python's version 3.x.y \
-    sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \    
+    sed -i -e "s/Python.*/$(python --version | cut -d '.' -f-2)\",/" /opt/app-root/share/jupyter/kernels/python3/kernel.json && \
     # Disable announcement plugin of jupyterlab \
-    jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \    
+    jupyter labextension disable "@jupyterlab/apputils-extension:announcements" && \
     # Apply JupyterLab addons \
-    /opt/app-root/bin/utils/addons/apply.sh && \    
+    /opt/app-root/bin/utils/addons/apply.sh && \
     # Fix permissions to support pip in Openshift environments \
     chmod -R g+w /opt/app-root/lib/python3.11/site-packages && \
     fix-permissions /opt/app-root -P

jupyter/pytorch/ubi9-python-3.11/Dockerfile.cuda renamed to jupyter/cuda/pytorch/ubi9-python-3.11/Dockerfile.cuda

@@ -1,13 +1,13 @@
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-namePrefix: jupyter-pytorch-ubi9-python-3-11-
-commonLabels:
-  app: jupyter-pytorch-ubi9-python-3-11
+namePrefix: cuda-jupyter-pytorch-ubi9-python-3-11-
 resources:
   - service.yaml
   - statefulset.yaml
+transformers:
+  - labels.yaml
 images:
   - name: quay.io/opendatahub/workbench-images
     newName: quay.io/opendatahub/workbench-images
-    newTag: jupyter-pytorch-ubi9-python-3.11
+    newTag: cuda-jupyter-pytorch-ubi9-python-3.11

jupyter/pytorch/ubi9-python-3.11/Pipfile renamed to jupyter/cuda/pytorch/ubi9-python-3.11/Pipfile

@@ -0,0 +1,13 @@
+apiVersion: builtin
+kind: LabelTransformer
+metadata:
+  name: add-labels
+labels:
+  app: cuda-jupyter-pytorch-ubi9-python-3-11
+fieldSpecs:
+  - path: metadata/labels
+    create: true
+  - path: spec/template/metadata/labels
+    create: false
+  - path: spec/selector/matchLabels
+    create: false

jupyter/pytorch/ubi9-python-3.11/Pipfile.lock renamed to jupyter/cuda/pytorch/ubi9-python-3.11/Pipfile.lock

@@ -2,14 +2,14 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: notebook
+  name: workbench
   labels:
-    app: notebook
+    app: workbench
 spec:
   type: ClusterIP
   ports:
     - port: 8888
       protocol: TCP
-      targetPort: notebook-port
+      targetPort: workbench-port
   selector:
-    app: notebook
+    app: workbench