Run PR check for guided notebooks

Author: sutaakar

.github/workflows/e2e_tests.yaml renamed to .github/workflows/guided_notebook_tests.yaml

@@ -1,4 +1,4 @@
-name: e2e
+name: Guided notebooks tests
 
 on:
   pull_request:
@@ -76,6 +76,8 @@ jobs:
 
       - name: Install NVidia GPU operator for KinD
         uses: ./common/github-actions/nvidia-gpu-operator
+        with:
+          enable-time-slicing: 'true'
 
       - name: Deploy CodeFlare stack
         id: deploy
@@ -113,46 +115,70 @@ jobs:
           kubectl create clusterrolebinding sdk-user-list-secrets --clusterrole=list-secrets --user=sdk-user
           kubectl config use-context sdk-user
 
-      - name: Run e2e tests
+      - name: Setup Guided notebooks execution
         run: |
-          export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
-          echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
+          echo "Installing papermill and dependencies..."
+          pip install poetry papermill ipython ipykernel
+          # Disable virtualenv due to problems using packaged in virtualenv in papermill
+          poetry config virtualenvs.create false
 
-          set -euo pipefail
-          pip install poetry
+          echo "Installing SDK..."
           poetry install --with test,docs
-          echo "Running e2e tests..."
-          poetry run pytest -v -s ./tests/e2e -m 'kind and nvidia_gpu' > ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log 2>&1
+
+      - name: Run 0_basic_ray.ipynb
+        run: |
+          set -euo pipefail
+
+          jq -r 'del(.cells[] | select(.source[] | contains("Create authentication object for user permissions")))' 0_basic_ray.ipynb > 0_basic_ray.ipynb.tmp && mv 0_basic_ray.ipynb.tmp 0_basic_ray.ipynb
+          jq -r 'del(.cells[] | select(.source[] | contains("auth.logout()")))' 0_basic_ray.ipynb > 0_basic_ray.ipynb.tmp && mv 0_basic_ray.ipynb.tmp 0_basic_ray.ipynb
+          poetry run papermill 0_basic_ray.ipynb 0_basic_ray_out.ipynb --log-output --execution-timeout 600
+        working-directory: demo-notebooks/guided-demos
+
+      - name: Run 1_cluster_job_client.ipynb
+        run: |
+          set -euo pipefail
+
+          jq -r 'del(.cells[] | select(.source[] | contains("Create authentication object for user permissions")))' 1_cluster_job_client.ipynb > 1_cluster_job_client.ipynb.tmp && mv 1_cluster_job_client.ipynb.tmp 1_cluster_job_client.ipynb
+          jq -r 'del(.cells[] | select(.source[] | contains("auth.logout()")))' 1_cluster_job_client.ipynb > 1_cluster_job_client.ipynb.tmp && mv 1_cluster_job_client.ipynb.tmp 1_cluster_job_client.ipynb
+          poetry run papermill 1_cluster_job_client.ipynb 1_cluster_job_client_out.ipynb --log-output --execution-timeout 600
         env:
           GRPC_DNS_RESOLVER: "native"
+        working-directory: demo-notebooks/guided-demos
 
       - name: Switch to kind-cluster context to print logs
         if: always() && steps.deploy.outcome == 'success'
         run: kubectl config use-context kind-cluster
 
-      - name: Print Pytest output log
+      - name: Print debug info
         if: always() && steps.deploy.outcome == 'success'
         run: |
-          echo "Printing Pytest output logs"
-          cat ${CODEFLARE_TEST_OUTPUT_DIR}/pytest_output.log
+          echo "Printing debug info"
+          kubectl describe pods -n default
 
       - name: Print CodeFlare operator logs
         if: always() && steps.deploy.outcome == 'success'
         run: |
           echo "Printing CodeFlare operator logs"
-          kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log
+          kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${TEMP_DIR}/codeflare-operator.log
+
+      - name: Print Kueue operator logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing Kueue operator logs"
+          KUEUE_CONTROLLER_POD=$(kubectl get pods -n kueue-system | grep kueue-controller | awk '{print $1}')
+          kubectl logs -n kueue-system --tail -1 ${KUEUE_CONTROLLER_POD} | tee ${TEMP_DIR}/kueue.log
 
       - name: Print KubeRay operator logs
         if: always() && steps.deploy.outcome == 'success'
         run: |
           echo "Printing KubeRay operator logs"
-          kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kuberay.log
+          kubectl logs -n ray-system --tail -1 -l app.kubernetes.io/name=kuberay | tee ${TEMP_DIR}/kuberay.log
 
       - name: Export all KinD pod logs
         uses: ./common/github-actions/kind-export-logs
         if: always() && steps.deploy.outcome == 'success'
         with:
-          output-directory: ${CODEFLARE_TEST_OUTPUT_DIR}
+          output-directory: ${TEMP_DIR}
 
       - name: Upload logs
         uses: actions/upload-artifact@v4
@@ -161,4 +187,4 @@ jobs:
           name: logs
           retention-days: 10
           path: |
-            ${{ env.CODEFLARE_TEST_OUTPUT_DIR }}/**/*.log
+            ${{ env.TEMP_DIR }}/**/*.log

demo-notebooks/guided-demos/0_basic_ray.ipynb

@@ -63,10 +63,12 @@
     "cluster = Cluster(ClusterConfiguration(\n",
     "    name='raytest', \n",
     "    namespace='default', # Update to your namespace\n",
+    "    head_cpus='500m',\n",
+    "    head_memory=2,\n",
     "    head_gpus=0, # For GPU enabled workloads set the head_gpus and num_gpus\n",
     "    num_gpus=0,\n",
     "    num_workers=2,\n",
-    "    min_cpus=1,\n",
+    "    min_cpus='250m',\n",
     "    max_cpus=1,\n",
     "    min_memory=4,\n",
     "    max_memory=4,\n",

demo-notebooks/guided-demos/1_cluster_job_client.ipynb

@@ -45,10 +45,12 @@
     "cluster = Cluster(ClusterConfiguration(\n",
     "    name='jobtest',\n",
     "    namespace='default', # Update to your namespace\n",
+    "    head_cpus='500m',\n",
+    "    head_memory=2,\n",
     "    head_gpus=1, # For GPU enabled workloads set the head_gpus and num_gpus\n",
     "    num_gpus=1,\n",
     "    num_workers=2,\n",
-    "    min_cpus=1,\n",
+    "    min_cpus='250m',\n",
     "    max_cpus=1,\n",
     "    min_memory=4,\n",
     "    max_memory=4,\n",