Debug

jiripetrlik · jiripetrlik · commit 18a81dfc6051 · 2024-11-24T23:21:42.000+01:00
diff --git a/.github/workflows/additional_demo_notebook_tests.yaml b/.github/workflows/additional_demo_notebook_tests.yaml
@@ -14,7 +14,7 @@ env:
 
 jobs:
   verify-hf_interactive:
-    # if: ${{ github.event.label.name == 'test-additional-notebooks' }}
+    if: ${{ github.event.label.name == 'test' }}
     runs-on: ubuntu-20.04-4core
 
     steps:
@@ -85,6 +85,8 @@ jobs:
           # Remove login/logout cells, as KinD doesn't support authentication using token
           jq -r 'del(.cells[] | select(.source[] | contains("Create authentication object for user permissions")))' hf_interactive.ipynb > hf_interactive.ipynb.tmp && mv hf_interactive.ipynb.tmp hf_interactive.ipynb
           jq -r 'del(.cells[] | select(.source[] | contains("auth.logout()")))' hf_interactive.ipynb > hf_interactive.ipynb.tmp && mv hf_interactive.ipynb.tmp hf_interactive.ipynb
+          # Rewrite cluster_uri() to local_client_url() to retrieve client URL available out of cluster, as the test is executed outside of cluster
+          sed -i "s/cluster_uri()/local_client_url()/g" hf_interactive.ipynb
           # Replace async logs with waiting for job to finish, async logs don't work properly in papermill
           JOB_WAIT=$(jq -r '.' ${GITHUB_WORKSPACE}/.github/resources/wait_for_job_cell.json)
           jq --argjson job_wait "$JOB_WAIT" -r '(.cells[] | select(.source[] | contains("async for lines in client.tail_job_logs"))) |= $job_wait' hf_interactive.ipynb > hf_interactive.ipynb.tmp && mv hf_interactive.ipynb.tmp hf_interactive.ipynb
@@ -136,7 +138,7 @@ jobs:
   verify-local_interactive:
     # if: ${{ github.event.label.name == 'test-additional-notebooks' }}
     # runs-on: ubuntu-20.04-4core
-    runs-on: ubuntu-20.04-4core
+    runs-on: ubuntu-20.04-4core-gpu
 
     steps:
       - name: Checkout code
@@ -174,9 +176,17 @@ jobs:
           python-version: '3.9'
           cache: 'pip' # caching pip dependencies
 
+      - name: Setup NVidia GPU environment for KinD
+        uses: ./common/github-actions/nvidia-gpu-setup
+
       - name: Setup and start KinD cluster
         uses: ./common/github-actions/kind
 
+      - name: Install NVidia GPU operator for KinD
+        uses: ./common/github-actions/nvidia-gpu-operator
+        with:
+          enable-time-slicing: 'true'
+
       - name: Deploy CodeFlare stack
         id: deploy
         run: |
@@ -188,6 +198,11 @@ jobs:
           kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager
           cd ..
 
+      - name: Install MINIO
+        run: |
+          kubectl apply -f ./tests/e2e/minio_deployment.yaml
+          kubectl wait --timeout=120s --for=condition=Available=true deployment -n default minio
+
       - name: Setup Additional demo notebooks execution
         run: |
           echo "Installing papermill and dependencies..."
@@ -205,14 +220,41 @@ jobs:
           # Remove login/logout cells, as KinD doesn't support authentication using token
           jq -r 'del(.cells[] | select(.source[] | contains("Create authentication object and log in to desired user account")))' local_interactive.ipynb > local_interactive.ipynb.tmp && mv local_interactive.ipynb.tmp local_interactive.ipynb
           jq -r 'del(.cells[] | select(.source[] | contains("auth.logout()")))' local_interactive.ipynb > local_interactive.ipynb.tmp && mv local_interactive.ipynb.tmp local_interactive.ipynb
+          # Rewrite cluster_uri() to local_client_url() to retrieve client URL available out of cluster, as the test is executed outside of cluster
+          sed -i "s/cluster_uri()/local_client_url()/" local_interactive.ipynb
           # Replace async logs with waiting for job to finish, async logs don't work properly in papermill
           JOB_WAIT=$(jq -r '.' ${GITHUB_WORKSPACE}/.github/resources/wait_for_job_cell.json)
           jq --argjson job_wait "$JOB_WAIT" -r '(.cells[] | select(.source[] | contains("async for lines in client.tail_job_logs"))) |= $job_wait' local_interactive.ipynb > local_interactive.ipynb.tmp && mv local_interactive.ipynb.tmp local_interactive.ipynb
           # Set explicit namespace as SDK need it (currently) to resolve local queues
-          sed -i "s/worker_cpu_requests=1,/worker_cpu_requests=1, namespace='default',/" local_interactive.ipynb
+          sed -i "s/worker_cpu_requests=1,/worker_cpu_requests='250m', namespace='default',/" local_interactive.ipynb
           # Run notebook
-          poetry run papermill local_interactive.ipynb hf_interactive_out.ipynb --log-output --execution-timeout 1200
+          poetry run papermill local_interactive.ipynb local_interactive_out.ipynb --log-output --execution-timeout 1200
+        env:
+           GRPC_DNS_RESOLVER: "native"
         working-directory: demo-notebooks/additional-demos
+      # - name: Run 2_basic_interactive.ipynb
+      #   run: |
+      #     set -euo pipefail
+
+      #     # Remove login/logout cells, as KinD doesn't support authentication using token
+      #     jq -r 'del(.cells[] | select(.source[] | contains("Create authentication object for user permissions")))' 2_basic_interactive.ipynb > 2_basic_interactive.ipynb.tmp && mv 2_basic_interactive.ipynb.tmp 2_basic_interactive.ipynb
+      #     jq -r 'del(.cells[] | select(.source[] | contains("auth.logout()")))' 2_basic_interactive.ipynb > 2_basic_interactive.ipynb.tmp && mv 2_basic_interactive.ipynb.tmp 2_basic_interactive.ipynb
+      #     # Rewrite cluster_uri() to local_client_url() to retrieve client URL available out of cluster, as the test is executed outside of cluster
+      #     sed -i "s/cluster_uri()/local_client_url()/" 2_basic_interactive.ipynb
+      #     # Set explicit namespace as SDK need it (currently) to resolve local queues
+      #     sed -i "s/head_cpu_limits=1,/head_cpu_limits=1, namespace='default',/" 2_basic_interactive.ipynb
+      #     # Add MINIO related modules to runtime environment
+      #     sed -i "s/\\\\\"transformers/\\\\\"s3fs\\\\\", \\\\\"pyarrow\\\\\", \\\\\"transformers/" 2_basic_interactive.ipynb
+      #     # Replace markdown cell with remote configuration for MINIO
+      #     MINIO_CONFIG=$(jq -r '.' ${GITHUB_WORKSPACE}/.github/resources/minio_remote_config_cell.json)
+      #     jq --argjson minio_config "$MINIO_CONFIG" -r '(.cells[] | select(.source[] | contains("Now that we are connected"))) |= $minio_config' 2_basic_interactive.ipynb > 2_basic_interactive.ipynb.tmp && mv 2_basic_interactive.ipynb.tmp 2_basic_interactive.ipynb
+      #     # Configure persistent storage for Ray trainer
+      #     sed -i -E "s/# run_config.*\)/, run_config=ray.get(get_minio_run_config.remote())/" 2_basic_interactive.ipynb
+      #     # Run notebook
+      #     poetry run papermill 2_basic_interactive.ipynb 2_basic_interactive_out.ipynb --log-output --execution-timeout 1200
+      #   env:
+      #     GRPC_DNS_RESOLVER: "native"
+      #   working-directory: demo-notebooks/guided-demos
 
       - name: Print CodeFlare operator logs
         if: always() && steps.deploy.outcome == 'success'
@@ -249,7 +291,7 @@ jobs:
             ${{ env.TEMP_DIR }}/**/*.log
 
   verify-ray_job_client:
-    # if: ${{ github.event.label.name == 'test-additional-notebooks' }}
+    if: ${{ github.event.label.name == 'test' }}
     runs-on: ubuntu-20.04-4core
 
     steps:
@@ -319,6 +361,8 @@ jobs:
           # Remove login/logout cells, as KinD doesn't support authentication using token
           jq -r 'del(.cells[] | select(.source[] | contains("Create authentication object for user permissions")))' ray_job_client.ipynb > ray_job_client.ipynb.tmp && mv ray_job_client.ipynb.tmp ray_job_client.ipynb
           jq -r 'del(.cells[] | select(.source[] | contains("auth.logout()")))' ray_job_client.ipynb > ray_job_client.ipynb.tmp && mv ray_job_client.ipynb.tmp ray_job_client.ipynb
+          # Rewrite cluster_uri() to local_client_url() to retrieve client URL available out of cluster, as the test is executed outside of cluster
+          sed -i "s/cluster_uri()/local_client_url()/g" ray_job_client.ipynb
           # Replace async logs with waiting for job to finish, async logs don't work properly in papermill
           JOB_WAIT=$(jq -r '.' ${GITHUB_WORKSPACE}/.github/resources/wait_for_job_cell.json)
           jq --argjson job_wait "$JOB_WAIT" -r '(.cells[] | select(.source[] | contains("async for lines in client.tail_job_logs"))) |= $job_wait' ray_job_client.ipynb > ray_job_client.ipynb.tmp && mv ray_job_client.ipynb.tmp ray_job_client.ipynb
diff --git a/.github/workflows/guided_notebook_tests.yaml b/.github/workflows/guided_notebook_tests.yaml
@@ -3,6 +3,7 @@ name: Guided notebooks tests
 on:
   pull_request:
     branches: [ main ]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.head_ref }}-${{ github.workflow }}
@@ -246,7 +247,7 @@ jobs:
             ${{ env.TEMP_DIR }}/**/*.log
 
   verify-2_basic_interactive:
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'test-guided-notebooks') }}
+    #if: ${{ contains(github.event.pull_request.labels.*.name, 'test-guided-notebooks') }}
     runs-on: ubuntu-20.04-4core-gpu
 
     steps:
diff --git a/demo-notebooks/additional-demos/local_interactive.ipynb b/demo-notebooks/additional-demos/local_interactive.ipynb
@@ -54,12 +54,16 @@
    "source": [
     "# Create and submit our cluster\n",
     "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n",
-    "cluster_name = \"hfgputest-1\"\n",
+    "cluster_name = \"hfgputest\"\n",
     "\n",
     "cluster = Cluster(ClusterConfiguration(\n",
     "                                       name=cluster_name,\n",
     "                                       head_extended_resource_requests={'nvidia.com/gpu':0}, # For GPU enabled workloads set the head_extended_resource_requests and worker_extended_resource_requests\n",
     "                                       worker_extended_resource_requests={'nvidia.com/gpu':0},\n",
+    "                                       head_cpu_requests=1,\n",
+    "                                       head_cpu_limits=1,\n",
+    "                                       head_memory_requests=6,\n",
+    "                                       head_memory_limits=6,\n",
     "                                       num_workers=1,\n",
     "                                       worker_cpu_requests=1,\n",
     "                                       worker_cpu_limits=1,\n",
@@ -95,6 +99,16 @@
     "cluster.wait_ready()"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2c36c2b5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "cluster.details()"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "12eef53c",
@@ -127,8 +141,29 @@
    "source": [
     "import ray\n",
     "\n",
-    "ray.shutdown()\n",
-    "ray.init(address=cluster.local_client_url(), logging_level=\"DEBUG\")"
+    "ray.shutdown()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "dd56c33c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ray_cluster_uri = cluster.cluster_uri()\n",
+    "print(f\"Ray cluster URI: {ray_cluster_uri}\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3dab4124",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "runtime_env = {\"pip\": [\"transformers==4.41.2\", \"datasets==2.17.0\", \"accelerate==0.31.0\", \"scikit-learn==1.5.0\"]}\n",
+    "ray.init(address=ray_cluster_uri, runtime_env=runtime_env)"
    ]
   },
   {
