Add AppWrapper v1beta2 CRD and controllers to Codeflare operator

.github/workflows/e2e_tests.yaml

@@ -86,6 +86,13 @@ jobs:
           echo "Printing CodeFlare operator logs"
           kubectl logs -n openshift-operators --tail -1 -l app.kubernetes.io/name=codeflare-operator | tee ${CODEFLARE_TEST_OUTPUT_DIR}/codeflare-operator.log
 
+      - name: Print Kueue operator logs
+        if: always() && steps.deploy.outcome == 'success'
+        run: |
+          echo "Printing Kueue operator logs"
+          KUEUE_CONTROLLER_POD=$(kubectl get pods -n kueue-system | grep kueue-controller | awk '{print $1}')
+          kubectl logs -n kueue-system --tail -1 ${KUEUE_CONTROLLER_POD} | tee ${CODEFLARE_TEST_OUTPUT_DIR}/kueue.log
+
       - name: Print KubeRay operator logs
         if: always() && steps.deploy.outcome == 'success'
         run: |

.github/workflows/project-codeflare-release.yml

@@ -13,9 +13,15 @@ on:
       codeflare-sdk-version:
         description: 'Version of CodeFlare-SDK to be released (for example: v0.0.0)'
         required: true
+      appwrapper-version:
+        description: 'Tested version of AppWrapper (for example: v0.0.0)'
+        required: true
       kuberay-version:
         description: 'Tested version of KubeRay (for example: v0.0.0)'
         required: true
+      kueue-version:
+        description: 'Tested version of Kueue (for example: v0.0.0)'
+        required: true
       is-stable:
         description: 'Select if the built images should be tagged as stable'
         required: true
@@ -43,14 +49,16 @@ jobs:
         echo "Operator Version: ${{ github.event.inputs.operator-version }}"
         echo "Replaces: ${{ github.event.inputs.replaces }}"
         echo "CodeFlare SDK Version: ${{ github.event.inputs.codeflare-sdk-version }}"
+        echo "Tested AppWrapper Version: ${{ github.event.inputs.appwrapper-version }}"
         echo "Tested KubeRay Version: ${{ github.event.inputs.kuberay-version }}"
+        echo "Tested Kueue Version: ${{ github.event.inputs.kueue-version }}"
         echo "Is Stable: ${{ github.event.inputs.is-stable }}"
         echo "CodeFlare Repository Organization: ${{ github.event.inputs.codeflare-repository-organization }}"
         echo "Quay Organization: ${{ github.event.inputs.quay-organization }}"
         echo "Community Operators Prod Organization: ${{ github.event.inputs.community-operators-prod-organization }}"
 
   release-codeflare-sdk:
-    needs: check-kuberay-version
+    needs: [check-appwrapper-version, check-kuberay-version, check-kueue-version]
     runs-on: ubuntu-latest
 
     steps:
@@ -90,7 +98,7 @@ jobs:
     steps:
     - name: Release CodeFlare operator
       run: |
-        gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }}
+        gh workflow run tag-and-build.yml --repo ${{ github.event.inputs.codeflare-repository-organization }}/codeflare-operator --ref ${{ github.ref }} --field is-stable=${{ github.event.inputs.is-stable }} --field version=${{ github.event.inputs.operator-version }} --field replaces=${{ github.event.inputs.replaces }} --field codeflare-sdk-version=${{ github.event.inputs.codeflare-sdk-version }} --field appwrapper-version=${{ github.event.inputs.appwrapper-version }} --field kuberay-version=${{ github.event.inputs.kuberay-version }} --field kueue-version=${{ github.event.inputs.kueue-version }} --field quay-organization=${{ github.event.inputs.quay-organization }} --field community-operators-prod-fork-organization=${{ github.event.inputs.codeflare-repository-organization }} --field community-operators-prod-organization=${{ github.event.inputs.community-operators-prod-organization }}
       env:
         GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }}
       shell: bash
@@ -105,6 +113,20 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.CODEFLARE_MACHINE_ACCOUNT_TOKEN }}
       shell: bash
 
+  check-appwrapper-version:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check if AppWrapper release exists
+      run: |
+        status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/project-codeflare/appwrapper/releases/tag/${{ github.event.inputs.appwrapper-version }})
+        if [[ "$status_code" == "200" ]]; then
+          echo "AppWrapper release with version ${{ github.event.inputs.appwrapper-version }} exists and available to use."
+        else
+          echo "AppWrapper release with version ${{ github.event.inputs.appwrapper-version }} does not exist. Please select an existing version."
+          exit 1
+        fi
+
   check-kuberay-version:
     runs-on: ubuntu-latest
 
@@ -118,3 +140,17 @@ jobs:
           echo "KubeRay release with version ${{ github.event.inputs.kuberay-version }} does not exist. Please select an existing version."
           exit 1
         fi
+
+  check-kueue-version:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Check if Kueue release exists
+      run: |
+        status_code=$(curl -s -o /dev/null -w "%{http_code}" https://github.com/kubernetes-sigs/kueue/releases/tag/${{ github.event.inputs.kueue-version }})
+        if [[ "$status_code" == "200" ]]; then
+          echo "Kueue release with version ${{ github.event.inputs.kueue-version }} exists and available to use."
+        else
+          echo "Kueue release with version ${{ github.event.inputs.kueue-version }} does not exist. Please select an existing version."
+          exit 1
+        fi

.github/workflows/tag-and-build.yml

@@ -16,10 +16,18 @@ on:
         description: 'Published version of CodeFlare-SDK'
         required: true
         default: 'v0.0.0-dev'
+      appwrapper-version:
+        description: 'Tested version of AppWrapper'
+        required: true
+        default: 'v0.0.0-dev'
       kuberay-version:
         description: 'Tested version of KubeRay'
         required: true
         default: 'v0.0.0-dev'
+      kueue-version:
+        description: 'Tested version of Kueue'
+        required: true
+        default: 'v0.0.0-dev'
       is-stable:
         description: 'Select if the built image should be tagged as stable'
         required: true
@@ -83,12 +91,22 @@ jobs:
       run: |
         sed -i -E "s|(.*CodeFlare Operator.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.version }}\2${{ github.event.inputs.version }}\3|" README.md
         sed -i -E "s|(.*CodeFlare-SDK.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.codeflare-sdk-version }}\2${{ github.event.inputs.codeflare-sdk-version }}\3|" README.md
+        sed -i -E "s|(.*AppWrapper.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.appwrapper-version }}\2${{ github.event.inputs.appwrapper-version }}\3|" README.md
         sed -i -E "s|(.*KubeRay.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kuberay-version }}\2${{ github.event.inputs.kuberay-version }}\3|" README.md
+        sed -i -E "s|(.*Kueue.*\[).*(\].*releases/tag/).*(\).*)|\1${{ github.event.inputs.kueue-version }}\2${{ github.event.inputs.kueue-version }}\3|" README.md
+
+    - name: Adjust AppWrapper dependency in the code
+      run: |
+        sed -i -E "s/(.*APPWRAPPER_VERSION \?= ).*/\1${{ github.event.inputs.appwrapper-version }}/" Makefile
 
     - name: Adjust Kuberay dependency in the code
       run: |
         sed -i -E "s/(.*KUBERAY_VERSION \?= ).*/\1${{ github.event.inputs.kuberay-version }}/" Makefile
 
+    - name: Adjust Kueue dependency in the code
+      run: |
+        sed -i -E "s/(.*KUEUE_VERSION \?= ).*/\1${{ github.event.inputs.kueue-version }}/" Makefile
+
     - name: Update image version in params.env
       run: |
         VERSION=${{ github.event.inputs.version }} perl -i -pe 's/:(.*)$/:$ENV{"VERSION"}/' config/manager/params.env

Makefile

@@ -11,6 +11,15 @@ PREVIOUS_VERSION ?= v0.0.0-dev
 VERSION ?= v0.0.0-dev
 BUNDLE_VERSION ?= $(VERSION:v%=%)
 
+# APPWRAPPER_VERSION defines the default version of the AppWrapper controller
+ APPWRAPPER_VERSION ?= v0.11.1
+ APPWRAPPER_REPO ?= github.com/project-codeflare/appwrapper
+ # Upstream AppWrapper is currently only creating release tags of the form `vX.Y.Z` (i.e the version)
+ APPWRAPPER_CRD ?= ${APPWRAPPER_REPO}/config/crd?ref=${APPWRAPPER_VERSION}
+
+ # KUEUE_VERSION defines the default version of Kueue (used for testing)
+ KUEUE_VERSION ?= v0.6.2
+
 # KUBERAY_VERSION defines the default version of the KubeRay operator (used for testing)
 KUBERAY_VERSION ?= v1.1.0
 
@@ -127,8 +136,11 @@ help: ## Display this help.
 
 # this encounters sed issues on MacOS, quick fix is to use gsed or to escape the parentheses i.e. \( \)
 .PHONY: manifests
-manifests: controller-gen kustomize install-yq ## Generate RBAC objects.
+manifests: controller-gen kustomize install-yq ## Generate RBAC objects and import upstream CRDs.
 	$(CONTROLLER_GEN) rbac:roleName=manager-role webhook paths="./..."
+	$(SED) -i -E "s|(- )\${APPWRAPPER_REPO}.*|\1\${APPWRAPPER_CRD}|" config/crd/appwrapper/kustomization.yaml
+	$(KUSTOMIZE) build config/crd/appwrapper | $(YQ) -s '"crd-" + .spec.names.singular' --no-doc
+	mv crd-*.yml config/crd
 
 .PHONY: fmt
 fmt: ## Run go fmt against code.
@@ -383,7 +395,7 @@ kind-e2e: ## Set up e2e KinD cluster
 
 .PHONY: setup-e2e
 setup-e2e: ## Set up e2e tests.
-	KUBERAY_VERSION=$(KUBERAY_VERSION) test/e2e/setup.sh
+	KUBERAY_VERSION=$(KUBERAY_VERSION) KUEUE_VERSION=$(KUEUE_VERSION) test/e2e/setup.sh
 
 .PHONY: imports
 imports: openshift-goimports ## Organize imports in go files using openshift-goimports. Example: make imports

README.md

@@ -9,8 +9,10 @@ CodeFlare Stack Compatibility Matrix
 | Component                    | Version                                                                                           |
 |------------------------------|---------------------------------------------------------------------------------------------------|
 | CodeFlare Operator           | [v1.4.1](https://github.com/project-codeflare/codeflare-operator/releases/tag/v1.4.1)             |
-| CodeFlare-SDK                | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1)                  |
-| KubeRay                      | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0)                              |
+| CodeFlare-SDK                | [v0.16.1](https://github.com/project-codeflare/codeflare-sdk/releases/tag/v0.16.1)                |
+| AppWrapper                   | [v0.11.1](https://github.com/project-codeflare/appwrapper/releases/tag/v0.11.1)                   |
+| KubeRay                      | [v1.1.0](https://github.com/opendatahub-io/kuberay/releases/tag/v1.1.0)                           |
+| Kueue                        | [v0.6.2](https://github.com/opendatahub-io/kueue/releases/tag/v0.6.2)                             |
 <!-- Compatibility Matrix end -->
 
 ## Development
@@ -40,20 +42,15 @@ The e2e tests can be executed locally by running the following commands:
    Some e2e tests cover the access to services via Ingresses, as end-users would do, which requires access to the Ingress controller load balancer by its IP.
    For it to work on macOS, this requires installing [docker-mac-net-connect](https://github.com/chipmk/docker-mac-net-connect).
 
-2. Start the operator locally:
-
-    ```bash
-    NAMESPACE=default make run
-    ```
-
-   Alternatively, You can run the operator from your IDE / debugger.
-
-3. Set up the test CodeFlare stack:
+2. Setup the rest of the CodeFlare stack.
 
    ```bash
    make setup-e2e
    ```
 
+   [!NOTE]
+   Kueue will only activate its Ray integration if KubeRay is installed before Kueue (as done by this make target).
+
    [!NOTE]
    In OpenShift the KubeRay operator pod gets random user assigned. This user is then used to run Ray cluster.
    However the random user assigned by OpenShift doesn't have rights to store dataset downloaded as part of test execution, causing tests to fail.
@@ -73,6 +70,14 @@ The e2e tests can be executed locally by running the following commands:
       - 'system:serviceaccount:$(namespace):kuberay-operator'
     ```
 
+3. Start the operator locally:
+
+    ```bash
+    NAMESPACE=default make run
+    ```
+
+   Alternatively, You can run the operator from your IDE / debugger.
+
 4.  In a separate terminal, set your output directory for test files, and run the e2e suite:
     ```bash
     export CODEFLARE_TEST_OUTPUT_DIR=<your_output_directory>

config/crd/appwrapper/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- github.com/project-codeflare/appwrapper/config/crd?ref=v0.11.1