Kueue-enable e2e test cases

Author: dgrove-oss

test/e2e/mnist_pytorch_mcad_job_test.go

@@ -35,8 +35,9 @@ func TestMNISTPyTorchMCAD(t *testing.T) {
 	test := With(t)
 	test.T().Parallel()
 
-	// Create a namespace
+	// Create a namespace and localqueue in that namespace
 	namespace := test.NewTestNamespace()
+	localQueue := EnsureLocalQueue(test, namespace)
 
 	// Test configuration
 	config := &corev1.ConfigMap{
@@ -130,8 +131,9 @@ func TestMNISTPyTorchMCAD(t *testing.T) {
 			Kind:       "AppWrapper",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "mnist",
-			Namespace: namespace.Name,
+			Name:        "mnist",
+			Namespace:   namespace.Name,
+			Annotations: map[string]string{"kueue.x-k8s.io/queue-name": localQueue.Name},
 		},
 		Spec: mcadv1beta2.AppWrapperSpec{
 			Components: []mcadv1beta2.AppWrapperComponent{

test/e2e/mnist_rayjob_mcad_raycluster_test.go

@@ -37,8 +37,9 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) {
 	test := With(t)
 	test.T().Parallel()
 
-	// Create a namespace
+	// Create a namespace and localqueue in that namespace
 	namespace := test.NewTestNamespace()
+	localQueue := EnsureLocalQueue(test, namespace)
 
 	// MNIST training script
 	mnist := &corev1.ConfigMap{
@@ -181,8 +182,9 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) {
 			Kind:       "AppWrapper",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      "ray-cluster",
-			Namespace: namespace.Name,
+			Name:        "ray-cluster",
+			Namespace:   namespace.Name,
+			Annotations: map[string]string{"kueue.x-k8s.io/queue-name": localQueue.Name},
 		},
 		Spec: mcadv1beta2.AppWrapperSpec{
 			Components: []mcadv1beta2.AppWrapperComponent{

test/e2e/setup.sh

@@ -56,3 +56,30 @@ roleRef:
   kind: ClusterRole
   name: mcad-controller-rayclusters
 EOF
+
+
+echo Creating Kueue ResourceFlavor and ClusterQueue
+cat <<EOF | kubectl apply -f -
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: ResourceFlavor
+metadata:
+  name: "default-flavor"
+EOF
+
+cat <<EOF | kubectl apply -f -
+apiVersion: kueue.x-k8s.io/v1beta1
+kind: ClusterQueue
+metadata:
+  name: "e2e-cluster-queue"
+spec:
+  namespaceSelector: {} # match all.
+  resourceGroups:
+  - coveredResources: ["cpu","memory"]
+    flavors:
+    - name: "default-flavor"
+      resources:
+      - name: "cpu"
+        nominalQuota: 4
+      - name: "memory"
+        nominalQuota: "4G"
+EOF

test/e2e/support.go

@@ -18,9 +18,17 @@ package e2e
 
 import (
 	"embed"
+	"fmt"
 
 	"github.com/onsi/gomega"
 	"github.com/project-codeflare/codeflare-common/support"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1"
 )
 
 //go:embed *.py *.txt *.sh
@@ -32,3 +40,21 @@ func ReadFile(t support.Test, fileName string) []byte {
 	t.Expect(err).NotTo(gomega.HaveOccurred())
 	return file
 }
+
+func EnsureLocalQueue(t support.Test, namespace *corev1.Namespace) *kueue.LocalQueue {
+	t.T().Helper()
+
+	lq := &kueue.LocalQueue{
+		TypeMeta:   metav1.TypeMeta{APIVersion: kueue.GroupVersion.String(), Kind: "LocalQueue"},
+		ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("lq-%v", namespace.Name), Namespace: namespace.Name},
+		Spec:       kueue.LocalQueueSpec{ClusterQueue: kueue.ClusterQueueReference("e2e-cluster-queue")},
+	}
+	localQueueResource := kueue.GroupVersion.WithResource("localqueues")
+	lqMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(lq)
+	t.Expect(err).NotTo(gomega.HaveOccurred())
+	unstruct := unstructured.Unstructured{Object: lqMap}
+	_, err = t.Client().Dynamic().Resource(localQueueResource).Namespace(namespace.Name).Create(t.Ctx(), &unstruct, metav1.CreateOptions{})
+	t.Expect(client.IgnoreAlreadyExists(err)).NotTo(gomega.HaveOccurred())
+
+	return lq
+}