e2e test

Makefile

@@ -42,6 +42,14 @@ ENVTEST_K8S_VERSION = 1.24
 
 INTEGRATION_TARGET ?= ./test/integration/...
 
+E2E_TARGET ?= ./test/e2e/...
+
+E2E_KIND_VERSION ?= kindest/node:v1.23.12
+
+# For local testing, we should allow user to use different kind cluster name
+# Default will delete default kind cluster
+KIND_CLUSTER_NAME ?= kind
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -135,6 +143,11 @@ test-integration: manifests generate fmt vet envtest ginkgo ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) --arch=amd64 use $(ENVTEST_K8S_VERSION) -p path)" \
 	$(GINKGO) --junit-report=junit.xml --output-dir=$(ARTIFACTS) -v $(INTEGRATION_TARGET)
 
+USE_EXISTING_CLUSTER ?= false
+.PHONY: test-e2e-kind
+test-e2e-kind: manifests generate fmt vet envtest ginkgo kind-image-build
+	E2E_KIND_VERSION=$(E2E_KIND_VERSION) KIND_CLUSTER_NAME=$(KIND_CLUSTER_NAME) USE_EXISTING_CLUSTER=$(USE_EXISTING_CLUSTER) ARTIFACTS=$(ARTIFACTS) IMAGE_TAG=$(IMAGE_TAG) ./hack/e2e-test.sh
+
 .PHONY: ci-lint
 ci-lint: golangci-lint
 	$(GOLANGCI_LINT) run --timeout 7m0s
@@ -177,6 +190,11 @@ image-build:
 image-push: PUSH=--push
 image-push: image-build
 
+.PHONY: kind-image-build
+kind-image-build: PLATFORMS=linux/amd64
+kind-image-build: IMAGE_BUILD_EXTRA_OPTS=--load
+kind-image-build: kind image-build
+
 ##@ Deployment
 
 ifndef ignore-not-found
@@ -248,3 +266,8 @@ GOTESTSUM = $(shell pwd)/bin/gotestsum
 .PHONY: gotestsum
 gotestsum: ## Download gotestsum locally if necessary.
 	@GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install gotest.tools/[email protected]
+KIND = $(shell pwd)/bin/kind
+.PHONY: kind
+kind:
+	@GOBIN=$(PROJECT_DIR)/bin GO111MODULE=on $(GO_CMD) install sigs.k8s.io/[email protected]
+

hack/e2e-test.sh

@@ -0,0 +1,29 @@
+export KUSTOMIZE=$PWD/bin/kustomize
+export GINKGO=$PWD/bin/ginkgo
+export KIND=$PWD/bin/kind
+function cleanup {
+    if [ $USE_EXISTING_CLUSTER == 'false' ] 
+    then 
+        $KIND delete cluster --name $KIND_CLUSTER_NAME
+    fi
+    (cd config/components/manager && $KUSTOMIZE edit set image controller=gcr.io/k8s-staging-kueue/kueue:main)
+}
+function startup {
+    if [ $USE_EXISTING_CLUSTER == 'false' ] 
+    then 
+        $KIND create cluster --name $KIND_CLUSTER_NAME --image $E2E_KIND_VERSION
+    fi
+}
+function kind_load {
+    $KIND load docker-image $IMAGE_TAG --name $KIND_CLUSTER_NAME
+}
+function kueue_deploy {
+    (cd config/components/manager && $KUSTOMIZE edit set image controller=$IMAGE_TAG)
+    kubectl apply -k test/e2e/config
+}
+trap cleanup EXIT
+startup
+kind_load
+kueue_deploy
+$GINKGO --junit-report=junit.xml --output-dir=$ARTIFACTS -v ./test/e2e/...
+

pkg/util/testing/wrappers.go

@@ -110,6 +110,16 @@ func (j *JobWrapper) Request(r corev1.ResourceName, v string) *JobWrapper {
 	return j
 }
 
+func (j *JobWrapper) Image(name string, image string, args []string) *JobWrapper {
+	j.Spec.Template.Spec.Containers[0] = corev1.Container{
+		Name:      name,
+		Image:     image,
+		Args:      args,
+		Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{}},
+	}
+	return j
+}
+
 // PriorityClassWrapper wraps a PriorityClass.
 type PriorityClassWrapper struct {
 	schedulingv1.PriorityClass

test/e2e/config/image_pull_policy.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        imagePullPolicy: IfNotPresent
+

test/e2e/config/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+bases:
+- ../../../config/default
+
+patchesStrategicMerge:
+- image_pull_policy.yaml
+

test/e2e/e2e_test.go

@@ -0,0 +1,122 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	kueue "sigs.k8s.io/kueue/apis/kueue/v1alpha2"
+	"sigs.k8s.io/kueue/pkg/util/testing"
+	"sigs.k8s.io/kueue/pkg/workload"
+	"sigs.k8s.io/kueue/test/e2e/framework"
+)
+
+// +kubebuilder:docs-gen:collapse=Imports
+
+var _ = ginkgo.Describe("Kueue", func() {
+	var ns *corev1.Namespace
+	var sampleJob *batchv1.Job
+	ginkgo.BeforeEach(func() {
+		ns = &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				GenerateName: "e2e-",
+			},
+		}
+		gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed())
+		sampleJob = testing.MakeJob("test-job", ns.Name).Request("cpu", "1").Request("memory", "20Mi").
+			Image("sleep", "gcr.io/k8s-staging-perf-tests/sleep:v0.0.3", []string{"5s"}).Obj()
+		annotations := map[string]string{
+			"kueue.x-k8s.io/queue-name": "main",
+		}
+		sampleJob.ObjectMeta.Annotations = annotations
+
+		gomega.Expect(k8sClient.Create(ctx, sampleJob)).Should(gomega.Succeed())
+	})
+	ginkgo.AfterEach(func() {
+		gomega.Expect(k8sClient.Delete(ctx, ns)).To(gomega.Succeed())
+	})
+	ginkgo.When("Creating a Job without a matching LocalQueue", func() {
+		ginkgo.It("Should stay in suspended", func() {
+			lookupKey := types.NamespacedName{Name: "test-job", Namespace: ns.Name}
+			createdJob := &batchv1.Job{}
+			gomega.Eventually(func() bool {
+				if err := k8sClient.Get(ctx, lookupKey, createdJob); err != nil {
+					return false
+				}
+				return *createdJob.Spec.Suspend
+			}, framework.Timeout, framework.Interval).Should(gomega.BeTrue())
+			createdWorkload := &kueue.Workload{}
+			gomega.Eventually(func() bool {
+				if err := k8sClient.Get(ctx, lookupKey, createdWorkload); err != nil {
+					return false
+				}
+				return workload.InCondition(createdWorkload, kueue.WorkloadAdmitted)
+
+			}, framework.Timeout, framework.Interval).Should(gomega.BeFalse())
+			gomega.Expect(k8sClient.Delete(ctx, sampleJob)).Should(gomega.Succeed())
+		})
+	})
+	ginkgo.When("Creating a Job With Queueing", func() {
+		var (
+			resourceKueue *kueue.ResourceFlavor
+			localQueue    *kueue.LocalQueue
+			clusterQueue  *kueue.ClusterQueue
+		)
+		ginkgo.BeforeEach(func() {
+			resourceKueue = testing.MakeResourceFlavor("default").Obj()
+			gomega.Expect(k8sClient.Create(ctx, resourceKueue)).Should(gomega.Succeed())
+			localQueue = testing.MakeLocalQueue("main", ns.Name).Obj()
+			clusterQueue = testing.MakeClusterQueue("cluster-queue").
+				Resource(testing.MakeResource(corev1.ResourceCPU).
+					Flavor(testing.MakeFlavor("default", "1").Obj()).Obj()).
+				Resource(testing.MakeResource(corev1.ResourceMemory).
+					Flavor(testing.MakeFlavor("default", "36Gi").Obj()).Obj()).Obj()
+			localQueue.Spec.ClusterQueue = "cluster-queue"
+			gomega.Expect(k8sClient.Create(ctx, clusterQueue)).Should(gomega.Succeed())
+			gomega.Expect(k8sClient.Create(ctx, localQueue)).Should(gomega.Succeed())
+		})
+		ginkgo.AfterEach(func() {
+			gomega.Expect(k8sClient.Delete(ctx, localQueue)).Should(gomega.Succeed())
+			gomega.Expect(k8sClient.Delete(ctx, clusterQueue)).Should(gomega.Succeed())
+			gomega.Expect(k8sClient.Delete(ctx, resourceKueue)).Should(gomega.Succeed())
+		})
+		ginkgo.It("Should unsuspend a job", func() {
+			lookupKey := types.NamespacedName{Name: "test-job", Namespace: ns.Name}
+			createdJob := &batchv1.Job{}
+			createdWorkload := &kueue.Workload{}
+
+			gomega.Eventually(func() bool {
+				if err := k8sClient.Get(ctx, lookupKey, createdJob); err != nil {
+					return false
+				}
+				return !*createdJob.Spec.Suspend && createdJob.Status.Succeeded > 0
+			}, framework.Timeout, framework.Interval).Should(gomega.BeTrue())
+			gomega.Eventually(func() bool {
+				if err := k8sClient.Get(ctx, lookupKey, createdWorkload); err != nil {
+					return false
+				}
+				return workload.InCondition(createdWorkload, kueue.WorkloadAdmitted) && workload.InCondition(createdWorkload, kueue.WorkloadFinished)
+
+			}, framework.Timeout, framework.Interval).Should(gomega.BeTrue())
+		})
+	})
+})

test/e2e/framework/constants.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"time"
+)
+
+const (
+	Timeout  = time.Second * 30
+	Interval = time.Millisecond * 250
+)

test/e2e/framework/framework.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"context"
+
+	"github.com/onsi/gomega"
+	"k8s.io/client-go/kubernetes/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+
+	kueue "sigs.k8s.io/kueue/apis/kueue/v1alpha2"
+	"sigs.k8s.io/kueue/pkg/util/testing"
+	//+kubebuilder:scaffold:imports
+)
+
+func CreateClientUsingCluster() client.Client {
+	cfg := config.GetConfigOrDie()
+	gomega.ExpectWithOffset(1, cfg).NotTo(gomega.BeNil())
+
+	err := kueue.AddToScheme(scheme.Scheme)
+	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
+
+	// +kubebuilder:scaffold:scheme
+	client, err := client.New(cfg, client.Options{Scheme: scheme.Scheme})
+	gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred())
+	return client
+}
+
+func KueueReadyForTesting(client client.Client) {
+	// To verify that webhooks are ready, let's create a simple resourceflavor
+	resourceKueue := testing.MakeResourceFlavor("default").Obj()
+	gomega.Eventually(func() error {
+		return client.Create(context.Background(), resourceKueue)
+	}, Timeout, Interval).Should(gomega.Succeed())
+	gomega.Expect(client.Delete(context.Background(), resourceKueue)).Should(gomega.Succeed())
+}

test/e2e/suite_test.go

@@ -0,0 +1,47 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"testing"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"sigs.k8s.io/kueue/test/e2e/framework"
+	//+kubebuilder:scaffold:imports
+)
+
+var (
+	k8sClient client.Client
+	ctx       context.Context
+)
+
+func TestAPIs(t *testing.T) {
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t,
+		"End To End Suite",
+	)
+}
+
+var _ = ginkgo.BeforeSuite(func() {
+	k8sClient = framework.CreateClientUsingCluster()
+	ctx = context.Background()
+	framework.KueueReadyForTesting(k8sClient)
+})