Enable e2e test

.github/workflows/ci.yml

@@ -4,6 +4,10 @@ on:
   # Run workflow on fork repository will help contributors find and resolve issues before sending a PR.
   push:
   pull_request:
+defaults:
+  run:
+    working-directory: go/src/sigs.k8s.io/work-api
+
 jobs:
   golangci:
     name: lint
@@ -18,28 +22,68 @@ jobs:
 
           # show only new issues if it's a pull request.
           only-new-issues: false
+          args: --timeout 3m0s
   verify:
     name: verify
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+          path: go/src/sigs.k8s.io/work-api
       - name: install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14.x
+          go-version: 1.16.x
       - name: verify
         run: hack/verify-all.sh -v
+        env:
+          GOPATH: '/home/runner/work/work-api/work-api/go'
   test:
     name: unit test
     needs: verify
     runs-on: ubuntu-latest
     steps:
       - name: checkout code
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+          path: go/src/sigs.k8s.io/work-api
       - name: install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.14.x
+          go-version: 1.16.x
       - name: make test
         run: make test
+        env:
+          GOPATH: '/home/runner/work/work-api/work-api/go'
+  e2e:
+    name: e2e
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+          path: go/src/sigs.k8s.io/work-api
+      - name: install Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16.x
+      - name: images
+        run: make docker-build
+        env:
+          GOPATH: '/home/runner/work/work-api/work-api/go'
+      - name: setup kind
+        uses: engineerd/[email protected]
+        with:
+          version: v0.11.1
+      - name: Load image on the nodes of the cluster
+        run: |
+          kind load docker-image --name=kind work-api-controller:latest
+      - name: Run e2e test
+        run: |
+          make test-e2e
+        env:
+          KUBECONFIG: /home/runner/.kube/config

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.14.6 as builder
+FROM golang:1.16 as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

Makefile

@@ -13,6 +13,19 @@
 # limitations under the License.
 
 DOCKER ?= docker
+
+GOHOSTOS ?=$(shell go env GOHOSTOS)
+GOHOSTARCH ?=$(shell go env GOHOSTARCH)
+K8S_VERSION ?=1.19.2
+KB_TOOLS_ARCHIVE_NAME :=kubebuilder-tools-$(K8S_VERSION)-$(GOHOSTOS)-$(GOHOSTARCH).tar.gz
+KB_TOOLS_ARCHIVE_PATH := /tmp/$(KB_TOOLS_ARCHIVE_NAME)
+export KUBEBUILDER_ASSETS ?=/tmp/kubebuilder/bin
+
+HUB_KUBECONFIG ?=$(KUBECONFIG)
+HUB_KUBECONFIG_CONTEXT ?=$(shell kubectl --kubeconfig $(HUB_KUBECONFIG) config current-context)
+SPOKE_KUBECONFIG ?=$(KUBECONFIG)
+SPOKE_KUBECONFIG_CONTEXT ?=$(shell kubectl --kubeconfig $(SPOKE_KUBECONFIG) config current-context)
+
 # TOP is the current directory where this Makefile lives.
 TOP := $(dir $(firstword $(MAKEFILE_LIST)))
 # ROOT is the root of the mkdocs tree.
@@ -23,8 +36,6 @@ IMG ?= work-api-controller:latest
 CRD_OPTIONS ?= "crd:crdVersions=v1"
 
 CONTROLLER_GEN=go run sigs.k8s.io/controller-tools/cmd/controller-gen
-# enable Go modules
-export GO111MODULE=on
 
 .PHONY: all
 all: generate manifests controller verify
@@ -53,11 +64,11 @@ generate:
 # Generate manifests e.g. CRD, RBAC etc.
 .PHONY: manifests
 manifests:
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=work-manager webhook schemapatch:manifests="config/crd-base" paths="./..." output:crd:none output:schemapatch:dir="config/crd"
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=work-manager webhook schemapatch:manifests="config/crd-base" paths="./pkg/apis/v1alpha1" output:crd:none output:schemapatch:dir="config/crd"
 
 # Run tests
 .PHONY: test
-test: generate fmt vet manifests
+test: generate fmt vet manifests ensure-kubebuilder-tools
 	go test ./pkg/... -coverprofile cover.out
 
 # Run static analysis.
@@ -74,3 +85,39 @@ docker-build: generate fmt vet manifests
 .PHONY: docker-push
 docker-push: docker-build
 	docker push ${IMG}
+
+.PHONY: deploy
+deploy:
+	kubectl apply -f config/crd
+	kubectl apply -k deploy
+
+cluster-ip:
+	kubectl config use-context $(HUB_KUBECONFIG_CONTEXT) --kubeconfig $(HUB_KUBECONFIG)
+  	CLUSTER_IP?=$(shell kubectl --kubeconfig $(HUB_KUBECONFIG) get svc kubernetes -n default -o jsonpath="{.spec.clusterIP}")
+
+e2e-hub-kubeconfig-secret: cluster-ip
+	cp $(HUB_KUBECONFIG) e2e-hub-kubeconfig
+	kubectl apply -f deploy/component_namespace.yaml --kubeconfig $(SPOKE_KUBECONFIG)
+	kubectl config set clusters.$(HUB_KUBECONFIG_CONTEXT).server https://$(CLUSTER_IP) --kubeconfig e2e-hub-kubeconfig
+	kubectl delete secret hub-kubeconfig-secret -n work --ignore-not-found --kubeconfig $(SPOKE_KUBECONFIG)
+	kubectl create secret generic hub-kubeconfig-secret --from-file=kubeconfig=e2e-hub-kubeconfig -n work --kubeconfig $(SPOKE_KUBECONFIG)
+	rm ./e2e-hub-kubeconfig
+
+build-e2e:
+	go test -c ./tests/e2e
+
+.PHONY: test-e2e
+test-e2e: build-e2e e2e-hub-kubeconfig-secret deploy
+	./e2e.test -test.v -ginkgo.v
+
+# download the kubebuilder-tools to get kube-apiserver binaries from it
+.PHONY: ensure-kubebuilder-tools
+ensure-kubebuilder-tools:
+ifeq "" "$(wildcard $(KUBEBUILDER_ASSETS))"
+	$(info Downloading kube-apiserver into '$(KUBEBUILDER_ASSETS)')
+	mkdir -p '$(KUBEBUILDER_ASSETS)'
+	curl -s -f -L https://storage.googleapis.com/kubebuilder-tools/$(KB_TOOLS_ARCHIVE_NAME) -o '$(KB_TOOLS_ARCHIVE_PATH)'
+	tar -C '$(KUBEBUILDER_ASSETS)' --strip-components=2 -zvxf '$(KB_TOOLS_ARCHIVE_PATH)'
+else
+	$(info Using existing kube-apiserver from "$(KUBEBUILDER_ASSETS)")
+endif

cmd/workcontroller/workcontroller.go

@@ -21,6 +21,7 @@ import (
 	"os"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/clientcmd"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -35,25 +36,29 @@ var (
 )
 
 func init() {
-	clientgoscheme.AddToScheme(scheme)
-	v1alpha1.AddToScheme(scheme)
+	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(v1alpha1.AddToScheme(scheme))
 }
 
 func main() {
 	var metricsAddr string
 	var enableLeaderElection bool
 	var hubkubeconfig string
+	var workNamespace string
 	flag.StringVar(&metricsAddr, "metrics-addr", ":8080", "The address the metric endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "enable-leader-election", false,
 		"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
-	flag.StringVar(&hubkubeconfig, "kubeconfig to connect to hub", "",
+	flag.StringVar(&hubkubeconfig, "hub-kubeconfig", "",
 		"Paths to a kubeconfig connect to hub.")
+	flag.StringVar(&workNamespace, "work-namespace", "",
+		"Namespace to watch for work.")
 	flag.Parse()
 	opts := ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,
 		LeaderElection:     enableLeaderElection,
 		Port:               9443,
+		Namespace:          workNamespace,
 	}
 	ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
 

deploy/clusterrole_binding.yaml

@@ -0,0 +1,14 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: work-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  # We deploy a controller that could work with permission lower than cluster-admin, the tradeoff is
+  # responsivity because list/watch cannot be maintained over too many namespaces.
+  name: admin
+subjects:
+  - kind: ServiceAccount
+    name: work-controller-sa
+    namespace: work

deploy/component_namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: work

deploy/deployment.yaml

@@ -0,0 +1,38 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: work-controller
+  labels:
+    app: work
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: work-controller
+  template:
+    metadata:
+      labels:
+        app: work-controller
+    spec:
+      serviceAccountName: work-controller-sa
+      containers:
+      - name: work-controller
+        image: work-api-controller:latest
+        imagePullPolicy: IfNotPresent
+        args:
+          - "--work-namespace=default"
+          - "--hub-kubeconfig=/spoke/hub-kubeconfig/kubeconfig"
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL
+          privileged: false
+        volumeMounts:
+        - name: hub-kubeconfig-secret
+          mountPath: "/spoke/hub-kubeconfig"
+          readOnly: true
+      volumes:
+      - name: hub-kubeconfig-secret
+        secret:
+          secretName: hub-kubeconfig-secret

deploy/kustomization.yaml

@@ -0,0 +1,16 @@
+
+# Adds namespace to all resources.
+namespace: work
+
+resources:
+- ./component_namespace.yaml
+- ./service_account.yaml
+- ./clusterrole_binding.yaml
+- ./deployment.yaml
+
+images:
+- name: work-api-controller:latest
+  newName: work-api-controller
+  newTag: latest
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization

deploy/service_account.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: work-controller-sa

go.mod

@@ -1,14 +1,15 @@
 module sigs.k8s.io/work-api
 
-go 1.15
+go 1.16
 
 require (
-	github.com/go-logr/logr v0.3.0
-	github.com/onsi/ginkgo v1.14.1
-	github.com/onsi/gomega v1.10.2
-	k8s.io/api v0.20.4
-	k8s.io/apimachinery v0.20.4
-	k8s.io/client-go v0.20.4
-	sigs.k8s.io/controller-runtime v0.8.3
+	github.com/go-logr/logr v0.4.0
+	github.com/onsi/ginkgo v1.16.4
+	github.com/onsi/gomega v1.15.0
+	k8s.io/api v0.22.2
+	k8s.io/apimachinery v0.22.2
+	k8s.io/client-go v0.22.2
+	k8s.io/code-generator v0.22.2
+	sigs.k8s.io/controller-runtime v0.10.1
 	sigs.k8s.io/controller-tools v0.5.0
 )