From 2d5ea2c1ebc36fb9363c917485738d8e6d84e282 Mon Sep 17 00:00:00 2001 From: Christie Wilson Date: Wed, 22 Jul 2020 16:46:21 -0400 Subject: [PATCH] =?UTF-8?q?Add=20a=20task=20for=20creating=20a=20GKE=20clu?= =?UTF-8?q?ster=20for=20e2e=20tests=20=F0=9F=A7=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As part of #373 I'm continuing to create Tasks that we can use to run tests for items in the catalog. This Task creates a cluster within a GKE project and is based on the way we create clusters for the existing end to end tests we have in Tekton, which are themselves based on Knative, when are THEMselves based on k8s. These tests all use a tool called kubetest, which is responsible for invoking boskos (see #408) and creating a GKE cluster (https://github.com/kubernetes/test-infra/tree/master/kubetest). This Task attempts to create a cluster in the same way, which based on the output in the end to end logs seems to consist of creating the cluster and setting up a firewall for it. I'm not sure if this belongs in the catalog itself since it is specific to our own end to end tests and isn't as generic as other catalog tests, so if we want to move this into the test directory that seems fine, but I also think it could be an okay addition to the catalog on its own (esp if other k8s based projects that test against GKE want to use it.) --- task/gke-cluster-create/0.1/README.md | 43 +++++++ .../0.1/gke-cluster-create.yaml | 108 ++++++++++++++++++ .../0.1/samples/create-gke-cluster.yaml | 34 ++++++ task/gke-cluster-create/OWNERS | 5 + 4 files changed, 190 insertions(+) create mode 100644 task/gke-cluster-create/0.1/README.md create mode 100644 task/gke-cluster-create/0.1/gke-cluster-create.yaml create mode 100644 task/gke-cluster-create/0.1/samples/create-gke-cluster.yaml create mode 100644 task/gke-cluster-create/OWNERS diff --git a/task/gke-cluster-create/0.1/README.md b/task/gke-cluster-create/0.1/README.md new file mode 100644 index 0000000000..b1e654de83 --- /dev/null +++ b/task/gke-cluster-create/0.1/README.md @@ -0,0 +1,43 @@ +# GKE End to End Cluster Create + +The Task `gke-e2e-cluster-create` can be used to create a GKE cluster in a GCP +project and fetch a kubeconfig that can be used (in a context with both kubectl and gcloud +available) to make requests to the cluster. + +The cluster created will have a firewall applied such that the only traffic allowed to instances +in the cluster will be SSH, TCP services running on 80 or 8080, and services exposed via the +NodePort default range (https://kubernetes.io/docs/concepts/services-networking/service/#nodeport). + +## Parameters + +* **project-name**: The name of the GCP project in which to create the GKE cluster. (_required_) +* **private-key-path**: The path to the private key within the gcp-service-account workspace. (_required_) +* **identifier**: A string which identifies the purpose for which this cluster is being created. Used to name other resources created. (_required_) +* **min-nodes**: The minimum number of nodes in the cluster. (_default_:1) +* **max-nodes**: The maximum number of nodes in the cluster. (_default_:3) +* **region**: The region to create the cluster in. (_default_:us-central1) +* **machine-type**: The machine type to create, from + https://cloud.google.com/compute/docs/machine-types. (_default_:n1-standard-4) +* **image-type**: The type of image to create the nodes, from + https://cloud.google.com/kubernetes-engine/docs/concepts/node-images. (_default_:cos) +* **cluster-version**: The GKE version to install, in a format that can be used as the + `--cluster-version` argument to https://cloud.google.com/sdk/gcloud/reference/beta/container/clusters/create + (_default_: latest) + +## Workspaces + +* **gcp-service-account**: A Secret or volume containing the private key of a GCP service account + that can create GKE clusters in the project +* **kubeconfig**: A workspace into which a kubeconfig file called `kubeconfig` will be written that + will contain the information required to access the cluster. The `kubeconfig` will expect to use + gcloud to authenticate, so in order for it to be used it must be run in a container which contains + both kubectl and gcloud. + +## Results + +* **cluster-name** The name of the cluster that was created. + +## Usage + +See [samples/create-gke-cluster.yaml](samples/create-gke-cluster.yaml) for an example of a TaskRun +that creates a GKE cluster and writes the kubeconfig to a PVC. diff --git a/task/gke-cluster-create/0.1/gke-cluster-create.yaml b/task/gke-cluster-create/0.1/gke-cluster-create.yaml new file mode 100644 index 0000000000..c903c8326a --- /dev/null +++ b/task/gke-cluster-create/0.1/gke-cluster-create.yaml @@ -0,0 +1,108 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: gke-cluster-create + labels: + app.kubernetes.io/version: "0.1" + annotations: + tekton.dev/pipelines.minVersion: "0.12.1" + tekton.dev/tags: "gke,test" + tekton.dev/displayName: "GKE Cluster Create" +spec: + description: | + Create a GKE cluster. + + This Task can be used to create a GKE cluster in a GCP project and fetch a kubeconfig that + can be used (in a context with both kubectl and gcloud available) to make requests to the cluster. + The cluster will be created with an initial firewall designed to only allow access to SSH, ports 80 and 8080, + and NodePorts. + + params: + - name: project-name + description: The name of the GCP project in which to create the GKE cluster. + - name: private-key-path + description: The path to the private key within the gcp-service-account workspace. + - name: identifier + description: A string which identifies the purpose for which this cluster is being created. Used to name other resources created. + - name: min-nodes + description: The minimum number of nodes in the cluster. + default: "1" + - name: max-nodes + description: The maximum number of nodes in the cluster. + default: "3" + - name: region + description: The region to create the cluster in. + default: us-central1 + - name: machine-type + description: The machine type to create, from https://cloud.google.com/compute/docs/machine-types. + default: n1-standard-4 + - name: image-type + description: The type of image to create the nodes, from https://cloud.google.com/kubernetes-engine/docs/concepts/node-images. + default: cos + - name: cluster-version + description: | + The GKE version to install, in a format that can be used as the `--cluster-version` argument to + https://cloud.google.com/sdk/gcloud/reference/beta/container/clusters/create + default: latest + workspaces: + - name: gcp-service-account + description: A Secret or volume containing the private key of a GCP service account that can create GKE clusters in the project + - name: kubeconfig + description: | + A workspace into which a kubeconfig file called `kubeconfig` will be written that will contain the information + required to access the cluster. The `kubeconfig` will expect to use gcloud to authenticate, so in order for it to + be used it must be run in a container which contains both kubectl and gcloud. + results: + - name: cluster-name + description: The name of the cluster that was created. + steps: + - name: gcloud + image: google/cloud-sdk:slim@sha256:27b2c22bf259d9bc1a291e99c63791ba0c27a04d2db0a43241ba0f1f20f4067f + script: | + UNIQUE_STR=$(head /dev/urandom | tr -dc a-z0-9 | head -c 10 ; echo '') + UNIQUE_NAME=$(params.identifier)-$UNIQUE_STR + + # Configure gcloud to use the provided service account + gcloud auth activate-service-account --key-file=$(workspaces.gcp-service-account.path)/$(params.private-key-path) + + # Create a network and a new cluster + gcloud compute networks create $UNIQUE_NAME --project $(params.project-name) --subnet-mode=auto + gcloud container clusters create \ + --quiet \ + --enable-autoscaling \ + --scopes=cloud-platform \ + --enable-basic-auth \ + --no-issue-client-certificate \ + --project=$(params.project-name) \ + --cluster-version=$(params.cluster-version) \ + --min-nodes=$(params.min-nodes) \ + --max-nodes=$(params.max-nodes) \ + --region=$(params.region) \ + --machine-type=$(params.machine-type) \ + --image-type=$(params.image-type) \ + --num-nodes=1 \ + --network=$UNIQUE_NAME\ + $UNIQUE_NAME + + # Write the kubeconfig for connecting to the new cluster to the provided workspace + KUBECONFIG=$(workspaces.kubeconfig.path)/kubeconfig gcloud container clusters get-credentials \ + --project=$(params.project-name) \ + --region=$(params.region) \ + $UNIQUE_NAME + + # Get the tag used for the instances created and use that to apply firewall rules to them + INSTANCE_TAG=$(gcloud compute instances list \ + --project=$(params.project-name) \ + --filter=metadata.cluster-name=$UNIQUE_NAME \ + --limit=1 \ + --format=get\(tags.items\) | tr -d '\n') + + # This firewall rule allows the cluster to expose SSH, TCP services running on 80 or 8080, + # and services exposed via the NodePort default range (https://kubernetes.io/docs/concepts/services-networking/service/#nodeport) + gcloud compute firewall-rules create ports-$UNIQUE_STR \ + --project=$(params.project-name) \ + --network=$UNIQUE_NAME \ + --allow=tcp:22,tcp:80,tcp:8080,tcp:30000-32767,udp:30000-32767 \ + --target-tags=$INSTANCE_TAG + + printf $UNIQUE_NAME > /tekton/results/cluster-name diff --git a/task/gke-cluster-create/0.1/samples/create-gke-cluster.yaml b/task/gke-cluster-create/0.1/samples/create-gke-cluster.yaml new file mode 100644 index 0000000000..0eea5e835c --- /dev/null +++ b/task/gke-cluster-create/0.1/samples/create-gke-cluster.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: gke-create-kubeconfig-pvc +spec: + resources: + requests: + storage: 5M + volumeMode: Filesystem + accessModes: + - ReadWriteOnce +--- +apiVersion: tekton.dev/v1beta1 +kind: TaskRun +metadata: + generateName: create-gke-cluster- +spec: + taskRef: + name: gke-cluster-create + params: + - name: project-name + value: tekton-prow-14 + - name: private-key-path + value: service-account.json + - name: identifier + value: try-it-out + workspaces: + - name: gcp-service-account + secret: + # This secret exists in the tekton-releases prow cluster + secretName: test-account + - name: kubeconfig + persistentVolumeClaim: + claimName: gke-create-kubeconfig-pvc \ No newline at end of file diff --git a/task/gke-cluster-create/OWNERS b/task/gke-cluster-create/OWNERS new file mode 100644 index 0000000000..74cf762fa7 --- /dev/null +++ b/task/gke-cluster-create/OWNERS @@ -0,0 +1,5 @@ +approvers: +- bobcatfish + +reviewers: +- bobcatfish \ No newline at end of file