Call gcloud auth activate-service-account if GOOGLE_APPLICATION_CRE…

…DENTIALS is set This used to be done by the gsutil image. When that image was replaced by the vanilla google/cloud-sdk image, this documented behavior broke. By adding this to the step added by the resource, the behavior is maintained without needing to maintain our own image to perform activation.
tektoncd · Dec 17, 2019 · 902e539 · 902e539
1 parent 0f20c35
commit 902e539
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 77 deletions.
diff --git a/go.mod b/go.mod
@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	cloud.google.com/go v0.47.0 // indirect
+	cloud.google.com/go/storage v1.0.0
 	contrib.go.opencensus.io/exporter/prometheus v0.1.0 // indirect
 	contrib.go.opencensus.io/exporter/stackdriver v0.12.8 // indirect
 	github.com/Azure/azure-sdk-for-go v36.1.0+incompatible // indirect
@@ -50,7 +51,7 @@ require (
 	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
 	golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4 // indirect
 	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
-	google.golang.org/api v0.10.0 // indirect
+	google.golang.org/api v0.10.0
 	google.golang.org/appengine v1.6.5 // indirect
 	google.golang.org/grpc v1.24.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect

diff --git a/pkg/apis/pipeline/v1alpha1/gcs_resource.go b/pkg/apis/pipeline/v1alpha1/gcs_resource.go
@@ -26,8 +26,14 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
-var (
-	gcsSecretVolumeMountPath = "/var/secret"
+const (
+	gcsSecretVolumeMountPath     = "/var/secret"
+	activateServiceAccountScript = `#!/usr/bin/env bash
+if [[ "${GOOGLE_APPLICATION_CREDENTIALS}" != "" ]]; then
+  echo GOOGLE_APPLICATION_CREDENTIALS is set, activating Service Account...
+  gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+`
 )
 
 // GCSResource is a GCS endpoint from which to get artifacts which is required
@@ -133,24 +139,26 @@ func (s *GCSResource) GetInputTaskModifier(ts *TaskSpec, path string) (TaskModif
 	if path == "" {
 		return nil, fmt.Errorf("GCSResource: Expect Destination Directory param to be set %s", s.Name)
 	}
-	var args []string
+	script := activateServiceAccountScript
 	if s.TypeDir {
-		args = []string{"rsync", "-d", "-r", s.Location, path}
+		script += fmt.Sprintf("gsutil rsync -d -r %s %s\n", s.Location, path)
 	} else {
-		args = []string{"cp", s.Location, path}
+		script += fmt.Sprintf("gsutil cp %s %s\n", s.Location, path)
 	}
 
 	envVars, secretVolumeMount := getSecretEnvVarsAndVolumeMounts(s.Name, gcsSecretVolumeMountPath, s.Secrets)
 	steps := []Step{
 		CreateDirStep(s.ShellImage, s.Name, path),
-		{Container: corev1.Container{
-			Name:         names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(fmt.Sprintf("fetch-%s", s.Name)),
-			Image:        s.GsutilImage,
-			Command:      []string{"gsutil"},
-			Args:         args,
-			Env:          envVars,
-			VolumeMounts: secretVolumeMount,
-		}}}
+		{
+			Script: script,
+			Container: corev1.Container{
+				Name:         names.SimpleNameGenerator.RestrictLengthWithRandomSuffix(fmt.Sprintf("fetch-%s", s.Name)),
+				Image:        s.GsutilImage,
+				Env:          envVars,
+				VolumeMounts: secretVolumeMount,
+			},
+		},
+	}
 
 	volumes := getStorageVolumeSpec(s, *ts)
 

diff --git a/pkg/apis/pipeline/v1alpha1/gcs_resource_test.go b/pkg/apis/pipeline/v1alpha1/gcs_resource_test.go
@@ -26,7 +26,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
-func Test_Invalid_NewStorageResource(t *testing.T) {
+func TestInvalidNewStorageResource(t *testing.T) {
 	for _, tc := range []struct {
 		name             string
 		pipelineResource *v1alpha1.PipelineResource
@@ -76,7 +76,7 @@ func Test_Invalid_NewStorageResource(t *testing.T) {
 	}
 }
 
-func Test_Valid_NewGCSResource(t *testing.T) {
+func TestValidNewGCSResource(t *testing.T) {
 	pr := tb.PipelineResource("gcs-resource", "default", tb.PipelineResourceSpec(
 		v1alpha1.PipelineResourceTypeStorage,
 		tb.PipelineResourceSpecParam("Location", "gs://fake-bucket"),
@@ -107,7 +107,7 @@ func Test_Valid_NewGCSResource(t *testing.T) {
 	}
 }
 
-func Test_GCSGetReplacements(t *testing.T) {
+func TestGCSGetReplacements(t *testing.T) {
 	gcsResource := &v1alpha1.GCSResource{
 		Name:     "gcs-resource",
 		Location: "gs://fake-bucket",
@@ -123,7 +123,7 @@ func Test_GCSGetReplacements(t *testing.T) {
 	}
 }
 
-func Test_GetParams(t *testing.T) {
+func TestGetParams(t *testing.T) {
 	pr := tb.PipelineResource("gcs-resource", "default", tb.PipelineResourceSpec(
 		v1alpha1.PipelineResourceTypeStorage,
 		tb.PipelineResourceSpecParam("Location", "gcs://some-bucket.zip"),
@@ -144,7 +144,7 @@ func Test_GetParams(t *testing.T) {
 	}
 }
 
-func Test_GetInputSteps(t *testing.T) {
+func TestGetInputSteps(t *testing.T) {
 	names.TestingSeed()
 
 	for _, tc := range []struct {
@@ -170,20 +170,27 @@ func Test_GetInputSteps(t *testing.T) {
 			Name:    "create-dir-gcs-valid-9l9zj",
 			Image:   "busybox",
 			Command: []string{"mkdir", "-p", "/workspace"},
-		}}, {Container: corev1.Container{
-			Name:    "fetch-gcs-valid-mz4c7",
-			Image:   "google/cloud-sdk",
-			Command: []string{"gsutil"},
-			Args:    []string{"rsync", "-d", "-r", "gs://some-bucket", "/workspace"},
-			Env: []corev1.EnvVar{{
-				Name:  "GOOGLE_APPLICATION_CREDENTIALS",
-				Value: "/var/secret/secretName/key.json",
-			}},
-			VolumeMounts: []corev1.VolumeMount{{
-				Name:      "volume-gcs-valid-secretName",
-				MountPath: "/var/secret/secretName",
-			}},
-		}}},
+		}}, {
+			Script: `#!/usr/bin/env bash
+if [[ "${GOOGLE_APPLICATION_CREDENTIALS}" != "" ]]; then
+  echo GOOGLE_APPLICATION_CREDENTIALS is set, activating Service Account...
+  gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
+fi
+gsutil rsync -d -r gs://some-bucket /workspace
+`,
+			Container: corev1.Container{
+				Name:  "fetch-gcs-valid-mz4c7",
+				Image: "google/cloud-sdk",
+				Env: []corev1.EnvVar{{
+					Name:  "GOOGLE_APPLICATION_CREDENTIALS",
+					Value: "/var/secret/secretName/key.json",
+				}},
+				VolumeMounts: []corev1.VolumeMount{{
+					Name:      "volume-gcs-valid-secretName",
+					MountPath: "/var/secret/secretName",
+				}},
+			},
+		}},
 	}, {
 		name: "duplicate secret mount paths",
 		gcsResource: &v1alpha1.GCSResource{
@@ -205,35 +212,42 @@ func Test_GetInputSteps(t *testing.T) {
 			Name:    "create-dir-gcs-valid-mssqb",
 			Image:   "busybox",
 			Command: []string{"mkdir", "-p", "/workspace"},
-		}}, {Container: corev1.Container{
-			Name:    "fetch-gcs-valid-78c5n",
-			Image:   "google/cloud-sdk",
-			Command: []string{"gsutil"},
-			Args:    []string{"cp", "gs://some-bucket", "/workspace"},
-			Env: []corev1.EnvVar{{
-				Name:  "GOOGLE_APPLICATION_CREDENTIALS",
-				Value: "/var/secret/secretName/key.json",
-			}},
-			VolumeMounts: []corev1.VolumeMount{{
-				Name:      "volume-gcs-valid-secretName",
-				MountPath: "/var/secret/secretName",
-			}},
-		}}},
+		}}, {
+			Script: `#!/usr/bin/env bash
+if [[ "${GOOGLE_APPLICATION_CREDENTIALS}" != "" ]]; then
+  echo GOOGLE_APPLICATION_CREDENTIALS is set, activating Service Account...
+  gcloud auth activate-service-account --key-file=$GOOGLE_APPLICATION_CREDENTIALS
+fi
+gsutil cp gs://some-bucket /workspace
+`,
+			Container: corev1.Container{
+				Name:  "fetch-gcs-valid-78c5n",
+				Image: "google/cloud-sdk",
+				Env: []corev1.EnvVar{{
+					Name:  "GOOGLE_APPLICATION_CREDENTIALS",
+					Value: "/var/secret/secretName/key.json",
+				}},
+				VolumeMounts: []corev1.VolumeMount{{
+					Name:      "volume-gcs-valid-secretName",
+					MountPath: "/var/secret/secretName",
+				}},
+			},
+		}},
 	}} {
 		t.Run(tc.name, func(t *testing.T) {
 			ts := v1alpha1.TaskSpec{}
 			gotSpec, err := tc.gcsResource.GetInputTaskModifier(&ts, "/workspace")
 			if tc.wantErr && err == nil {
 				t.Fatalf("Expected error to be %t but got %v:", tc.wantErr, err)
 			}
-			if d := cmp.Diff(gotSpec.GetStepsToPrepend(), tc.wantSteps); d != "" {
-				t.Errorf("Error mismatch between download containers spec: %s", d)
+			if d := cmp.Diff(tc.wantSteps, gotSpec.GetStepsToPrepend()); d != "" {
+				t.Errorf("Diff(-want, +got): %s", d)
 			}
 		})
 	}
 }
 
-func Test_GetOutputTaskModifier(t *testing.T) {
+func TestGetOutputTaskModifier(t *testing.T) {
 	names.TestingSeed()
 
 	for _, tc := range []struct {

diff --git a/pkg/pod/pod.go b/pkg/pod/pod.go
@@ -30,10 +30,8 @@ import (
 )
 
 const (
-	workspaceVolumeName = "tekton-internal-workspace"
-	homeVolumeName      = "tekton-internal-home"
-	workspaceDir        = "/workspace"
-	homeDir             = "/tekton/home"
+	workspaceDir = "/workspace"
+	homeDir      = "/tekton/home"
 
 	taskRunLabelKey     = pipeline.GroupName + pipeline.TaskRunLabelKey
 	ManagedByLabelKey   = "app.kubernetes.io/managed-by"

diff --git a/pkg/reconciler/taskrun/resources/input_resource_test.go b/pkg/reconciler/taskrun/resources/input_resource_test.go
@@ -582,12 +582,19 @@ func TestAddResourceToTask(t *testing.T) {
 				Name:    "create-dir-storage1-9l9zj",
 				Image:   "busybox",
 				Command: []string{"mkdir", "-p", "/workspace/gcs-dir"},
-			}}, {Container: corev1.Container{
-				Name:    "fetch-storage1-mz4c7",
-				Image:   "google/cloud-sdk",
-				Command: []string{"gsutil"},
-				Args:    []string{"cp", "gs://fake-bucket/rules.zip", "/workspace/gcs-dir"},
-			}}},
+			}}, {
+				Script: `#!/usr/bin/env bash
+if [[ "${GOOGLE_APPLICATION_CREDENTIALS}" != "" ]]; then
+  echo GOOGLE_APPLICATION_CREDENTIALS is set, activating Service Account...
+  gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+gsutil cp gs://fake-bucket/rules.zip /workspace/gcs-dir
+`,
+				Container: corev1.Container{
+					Name:  "fetch-storage1-mz4c7",
+					Image: "google/cloud-sdk",
+				},
+			}},
 		},
 	}, {
 		desc: "storage resource as input from previous task",
@@ -942,12 +949,19 @@ func TestStorageInputResource(t *testing.T) {
 				Name:    "create-dir-gcs-input-resource-9l9zj",
 				Image:   "busybox",
 				Command: []string{"mkdir", "-p", "/workspace/gcs-input-resource"},
-			}}, {Container: corev1.Container{
-				Name:    "fetch-gcs-input-resource-mz4c7",
-				Image:   "google/cloud-sdk",
-				Command: []string{"gsutil"},
-				Args:    []string{"cp", "gs://fake-bucket/rules.zip", "/workspace/gcs-input-resource"},
-			}}},
+			}}, {
+				Script: `#!/usr/bin/env bash
+if [[ "${GOOGLE_APPLICATION_CREDENTIALS}" != "" ]]; then
+  echo GOOGLE_APPLICATION_CREDENTIALS is set, activating Service Account...
+  gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+gsutil cp gs://fake-bucket/rules.zip /workspace/gcs-input-resource
+`,
+				Container: corev1.Container{
+					Name:  "fetch-gcs-input-resource-mz4c7",
+					Image: "google/cloud-sdk",
+				},
+			}},
 		},
 	}, {
 		desc: "no inputs",
@@ -1001,18 +1015,25 @@ func TestStorageInputResource(t *testing.T) {
 				Name:    "create-dir-storage-gcs-keys-9l9zj",
 				Image:   "busybox",
 				Command: []string{"mkdir", "-p", "/workspace/gcs-input-resource"},
-			}}, {Container: corev1.Container{
-				Name:    "fetch-storage-gcs-keys-mz4c7",
-				Image:   "google/cloud-sdk",
-				Command: []string{"gsutil"},
-				Args:    []string{"rsync", "-d", "-r", "gs://fake-bucket/rules.zip", "/workspace/gcs-input-resource"},
-				VolumeMounts: []corev1.VolumeMount{
-					{Name: "volume-storage-gcs-keys-secret-name", MountPath: "/var/secret/secret-name"},
-				},
-				Env: []corev1.EnvVar{
-					{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/secret/secret-name/key.json"},
+			}}, {
+				Script: `#!/usr/bin/env bash
+if [[ "${GOOGLE_APPLICATION_CREDENTIALS}" != "" ]]; then
+  echo GOOGLE_APPLICATION_CREDENTIALS is set, activating Service Account...
+  gcloud auth activate-service-account --key-file=${GOOGLE_APPLICATION_CREDENTIALS}
+fi
+gsutil rsync -d -r gs://fake-bucket/rules.zip /workspace/gcs-input-resource
+`,
+				Container: corev1.Container{
+					Name:  "fetch-storage-gcs-keys-mz4c7",
+					Image: "google/cloud-sdk",
+					VolumeMounts: []corev1.VolumeMount{
+						{Name: "volume-storage-gcs-keys-secret-name", MountPath: "/var/secret/secret-name"},
+					},
+					Env: []corev1.EnvVar{
+						{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/secret/secret-name/key.json"},
+					},
 				},
-			}}},
+			}},
 			Volumes: []corev1.Volume{{
 				Name:         "volume-storage-gcs-keys-secret-name",
 				VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "secret-name"}},