sharing artifacts between pipeline tasks using bucket

- using gcp buckets only for now
- bucket information configured using a config map
- refactor of pvc implementation for the same feature to use same interface
- artifact bucket and artifact pvc return the container spec to execute the upload and download steps
- issue tektoncd#384

config/config-artifact-bucket.yaml

@@ -0,0 +1,30 @@
+# Copyright 2018 The Knative Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-artifact-bucket
+  namespace: knative-build-pipeline
+data:
+  # location of the gcs bucket to be used for artifact storage
+  # location: "gs://bucket-name"
+
+  # name of the secret that will contain the credentials for the service account
+  # with access to the bucket
+  # bucket.service.account.secret.name: 
+
+  # The key in the secret with the required service account json
+  # bucket.service.account.secret.key: 
+

docs/developers/README.md

@@ -16,6 +16,15 @@ on path `/pvc` by PipelineRun.
   adds a step to copy from PVC to directory path
   `/pvc/previous_task/resource_name`.
 
+Another alternatives is to use a GCS storage bucket to share the artifacts. This can 
+be configured using a ConfigMap with the name `config-artifact-bucket` with the following attributes:
+
+- location: the address of the bucket (for example gs://mybucket)
+- bucket.service.account.secret.name: the name of the secret that will contain the credentials for the service account
+  with access to the bucket
+- bucket.service.account.secret.key: the key in the secret with the required service account json
+  The bucket is configured with a retention policy of 24 hours after which files will be deleted
+
 ### How are inputs handled?
 
 Input resources, like source code (git) or artifacts, are dumped at path

docs/using.md

@@ -183,9 +183,19 @@ configure that by edit the `image`'s value in a configmap named
 ### Resource sharing between tasks
 
 Pipeline `Tasks` are allowed to pass resources from previous `Tasks` via the
-[`from`](#from) field. This feature is implemented using Persistent Volume
-Claims under the hood but however has an implication that tasks cannot have any
-volume mounted under path `/pvc`.
+[`from`](#from) field. This feature is implemented using the two
+following alternatives:
+
+- Persistent Volume Claims under the hood but however has an implication
+  that tasks cannot have any volume mounted under path `/pvc`.
+
+- [GCS storage bucket](https://cloud.google.com/storage/docs/json_api/v1/buckets)
+  A storage bucket can be configured using a ConfigMap with the name `config-artifact-bucket` with the following attributes:
+- location: the address of the bucket (for example gs://mybucket)
+- bucket.service.account.secret.name: the name of the secret that will contain the credentials for the service account
+  with access to the bucket
+- bucket.service.account.secret.key: the key in the secret with the required service account json
+The bucket is configured with a retention policy of 24 hours after which files will be deleted
 
 ### Outputs
 

pkg/apis/pipeline/v1alpha1/artifact_bucket.go

@@ -0,0 +1,147 @@
+/*
+Copyright 2018 The Knative Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	// BucketConfigName is the name of the configmap containing all
+	// customizations for the storage bucket.
+	BucketConfigName = "config-artifact-bucket"
+
+	// BucketLocationKey is the name of the configmap entry that specifies
+	// loction of the bucket.
+	BucketLocationKey = "location"
+
+	// BucketServiceAccountSecretName is the name of the configmap entry that specifies
+	// the name of the secret that will provide the servie account with bucket access.
+	// This secret must  have a key called serviceaccount that will have a value with
+	// the service account with access to the bucket
+	BucketServiceAccountSecretName = "bucket.service.account.secret.name"
+
+	// BucketServiceAccountSecretKey is the name of the configmap entry that specifies
+	// the secret key that will have a value with the service account json with access
+	// to the bucket
+	BucketServiceAccountSecretKey = "bucket.service.account.secret.key"
+)
+
+var (
+	secretVolumeMountPath = "/var/bucketsecret"
+)
+
+// ArtifactBucket contains the Storage bucket configuration defined in the
+// Bucket config map.
+type ArtifactBucket struct {
+	Name     string
+	Location string
+	Secrets  []SecretParam
+}
+
+// IsPVC indicates if the temporary storage used for artifacts in a pipelinerun is a PVC
+func (b *ArtifactBucket) IsPVC() bool {
+	return false
+}
+
+// StorageBasePath returns the path to be used to store artifacts in a pipelinerun temporary storage
+func (b *ArtifactBucket) StorageBasePath(pr *PipelineRun) string {
+	return fmt.Sprintf("%s-%s-bucket", pr.Name, pr.Namespace)
+}
+
+// GetCopyFromContainerSpec returns a container used to download artifacts from temporary storage
+func (b *ArtifactBucket) GetCopyFromContainerSpec(name, sourcePath, destinationPath string) []corev1.Container {
+	args := []string{"-args", fmt.Sprintf("cp -r %s %s", fmt.Sprintf("%s/%s/**", b.Location, sourcePath), destinationPath)}
+
+	envVars, secretVolumeMount := getBucketSecretEnvVarsAndVolumeMounts(b.Secrets)
+
+	return []corev1.Container{{
+		Name:  fmt.Sprintf("artifact-dest-mkdir-%s", name),
+		Image: *bashNoopImage,
+		Args: []string{
+			"-args", strings.Join([]string{"mkdir", "-p", destinationPath}, " "),
+		},
+	}, {
+		Name:         fmt.Sprintf("artifact-copy-from-%s", name),
+		Image:        *gsutilImage,
+		Args:         args,
+		Env:          envVars,
+		VolumeMounts: secretVolumeMount,
+	}}
+}
+
+// GetCopyToContainerSpec returns a container used to upload artifacts for temporary storage
+func (b *ArtifactBucket) GetCopyToContainerSpec(name, sourcePath, destinationPath string) []corev1.Container {
+	args := []string{"-args", fmt.Sprintf("cp -r %s %s", sourcePath, fmt.Sprintf("%s/%s", b.Location, destinationPath))}
+
+	envVars, secretVolumeMount := getBucketSecretEnvVarsAndVolumeMounts(b.Secrets)
+
+	return []corev1.Container{{
+		Name:         fmt.Sprintf("artifact-copy-to-%s", name),
+		Image:        *gsutilImage,
+		Args:         args,
+		Env:          envVars,
+		VolumeMounts: secretVolumeMount,
+	}}
+}
+
+// GetSecretsVolumes retunrs the list of volumes for secrets to be mounted
+// on pod
+func (b *ArtifactBucket) GetSecretsVolumes() []corev1.Volume {
+	volumes := []corev1.Volume{}
+	for _, sec := range b.Secrets {
+		v := corev1.Volume{
+			Name: fmt.Sprintf("bucket-secret-volume-%s", sec.SecretName),
+			VolumeSource: corev1.VolumeSource{
+				Secret: &corev1.SecretVolumeSource{
+					SecretName: sec.SecretName,
+				},
+			},
+		}
+		volumes = append(volumes, v)
+	}
+	return volumes
+}
+
+func getBucketSecretEnvVarsAndVolumeMounts(secrets []SecretParam) ([]corev1.EnvVar, []corev1.VolumeMount) {
+	mountPaths := make(map[string]struct{})
+	var (
+		envVars           []corev1.EnvVar
+		secretVolumeMount []corev1.VolumeMount
+	)
+	for _, sec := range secrets {
+		if sec.FieldName != "" {
+			mountPath := filepath.Join(secretVolumeMountPath, sec.SecretName)
+			envVars = append(envVars, corev1.EnvVar{
+				Name:  strings.ToUpper(sec.FieldName),
+				Value: filepath.Join(mountPath, sec.SecretKey),
+			})
+			if _, ok := mountPaths[mountPath]; !ok {
+				secretVolumeMount = append(secretVolumeMount, corev1.VolumeMount{
+					Name:      fmt.Sprintf("bucket-secret-volume-%s", sec.SecretName),
+					MountPath: mountPath,
+				})
+				mountPaths[mountPath] = struct{}{}
+			}
+		}
+	}
+	return envVars, secretVolumeMount
+}

pkg/apis/pipeline/v1alpha1/artifact_bucket_test.go

@@ -0,0 +1,97 @@
+/*
+Copyright 2018 The Knative Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	corev1 "k8s.io/api/core/v1"
+)
+
+func TestBucketGetCopyFromContainerSpec(t *testing.T) {
+	bucket := ArtifactBucket{
+		Location: "gs://fake-bucket",
+		Secrets: []SecretParam{{
+			FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
+			SecretName: "secret1",
+			SecretKey:  "serviceaccount",
+		}},
+	}
+	want := []corev1.Container{{
+		Name:  "artifact-dest-mkdir-workspace",
+		Image: "override-with-bash-noop:latest",
+		Args:  []string{"-args", "mkdir -p /workspace/destination"},
+	}, {
+		Name:         "artifact-copy-from-workspace",
+		Image:        "override-with-gsutil-image:latest",
+		Args:         []string{"-args", "cp -r gs://fake-bucket/src-path/** /workspace/destination"},
+		Env:          []corev1.EnvVar{{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/bucketsecret/secret1/serviceaccount"}},
+		VolumeMounts: []corev1.VolumeMount{{Name: "bucket-secret-volume-secret1", MountPath: "/var/bucketsecret/secret1"}},
+	}}
+
+	got := bucket.GetCopyFromContainerSpec("workspace", "src-path", "/workspace/destination")
+	if d := cmp.Diff(got, want); d != "" {
+		t.Errorf("Diff:\n%s", d)
+	}
+}
+
+func TestBucketGetCopyToContainerSpec(t *testing.T) {
+	bucket := ArtifactBucket{
+		Location: "gs://fake-bucket",
+		Secrets: []SecretParam{{
+			FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
+			SecretName: "secret1",
+			SecretKey:  "serviceaccount",
+		}},
+	}
+	want := []corev1.Container{{
+		Name:         "artifact-copy-to-workspace",
+		Image:        "override-with-gsutil-image:latest",
+		Args:         []string{"-args", "cp -r src-path gs://fake-bucket/workspace/destination"},
+		Env:          []corev1.EnvVar{{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/bucketsecret/secret1/serviceaccount"}},
+		VolumeMounts: []corev1.VolumeMount{{Name: "bucket-secret-volume-secret1", MountPath: "/var/bucketsecret/secret1"}},
+	}}
+
+	got := bucket.GetCopyToContainerSpec("workspace", "src-path", "workspace/destination")
+	if d := cmp.Diff(got, want); d != "" {
+		t.Errorf("Diff:\n%s", d)
+	}
+}
+
+func TestGetSecretsVolumes(t *testing.T) {
+	bucket := ArtifactBucket{
+		Location: "gs://fake-bucket",
+		Secrets: []SecretParam{{
+			FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
+			SecretName: "secret1",
+			SecretKey:  "serviceaccount",
+		}},
+	}
+	want := []corev1.Volume{{
+		Name: "bucket-secret-volume-secret1",
+		VolumeSource: corev1.VolumeSource{
+			Secret: &corev1.SecretVolumeSource{
+				SecretName: "secret1",
+			},
+		},
+	}}
+	got := bucket.GetSecretsVolumes()
+	if d := cmp.Diff(got, want); d != "" {
+		t.Errorf("Diff:\n%s", d)
+	}
+}