Enable private customers list and merging of secrets

Fixes an issues with merging lists of secrets where the mergo
library was overwriting values instead of doing a union.

Ref:
darccio/mergo#107

Enables a private customer list via a secret instead of a public
HTTPS URL.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <[email protected]>

example.init.yaml

@@ -174,6 +174,21 @@ secrets:
       - "ecr"
     namespace: "openfaas-fn"
 
+## If not using a HTTPS URL, then set a list of CUSTOMERS
+  - name: "customers"
+    literals:
+      - name: "customers"
+        value: ""
+    namespace: "openfaas"
+    filters:
+      - "default"
+  - name: "customers"
+    literals:
+      - name: "customers"
+        value: ""
+    namespace: "openfaas-fn"
+    filters:
+      - "default"
 
 ### Docker registry
 #### This can be any cluster accessible by your cluster. To populate the file

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/alexellis/k3sup v0.0.0-20200211110243-d7d1e594bc55
 	github.com/imdario/mergo v0.3.8
 	github.com/inlets/inletsctl v0.0.0-20200211123457-caff14436308
+	github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a
 	github.com/morikuni/aec v1.0.0
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v0.0.5

go.sum

@@ -28,6 +28,7 @@ github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8Nz
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/digitalocean/godo v1.27.0/go.mod h1:iJnN9rVu6K5LioLxLimlq0uRI+y/eAQjROUmeU/r0hY=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
@@ -61,14 +62,19 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ=
 github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/inlets/inletsctl v0.0.0-20200211123457-caff14436308 h1:/XQwGX+MbAi21PztExZRDZwn0Hev0EQgfPYkecRJd8M=
 github.com/inlets/inletsctl v0.0.0-20200211123457-caff14436308/go.mod h1:tg0BetuaJwCrzH2DIWDhKxlXF+kXs2kUywjtKVhFhNg=
+github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o=
+github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -80,6 +86,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -99,6 +106,7 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
@@ -210,6 +218,7 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/types/merge.go

@@ -7,8 +7,11 @@ import (
 	"fmt"
 
 	"github.com/imdario/mergo"
+	"github.com/jinzhu/copier"
 )
 
+// MergePlans combines one or more plan with a manual merge for
+// the list of secrets.
 func MergePlans(plans []Plan) (*Plan, error) {
 	var err error
 	masterPlan := &Plan{}
@@ -28,5 +31,37 @@ func MergePlans(plans []Plan) (*Plan, error) {
 		}
 	}
 
+	patchSecrets(masterPlan, plans)
+
 	return masterPlan, err
 }
+
+func patchSecrets(masterPlan *Plan, plans []Plan) {
+	masterList := []KeyValueNamespaceTuple{}
+
+	// Read each plan
+	for _, plan := range plans {
+
+		// Process each secret
+		for _, v := range plan.Secrets {
+
+			// Apply to master list
+			index := -1
+			for i, mv := range masterList {
+				if mv.Name == v.Name {
+					index = i
+					break
+				}
+			}
+
+			if index == -1 {
+				item := KeyValueNamespaceTuple{}
+				copier.Copy(&item, &v)
+				masterList = append(masterList, item)
+			} else {
+				copier.Copy(&masterList[index], &v)
+			}
+		}
+	}
+	masterPlan.Secrets = masterList
+}

pkg/types/merge_test.go

@@ -69,11 +69,11 @@ func Test_mergePlans_MergeEmptyItemsFromBoth(t *testing.T) {
 func Test_mergePlans_PlanValuesOverwriteAccordingToOrder(t *testing.T) {
 
 	plan1 := Plan{
-		OpenFaaSCloudVersion: "0.12.0",
+		OpenFaaSCloudVersion: "0.11.0",
 	}
 
 	plan2 := Plan{
-		OpenFaaSCloudVersion: "0.11.0",
+		OpenFaaSCloudVersion: "0.12.0",
 	}
 
 	planOut, err := MergePlans([]Plan{plan1, plan2})
@@ -84,8 +84,67 @@ func Test_mergePlans_PlanValuesOverwriteAccordingToOrder(t *testing.T) {
 	}
 
 	wantVer := plan2.OpenFaaSCloudVersion
-	if planOut.OpenFaaSCloudVersion != wantVer {
-		t.Errorf("OpenFaaSCloudVersion want: %s, but got: %s", wantVer, planOut.OpenFaaSCloudVersion)
+	gotVer := planOut.OpenFaaSCloudVersion
+	if gotVer != wantVer {
+		t.Errorf("OpenFaaSCloudVersion want: %s, but got: %s", wantVer, gotVer)
+	}
+}
+
+func Test_mergePlans_CombineSecretsDifferentNames(t *testing.T) {
+
+	plan1 := Plan{
+		Secrets: []KeyValueNamespaceTuple{
+			KeyValueNamespaceTuple{Name: "one"},
+		},
+	}
+
+	plan2 := Plan{
+		Secrets: []KeyValueNamespaceTuple{
+			KeyValueNamespaceTuple{Name: "two"},
+		},
+	}
+
+	planOut, err := MergePlans([]Plan{plan1, plan2})
+
+	if err != nil {
+		t.Errorf("Got error, expected no error: %s", err.Error())
+		t.Fail()
 	}
 
+	wantLen := 2
+	gotLen := len(planOut.Secrets)
+	if gotLen != wantLen {
+		t.Errorf("Secrets want length %d, but got: %d", wantLen, gotLen)
+	}
+}
+
+func Test_mergePlans_CombineSecretsMatchingNames(t *testing.T) {
+
+	plan1 := Plan{
+		Secrets: []KeyValueNamespaceTuple{
+			KeyValueNamespaceTuple{Name: "one",
+				Namespace: "openfaas-fn",
+			},
+		},
+	}
+
+	plan2 := Plan{
+		Secrets: []KeyValueNamespaceTuple{
+			KeyValueNamespaceTuple{Name: "one",
+				Namespace: "openfaas-stag"},
+		},
+	}
+
+	planOut, err := MergePlans([]Plan{plan1, plan2})
+
+	if err != nil {
+		t.Errorf("Got error, expected no error: %s", err.Error())
+		t.Fail()
+	}
+
+	wantLen := 1
+	gotLen := len(planOut.Secrets)
+	if gotLen != wantLen {
+		t.Errorf("Secrets want length %d, but got: %d", wantLen, gotLen)
+	}
 }

templates/edge-auth-dep.yml

@@ -27,9 +27,12 @@ spec:
         - name: of-client-secret
           secret:
             secretName: of-client-secret
+        - name: customers
+          secret:
+            secretName: customers
       containers:
       - name: edge-auth
-        image: openfaas/edge-auth:0.6.3
+        image: openfaas/edge-auth:0.7.0
         imagePullPolicy: Always
         livenessProbe:
           httpGet:
@@ -47,6 +50,8 @@ spec:
             value: "/var/secrets/public/key.pub"
           - name: private_key_path
             value: "/var/secrets/private/key"
+          - name: customers_path
+            value: "/var/secrets/customers/customers"
 # Update for your configuration:
           - name: client_secret # this can also be provided via a secret named of-client-secret
             value: ""
@@ -68,7 +73,7 @@ spec:
           - name: cookie_root_domain
             value: ".system.{{.RootDomain}}"
 
-# This is a default and can be overriden
+# This is a default and can be overridden
           - name: customers_url
             value: "{{.CustomersURL}}"
 
@@ -88,3 +93,6 @@ spec:
         - name: of-client-secret
           readOnly: true
           mountPath: "/var/secrets/of-client-secret"
+        - name: customers
+          readOnly: true
+          mountPath: "/var/secrets/customers"

templates/gitlab.yml

@@ -6,7 +6,7 @@ functions:
   system-gitlab-event:
     lang: go
     handler: ./gitlab-event
-    image: functions/gitlab-event:0.1.3
+    image: functions/gitlab-event:0.2.0
     labels:
       openfaas-cloud: "1"
       role: openfaas-system
@@ -19,12 +19,14 @@ functions:
       read_debug: true
       installation_tag: "openfaas-cloud"
       gitlab_instance: "{{.GitLabInstance}}"
+      customers_path: /var/openfaas/secrets/customers
     environment_file:
       - gateway_config.yml
     secrets:
       - gitlab-webhook-secret
       - payload-secret
       - gitlab-api-token
+      - customers
 
   gitlab-push:
     lang: go

templates/stack.yml

@@ -7,7 +7,7 @@ functions:
   system-github-event:
     lang: go
     handler: ./github-event
-    image: functions/github-event:0.8.1
+    image: functions/github-event:0.9.0
     labels:
       openfaas-cloud: "1"
       role: openfaas-system
@@ -17,12 +17,14 @@ functions:
       write_debug: true
       read_debug: true
       validate_customers: true
+      customers_path: /var/openfaas/secrets/customers
     environment_file:
       - github.yml
       - gateway_config.yml
     secrets:
       - github-webhook-secret
       - payload-secret
+      - customers
 
   github-push:
     lang: go