Allow images to be excluded from scanning

cluster/kubernetes/images.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/pkg/errors"
+	"github.com/ryanuber/go-glob"
 	apiv1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -14,7 +15,39 @@ import (
 	"github.com/weaveworks/flux/registry"
 )
 
-func mergeCredentials(log func(...interface{}) error, client extendedClient, namespace string, podTemplate apiv1.PodTemplateSpec, imageCreds registry.ImageCreds, seenCreds map[string]registry.Credentials) {
+func mergeCredentials(log func(...interface{}) error,
+	includeImage func(imageName string) bool,
+	client extendedClient,
+	namespace string, podTemplate apiv1.PodTemplateSpec,
+	imageCreds registry.ImageCreds,
+	seenCreds map[string]registry.Credentials) {
+	var images []image.Name
+	for _, container := range podTemplate.Spec.InitContainers {
+		r, err := image.ParseRef(container.Image)
+		if err != nil {
+			log("err", err.Error())
+			continue
+		}
+		if includeImage(r.CanonicalName().Name.String()) {
+			images = append(images, r.Name)
+		}
+	}
+
+	for _, container := range podTemplate.Spec.Containers {
+		r, err := image.ParseRef(container.Image)
+		if err != nil {
+			log("err", err.Error())
+			continue
+		}
+		if includeImage(r.CanonicalName().Name.String()) {
+			images = append(images, r.Name)
+		}
+	}
+
+	if len(images) < 1 {
+		return
+	}
+
 	creds := registry.NoCredentials()
 	var imagePullSecrets []string
 	saName := podTemplate.Spec.ServiceAccountName
@@ -81,21 +114,8 @@ func mergeCredentials(log func(...interface{}) error, client extendedClient, nam
 	}
 
 	// Now create the service and attach the credentials
-	for _, container := range podTemplate.Spec.Containers {
-		r, err := image.ParseRef(container.Image)
-		if err != nil {
-			log("err", err.Error())
-			continue
-		}
-		imageCreds[r.Name] = creds
-	}
-	for _, container := range podTemplate.Spec.InitContainers {
-		r, err := image.ParseRef(container.Image)
-		if err != nil {
-			log("err", err.Error())
-			continue
-		}
-		imageCreds[r.Name] = creds
+	for _, image := range images {
+		imageCreds[image] = creds
 	}
 }
 
@@ -125,7 +145,7 @@ func (c *Cluster) ImagesToFetch() registry.ImageCreds {
 			imageCreds := make(registry.ImageCreds)
 			for _, podController := range podControllers {
 				logger := log.With(c.logger, "resource", flux.MakeResourceID(ns.Name, kind, podController.name))
-				mergeCredentials(logger.Log, c.client, ns.Name, podController.podTemplate, imageCreds, seenCreds)
+				mergeCredentials(logger.Log, c.includeImage, c.client, ns.Name, podController.podTemplate, imageCreds, seenCreds)
 			}
 
 			// Merge creds
@@ -142,3 +162,12 @@ func (c *Cluster) ImagesToFetch() registry.ImageCreds {
 
 	return allImageCreds
 }
+
+func (c *Cluster) includeImage(imageName string) bool {
+	for _, exp := range c.imageExcludeList {
+		if glob.Glob(exp, imageName) {
+			return false
+		}
+	}
+	return true
+}

cluster/kubernetes/images_test.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"testing"
 
+	"github.com/ryanuber/go-glob"
 	"github.com/stretchr/testify/assert"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes/fake"
@@ -66,11 +67,56 @@ func TestMergeCredentials(t *testing.T) {
 	client := extendedClient{clientset, nil}
 
 	creds := registry.ImageCreds{}
-	mergeCredentials(noopLog, client, ns, spec, creds, make(map[string]registry.Credentials))
+
+	mergeCredentials(noopLog, func(imageName string) bool { return true },
+		client, ns, spec, creds, make(map[string]registry.Credentials))
 
 	// check that we accumulated some credentials
 	assert.Contains(t, creds, ref.Name)
 	c := creds[ref.Name]
 	hosts := c.Hosts()
 	assert.ElementsMatch(t, []string{"docker.io", "quay.io"}, hosts)
 }
+
+func TestMergeCredentials_ImageExclusion(t *testing.T) {
+	creds := registry.ImageCreds{}
+	gcrImage, _ := image.ParseRef("gcr.io/foo/bar:tag")
+	k8sImage, _ := image.ParseRef("k8s.gcr.io/foo/bar:tag")
+	testImage, _ := image.ParseRef("docker.io/test/bar:tag")
+
+	spec := apiv1.PodTemplateSpec{
+		Spec: apiv1.PodSpec{
+			InitContainers: []apiv1.Container{
+				{Name: "container1", Image: testImage.String()},
+			},
+			Containers: []apiv1.Container{
+				{Name: "container1", Image: k8sImage.String()},
+				{Name: "container2", Image: gcrImage.String()},
+			},
+		},
+	}
+
+	clientset := fake.NewSimpleClientset()
+	client := extendedClient{clientset, nil}
+
+	var includeImage = func(imageName string) bool {
+		for _, exp := range []string{"k8s.gcr.io/*", "*test*"} {
+			if glob.Glob(exp, imageName) {
+				return false
+			}
+		}
+		return true
+	}
+
+	mergeCredentials(noopLog, includeImage, client, "default", spec, creds,
+		make(map[string]registry.Credentials))
+
+	// check test image has been excluded
+	assert.NotContains(t, creds, testImage.Name)
+
+	// check k8s.gcr.io image has been excluded
+	assert.NotContains(t, creds, k8sImage.Name)
+
+	// check gcr.io image exists
+	assert.Contains(t, creds, gcrImage.Name)
+}

cluster/kubernetes/kubernetes.go

@@ -104,7 +104,8 @@ type Cluster struct {
 	nsWhitelist       []string
 	nsWhitelistLogged map[string]bool // to keep track of whether we've logged a problem with seeing a whitelisted ns
 
-	mu sync.Mutex
+	imageExcludeList []string
+	mu               sync.Mutex
 }
 
 // NewCluster returns a usable cluster.
@@ -113,7 +114,8 @@ func NewCluster(clientset k8sclient.Interface,
 	applier Applier,
 	sshKeyRing ssh.KeyRing,
 	logger log.Logger,
-	nsWhitelist []string) *Cluster {
+	nsWhitelist []string,
+	imageExcludeList []string) *Cluster {
 
 	c := &Cluster{
 		client: extendedClient{
@@ -125,6 +127,7 @@ func NewCluster(clientset k8sclient.Interface,
 		sshKeyRing:        sshKeyRing,
 		nsWhitelist:       nsWhitelist,
 		nsWhitelistLogged: map[string]bool{},
+		imageExcludeList:  imageExcludeList,
 	}
 
 	return c
@@ -178,16 +181,16 @@ func (c *Cluster) AllControllers(namespace string) (res []cluster.Controller, er
 			podControllers, err := resourceKind.getPodControllers(c, ns.Name)
 			if err != nil {
 				if se, ok := err.(*apierrors.StatusError); ok {
-					switch (se.ErrStatus.Reason) {
-						case meta_v1.StatusReasonNotFound:
-							// Kind not supported by API server, skip
-							continue
-						case meta_v1.StatusReasonForbidden:
-							// K8s can return forbidden instead of not found for non super admins
-							c.logger.Log("warning", "not allowed to list resources", "err", err)
-							continue
-						default:
-							return nil, err
+					switch se.ErrStatus.Reason {
+					case meta_v1.StatusReasonNotFound:
+						// Kind not supported by API server, skip
+						continue
+					case meta_v1.StatusReasonForbidden:
+						// K8s can return forbidden instead of not found for non super admins
+						c.logger.Log("warning", "not allowed to list resources", "err", err)
+						continue
+					default:
+						return nil, err
 					}
 				} else {
 					return nil, err
@@ -291,16 +294,16 @@ func (c *Cluster) Export() ([]byte, error) {
 			podControllers, err := resourceKind.getPodControllers(c, ns.Name)
 			if err != nil {
 				if se, ok := err.(*apierrors.StatusError); ok {
-					switch (se.ErrStatus.Reason) {
-						case meta_v1.StatusReasonNotFound:
-							// Kind not supported by API server, skip
-							continue
-						case meta_v1.StatusReasonForbidden:
-							// K8s can return forbidden instead of not found for non super admins
-							c.logger.Log("warning", "not allowed to list resources", "err", err)
-							continue
-						default:
-							return nil, err
+					switch se.ErrStatus.Reason {
+					case meta_v1.StatusReasonNotFound:
+						// Kind not supported by API server, skip
+						continue
+					case meta_v1.StatusReasonForbidden:
+						// K8s can return forbidden instead of not found for non super admins
+						c.logger.Log("warning", "not allowed to list resources", "err", err)
+						continue
+					default:
+						return nil, err
 					}
 				} else {
 					return nil, err

cluster/kubernetes/kubernetes_test.go

@@ -26,7 +26,7 @@ func testGetAllowedNamespaces(t *testing.T, namespace []string, expected []strin
 	clientset := fakekubernetes.NewSimpleClientset(newNamespace("default"),
 		newNamespace("kube-system"))
 
-	c := NewCluster(clientset, nil, nil, nil, log.NewNopLogger(), namespace)
+	c := NewCluster(clientset, nil, nil, nil, log.NewNopLogger(), namespace, []string{})
 
 	namespaces, err := c.getAllowedNamespaces()
 	if err != nil {

cmd/fluxd/main.go

@@ -74,17 +74,17 @@ func main() {
 	}
 	// This mirrors how kubectl extracts information from the environment.
 	var (
-		listenAddr        = fs.StringP("listen", "l", ":3030", "Listen address where /metrics and API will be served")
-		listenMetricsAddr = fs.String("listen-metrics", "", "Listen address for /metrics endpoint")
-		kubernetesKubectl = fs.String("kubernetes-kubectl", "", "Optional, explicit path to kubectl tool")
-		versionFlag       = fs.Bool("version", false, "Get version number")
+		listenAddr        = fs.StringP("listen", "l", ":3030", "listen address where /metrics and API will be served")
+		listenMetricsAddr = fs.String("listen-metrics", "", "listen address for /metrics endpoint")
+		kubernetesKubectl = fs.String("kubernetes-kubectl", "", "optional, explicit path to kubectl tool")
+		versionFlag       = fs.Bool("version", false, "get version number")
 		// Git repo & key etc.
 		gitURL       = fs.String("git-url", "", "URL of git repo with Kubernetes manifests; e.g., [email protected]:weaveworks/flux-get-started")
 		gitBranch    = fs.String("git-branch", "master", "branch of git repo to use for Kubernetes manifests")
 		gitPath      = fs.StringSlice("git-path", []string{}, "relative paths within the git repo to locate Kubernetes manifests")
 		gitUser      = fs.String("git-user", "Weave Flux", "username to use as git committer")
 		gitEmail     = fs.String("git-email", "[email protected]", "email to use as git committer")
-		gitSetAuthor = fs.Bool("git-set-author", false, "If set, the author of git commits will reflect the user who initiated the commit and will differ from the git committer.")
+		gitSetAuthor = fs.Bool("git-set-author", false, "if set, the author of git commits will reflect the user who initiated the commit and will differ from the git committer.")
 		gitLabel     = fs.String("git-label", "", "label to keep track of sync progress; overrides both --git-sync-tag and --git-notes-ref")
 		// Old git config; still used if --git-label is not supplied, but --git-label is preferred.
 		gitSyncTag     = fs.String("git-sync-tag", defaultGitSyncTag, "tag to use to mark sync progress for this cluster")
@@ -97,32 +97,33 @@ func main() {
 		// syncing
 		syncInterval = fs.Duration("sync-interval", 5*time.Minute, "apply config in git to cluster at least this often, even if there are no new commits")
 		// registry
-		memcachedHostname    = fs.String("memcached-hostname", "memcached", "Hostname for memcached service.")
-		memcachedTimeout     = fs.Duration("memcached-timeout", time.Second, "Maximum time to wait before giving up on memcached requests.")
+		memcachedHostname    = fs.String("memcached-hostname", "memcached", "hostname for memcached service.")
+		memcachedTimeout     = fs.Duration("memcached-timeout", time.Second, "maximum time to wait before giving up on memcached requests.")
 		memcachedService     = fs.String("memcached-service", "memcached", "SRV service used to discover memcache servers.")
 		registryPollInterval = fs.Duration("registry-poll-interval", 5*time.Minute, "period at which to check for updated images")
 		registryRPS          = fs.Float64("registry-rps", 50, "maximum registry requests per second per host")
 		registryBurst        = fs.Int("registry-burst", defaultRemoteConnections, "maximum number of warmer connections to remote and memcache")
 		registryTrace        = fs.Bool("registry-trace", false, "output trace of image registry requests to log")
 		registryInsecure     = fs.StringSlice("registry-insecure-host", []string{}, "use HTTP for this image registry domain (e.g., registry.cluster.local), instead of HTTPS")
+		registryExcludeImage = fs.StringSlice("registry-exclude-image", []string{"k8s.gcr.io/*"}, "do not scan images that match these glob expressions; the default is to exclude the 'k8s.gcr.io/*' images")
 
 		// AWS authentication
-		registryAWSRegions         = fs.StringSlice("registry-ecr-region", nil, "Restrict ECR scanning to these AWS regions; if empty, only the cluster's region will be scanned")
-		registryAWSAccountIDs      = fs.StringSlice("registry-ecr-include-id", nil, "Restrict ECR scanning to these AWS account IDs; if empty, all account IDs that aren't excluded may be scanned")
-		registryAWSBlockAccountIDs = fs.StringSlice("registry-ecr-exclude-id", []string{registry.EKS_SYSTEM_ACCOUNT}, "Do not scan ECR for images in these AWS account IDs; the default is to exclude the EKS system account")
+		registryAWSRegions         = fs.StringSlice("registry-ecr-region", nil, "restrict ECR scanning to these AWS regions; if empty, only the cluster's region will be scanned")
+		registryAWSAccountIDs      = fs.StringSlice("registry-ecr-include-id", nil, "restrict ECR scanning to these AWS account IDs; if empty, all account IDs that aren't excluded may be scanned")
+		registryAWSBlockAccountIDs = fs.StringSlice("registry-ecr-exclude-id", []string{registry.EKS_SYSTEM_ACCOUNT}, "do not scan ECR for images in these AWS account IDs; the default is to exclude the EKS system account")
 
 		// k8s-secret backed ssh keyring configuration
-		k8sSecretName            = fs.String("k8s-secret-name", "flux-git-deploy", "Name of the k8s secret used to store the private SSH key")
-		k8sSecretVolumeMountPath = fs.String("k8s-secret-volume-mount-path", "/etc/fluxd/ssh", "Mount location of the k8s secret storing the private SSH key")
-		k8sSecretDataKey         = fs.String("k8s-secret-data-key", "identity", "Data key holding the private SSH key within the k8s secret")
-		k8sNamespaceWhitelist    = fs.StringSlice("k8s-namespace-whitelist", []string{}, "Experimental, optional: restrict the view of the cluster to the namespaces listed. All namespaces are included if this is not set.")
+		k8sSecretName            = fs.String("k8s-secret-name", "flux-git-deploy", "name of the k8s secret used to store the private SSH key")
+		k8sSecretVolumeMountPath = fs.String("k8s-secret-volume-mount-path", "/etc/fluxd/ssh", "mount location of the k8s secret storing the private SSH key")
+		k8sSecretDataKey         = fs.String("k8s-secret-data-key", "identity", "data key holding the private SSH key within the k8s secret")
+		k8sNamespaceWhitelist    = fs.StringSlice("k8s-namespace-whitelist", []string{}, "experimental, optional: restrict the view of the cluster to the namespaces listed. All namespaces are included if this is not set.")
 		// SSH key generation
 		sshKeyBits   = optionalVar(fs, &ssh.KeyBitsValue{}, "ssh-keygen-bits", "-b argument to ssh-keygen (default unspecified)")
 		sshKeyType   = optionalVar(fs, &ssh.KeyTypeValue{}, "ssh-keygen-type", "-t argument to ssh-keygen (default unspecified)")
 		sshKeygenDir = fs.String("ssh-keygen-dir", "", "directory, ideally on a tmpfs volume, in which to generate new SSH keys when necessary")
 
-		upstreamURL = fs.String("connect", "", "Connect to an upstream service e.g., Weave Cloud, at this base address")
-		token       = fs.String("token", "", "Authentication token for upstream service")
+		upstreamURL = fs.String("connect", "", "connect to an upstream service e.g., Weave Cloud, at this base address")
+		token       = fs.String("token", "", "authentication token for upstream service")
 
 		dockerConfig = fs.String("docker-config", "", "path to a docker config to use for image registry credentials")
 
@@ -258,7 +259,7 @@ func main() {
 		logger.Log("kubectl", kubectl)
 
 		kubectlApplier := kubernetes.NewKubectl(kubectl, restClientConfig)
-		k8sInst := kubernetes.NewCluster(clientset, ifclientset, kubectlApplier, sshKeyRing, logger, *k8sNamespaceWhitelist)
+		k8sInst := kubernetes.NewCluster(clientset, ifclientset, kubectlApplier, sshKeyRing, logger, *k8sNamespaceWhitelist, *registryExcludeImage)
 
 		if err := k8sInst.Ping(); err != nil {
 			logger.Log("ping", err)

site/daemon.md

@@ -73,6 +73,7 @@ fluxd requires setup and offers customization though a multitude of flags.
 |--registry-rps          | `200`                           | maximum registry requests per second per host|
 |--registry-burst        | `125`      | maximum number of warmer connections to remote and memcache|
 |--registry-insecure-host| []         | registry hosts to use HTTP for (instead of HTTPS) |
+|--registry-exclude-image| `["k8s.gcr.io/*"]` | do not scan images that match these glob expressions |
 |--docker-config         | `""`       | path to a Docker config file with default image registry credentials |
 |--registry-ecr-region   | `[]`       | Allow these AWS regions when scanning images from ECR (multiple values alllowed); defaults to the detected cluster region |
 |--registry-ecr-include-id | `[]`       | Include these AWS account ID(s) when scanning images in ECR (multiple values allowed); empty means allow all, unless excluded |