Merge pull request #1199 from rdpsin/sc-tags

Adding tagging support through StorageClass.parameters

cmd/main.go

@@ -51,6 +51,7 @@ func main() {
 		driver.WithVolumeAttachLimit(options.NodeOptions.VolumeAttachLimit),
 		driver.WithKubernetesClusterID(options.ControllerOptions.KubernetesClusterID),
 		driver.WithAwsSdkDebugLog(options.ControllerOptions.AwsSdkDebugLog),
+		driver.WithWarnOnInvalidTag(options.ControllerOptions.WarnOnInvalidTag),
 	)
 	if err != nil {
 		klog.Fatalln(err)

cmd/options/controller_options.go

@@ -35,11 +35,14 @@ type ControllerOptions struct {
 	KubernetesClusterID string
 	// flag to enable sdk debug log
 	AwsSdkDebugLog bool
+	// flag to warn on invalid tag, instead of returning an error
+	WarnOnInvalidTag bool
 }
 
 func (s *ControllerOptions) AddFlags(fs *flag.FlagSet) {
 	fs.Var(cliflag.NewMapStringString(&s.ExtraTags), "extra-tags", "Extra tags to attach to each dynamically provisioned resource. It is a comma separated list of key value pairs like '<key1>=<value1>,<key2>=<value2>'")
 	fs.Var(cliflag.NewMapStringString(&s.ExtraVolumeTags), "extra-volume-tags", "DEPRECATED: Please use --extra-tags instead. Extra volume tags to attach to each dynamically provisioned volume. It is a comma separated list of key value pairs like '<key1>=<value1>,<key2>=<value2>'")
 	fs.StringVar(&s.KubernetesClusterID, "k8s-tag-cluster-id", "", "ID of the Kubernetes cluster used for tagging provisioned EBS volumes (optional).")
 	fs.BoolVar(&s.AwsSdkDebugLog, "aws-sdk-debug-log", false, "To enable the aws sdk debug log level (default to false).")
+	fs.BoolVar(&s.WarnOnInvalidTag, "warn-on-invalid-tag", false, "To warn on invalid tags, instead of returning an error")
 }

docs/README.md

@@ -51,6 +51,9 @@ To help manage volumes in the aws account, CSI driver will automatically add tag
 | kubernetes.io/cluster/X| owned                     | kubernetes.io/cluster/aws-cluster-id-1 = owned                      | add to all volumes and snapshots if k8s-tag-cluster-id argument is set to X.|
 | extra-key              | extra-value               | extra-key = extra-value                                             | add to all volumes and snapshots if extraTags argument is set|
 
+
+The CSI driver also supports passing tags through `StorageClass.parameters`. For more information, please refer to the [tagging doc](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/TAGGING.md).
+
 ## Driver Options
 There are couple driver options that can be passed as arguments when starting driver container.
 

docs/TAGGING.md

@@ -0,0 +1,103 @@
+# Tagging
+
+The AWS EBS CSI Driver supports tagging through `StorageClass.parameters`. 
+
+If a key has the prefix `tagSpecification`, the CSI driver will treat the value as a key-value pair to be applied to the dynamically provisioned volume as tags.
+
+
+**Example 1**
+```
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: ebs-sc
+provisioner: ebs.csi.aws.com
+parameters:
+  tagSpecification_1: "key1=value1"
+  tagSpecification_2: "key2=hello world"
+  tagSpecification_3: "key3="
+```
+
+Provisioning a volume using this StorageClass will apply two tags:
+
+```
+key1=value1
+key2=hello world
+key3=<empty string>
+```
+
+________
+
+To allow for PV-level granularity, the CSI driver support runtime string interpolation on the tag values. You can specify placeholders for PVC namespace, PVC name and PV name, which will then be dynamically computed at runtime.
+
+**NOTE: This requires the `--extra-create-metadata` flag to be enabled on the `external-provisioner` sidecar.**
+
+**Example 2**
+```
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: ebs-sc
+provisioner: ebs.csi.aws.com
+parameters:
+  tagSpecification_1: "pvcnamespace={{ .PVCNamespace }}"
+  tagSpecification_2: "pvcname={{ .PVCName }}"
+  tagSpecification_3: "pvname={{ .PVName }}"
+```
+Provisioning a volume using this StorageClass, with a PVC named 'ebs-claim' in namespace 'default', will apply the following tags:
+
+```
+pvcnamespace=default
+pvcname=ebs-claim
+pvname=<the computed pv name>
+```
+
+
+_________
+
+The driver uses Go's `text/template` package for string interpolation. As such, cluster admins are free to use the constructs provided by the package (except for certain function, see `Failure Modes` below). To aid cluster admins to be more expressive, certain functions have been provided.
+
+They include:
+
+-   **field** delim index str: Split `str` by `delim` and extract the  word at position `index`.
+-   **substring** start end str: Get a substring of `str` given the `start` and `end` indices
+-   **toUpper** str: Convert `str` to uppercase
+-   **toLower** str: Convert `str` to lowercase
+-   **contains** str1 str2: Returns a boolean if `str2` contains `str1`
+
+
+**Example 3**
+```
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: ebs-sc
+provisioner: ebs.csi.aws.com
+parameters:
+  tagSpecification_1: 'backup={{ .PVCNamespace | contains "prod" }}'
+  tagSpecification_2: 'billingID={{ .PVCNamespace | field "-" 2 | toUpper }}'
+```
+
+Assuming the PVC namespace is `ns-prod-abcdef`, the attached tags will be
+
+```
+backup=true
+billingID=ABCDEF
+```
+
+____
+
+## Failure Modes
+
+There can be multipe failure modes:
+
+* The template cannot be parsed.
+* The key/interpolated value do not meet the [AWS Tag Requirements](https://docs.aws.amazon.com/general/latest/gr/aws_tagging.html)
+* The key is not allowed (such as keys used internally by the CSI driver e.g., 'CSIVolumeName').
+* The template uses one of the disabled function calls. The driver disables the following `text/template` functions: `js`, `call`, `html`, `urlquery`. 
+
+In this case, the CSI driver will not provision a volume, but instead return an error.
+
+The driver also defines another flag, `--warn-on-invalid-tag` that will (if set), instead of returning an error, log a warning and skip the offending tag.
+
+

go.mod

@@ -5,6 +5,7 @@ require (
 	github.com/container-storage-interface/spec v1.3.0
 	github.com/golang/mock v1.5.0
 	github.com/golang/protobuf v1.4.3
+	github.com/google/go-cmp v0.5.5
 	github.com/kubernetes-csi/csi-proxy/client v1.0.1
 	github.com/kubernetes-csi/csi-test v2.0.0+incompatible
 	github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0
@@ -36,7 +37,6 @@ require (
 	github.com/go-logr/logr v0.4.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
-	github.com/google/go-cmp v0.5.5 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/uuid v1.1.2 // indirect
 	github.com/googleapis/gnostic v0.4.1 // indirect

pkg/cloud/cloud.go

@@ -98,6 +98,8 @@ const (
 const (
 	// MaxNumTagsPerResource represents the maximum number of tags per AWS resource.
 	MaxNumTagsPerResource = 50
+	// MinTagKeyLength represents the minimum key length for a tag.
+	MinTagKeyLength = 1
 	// MaxTagKeyLength represents the maximum key length for a tag.
 	MaxTagKeyLength = 128
 	// MaxTagValueLength represents the maximum value length for a tag.

pkg/driver/constants.go

@@ -68,6 +68,10 @@ const (
 	// PVNameKey contains name of the final PV that will be used for the dynamically
 	// provisioned volume
 	PVNameKey = "csi.storage.k8s.io/pv/name"
+
+	// TagKeyPrefix contains the prefix of a volume parameter that designates it as
+	// a tag to be attached to the resource
+	TagKeyPrefix = "tagSpecification"
 )
 
 // constants for volume tags and their values

pkg/driver/controller.go

@@ -29,6 +29,7 @@ import (
 	"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud"
 	"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver/internal"
 	"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util"
+	"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util/template"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 	"k8s.io/klog"
@@ -123,12 +124,15 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol
 		throughput             int
 		isEncrypted            bool
 		kmsKeyID               string
+		scTags                 []string
 		volumeTags             = map[string]string{
 			cloud.VolumeNameTagKey:   volName,
 			cloud.AwsEbsDriverTagKey: isManagedByDriver,
 		}
 	)
 
+	tProps := new(template.Props)
+
 	for key, value := range req.GetParameters() {
 		switch strings.ToLower(key) {
 		case "fstype":
@@ -162,12 +166,19 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol
 			kmsKeyID = value
 		case PVCNameKey:
 			volumeTags[PVCNameTag] = value
+			tProps.PVCName = value
 		case PVCNamespaceKey:
 			volumeTags[PVCNamespaceTag] = value
+			tProps.PVCNamespace = value
 		case PVNameKey:
 			volumeTags[PVNameTag] = value
+			tProps.PVName = value
 		default:
-			return nil, status.Errorf(codes.InvalidArgument, "Invalid parameter key %s for CreateVolume", key)
+			if strings.HasPrefix(key, TagKeyPrefix) {
+				scTags = append(scTags, value)
+			} else {
+				return nil, status.Errorf(codes.InvalidArgument, "Invalid parameter key %s for CreateVolume", key)
+			}
 		}
 	}
 
@@ -205,6 +216,19 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol
 		volumeTags[k] = v
 	}
 
+	addTags, err := template.Evaluate(scTags, tProps, d.driverOptions.warnOnInvalidTag)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "Error interpolating the tag value: %v", err)
+	}
+
+	if err := validateExtraTags(addTags, d.driverOptions.warnOnInvalidTag); err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "Invalid tag value: %v", err)
+	}
+
+	for k, v := range addTags {
+		volumeTags[k] = v
+	}
+
 	opts := &cloud.DiskOptions{
 		CapacityBytes:          volSizeBytes,
 		Tags:                   volumeTags,

pkg/driver/driver.go

@@ -66,6 +66,7 @@ type DriverOptions struct {
 	volumeAttachLimit   int64
 	kubernetesClusterID string
 	awsSdkDebugLog      bool
+	warnOnInvalidTag    bool
 }
 
 func NewDriver(options ...func(*DriverOptions)) (*Driver, error) {
@@ -191,3 +192,9 @@ func WithAwsSdkDebugLog(enableSdkDebugLog bool) func(*DriverOptions) {
 		o.awsSdkDebugLog = enableSdkDebugLog
 	}
 }
+
+func WithWarnOnInvalidTag(warnOnInvalidTag bool) func(*DriverOptions) {
+	return func(o *DriverOptions) {
+		o.warnOnInvalidTag = warnOnInvalidTag
+	}
+}

pkg/driver/validation.go

@@ -18,13 +18,15 @@ package driver
 
 import (
 	"fmt"
+	"regexp"
 	"strings"
 
 	"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud"
+	"k8s.io/klog"
 )
 
 func ValidateDriverOptions(options *DriverOptions) error {
-	if err := validateExtraTags(options.extraTags); err != nil {
+	if err := validateExtraTags(options.extraTags, false); err != nil {
 		return fmt.Errorf("Invalid extra tags: %v", err)
 	}
 
@@ -35,14 +37,21 @@ func ValidateDriverOptions(options *DriverOptions) error {
 	return nil
 }
 
-func validateExtraTags(tags map[string]string) error {
+var (
+	/// https://docs.aws.amazon.com/general/latest/gr/aws_tagging.html
+	awsTagValidRegex = regexp.MustCompile(`[a-zA-Z0-9_.:=+\-@]*`)
+)
+
+func validateExtraTags(tags map[string]string, warnOnly bool) error {
 	if len(tags) > cloud.MaxNumTagsPerResource {
 		return fmt.Errorf("Too many tags (actual: %d, limit: %d)", len(tags), cloud.MaxNumTagsPerResource)
 	}
 
-	for k, v := range tags {
+	validate := func(k, v string) error {
 		if len(k) > cloud.MaxTagKeyLength {
 			return fmt.Errorf("Tag key too long (actual: %d, limit: %d)", len(k), cloud.MaxTagKeyLength)
+		} else if len(k) < cloud.MinTagKeyLength {
+			return fmt.Errorf("Tag key cannot be empty (min: 1)")
 		}
 		if len(v) > cloud.MaxTagValueLength {
 			return fmt.Errorf("Tag value too long (actual: %d, limit: %d)", len(v), cloud.MaxTagValueLength)
@@ -62,8 +71,25 @@ func validateExtraTags(tags map[string]string) error {
 		if strings.HasPrefix(k, cloud.AWSTagKeyPrefix) {
 			return fmt.Errorf("Tag key prefix '%s' is reserved", cloud.AWSTagKeyPrefix)
 		}
+		if !awsTagValidRegex.MatchString(k) {
+			return fmt.Errorf("Tag key '%s' is not a valid AWS tag key", k)
+		}
+		if !awsTagValidRegex.MatchString(v) {
+			return fmt.Errorf("Tag value '%s' is not a valid AWS tag value", v)
+		}
+		return nil
 	}
 
+	for k, v := range tags {
+		err := validate(k, v)
+		if err != nil {
+			if warnOnly {
+				klog.Warningf("Skipping tag: the following key-value pair is not valid: (%s, %s): (%v)", k, v, err)
+			} else {
+				return err
+			}
+		}
+	}
 	return nil
 }
 

pkg/driver/validation_test.go

@@ -108,7 +108,7 @@ func TestValidateExtraVolumeTags(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			err := validateExtraTags(tc.tags)
+			err := validateExtraTags(tc.tags, false)
 			if !reflect.DeepEqual(err, tc.expErr) {
 				t.Fatalf("error not equal\ngot:\n%s\nexpected:\n%s", err, tc.expErr)
 			}