feat(bundle): rego file selector.

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ### Not released yet
 
+FEATURES:
+
+* bundle/selector:
+  * support `regoFile` to load a Rego filter policy from a file.
+  * `cel` query language
+    * `p.match_label(globstring, globstring)` can be used to match label key and value
+    * `p.match_annotation(globstring, globstring)` can be used to match annotation key and value
+
 ## 0.2.6
 
 ### 2022-02-07

api/proto/harp/bundle/v1/patch.proto

@@ -81,10 +81,12 @@ message PatchSelector {
   string jmesPath = 2;
   // Match a package using a Rego policy.
   string rego = 3;
+  // Match a package using a REgo policy stored in an external file.
+  string regoFile = 4;
   // Match a package by secret.
-  PatchSelectorMatchSecret matchSecret = 4;
+  PatchSelectorMatchSecret matchSecret = 5;
   // Match a package using CEL expressions.
-  repeated string cel = 5;
+  repeated string cel = 6;
 }
 
 // PatchSelectorMatchPath represents package path matching strategies.

docs/onboarding/3-secret-bundle/4-patch.md

@@ -21,6 +21,7 @@ bundle source without altering the source bundle.
       - [Match by regex path](#match-by-regex-path)
       - [Match by JMES filter](#match-by-jmes-filter)
       - [Match by Rego policy](#match-by-rego-policy)
+      - [Match by Rego policy file](#match-by-rego-policy-file)
       - [Match by CEL expression](#match-by-cel-expression)
       - [Match by secret key](#match-by-secret-key)
       - [PatchSelectorMatchPath](#patchselectormatchpath)
@@ -47,6 +48,7 @@ bundle source without altering the source bundle.
 * `remove` is used to remove a key from the map
 * `update` is used to update a value from an existing `key` only
 * `replaceKeys` is used to rename a key to another key in the map.
+* `removeKeys` is used to remove all keys that match one of the given regex patterns.
 
 > All keys and values can contain template instructions.
 
@@ -186,10 +188,12 @@ message PatchSelector {
   string jmesPath = 2;
   // Match a package using a Rego policy.
   string rego = 3;
+  // Match a package using a REgo policy stored in an external file.
+  string regoFile = 4;
   // Match a package by secret.
-  PatchSelectorMatchSecret matchSecret = 4;
+  PatchSelectorMatchSecret matchSecret = 5;
   // Match a package using CEL expressions.
-  repeated string cel = 5;
+  repeated string cel = 6;
 }
 ```
 
@@ -281,6 +285,13 @@ spec:
 
 ```
 
+#### Match by Rego policy file
+
+```yaml
+selector:
+  regoFile: deprecation.rego
+```
+
 #### Match by CEL expression
 
 ```yaml

docs/onboarding/3-secret-bundle/5-ruleset.md

@@ -13,7 +13,9 @@
 
 * `p.match_path(globstring) bool` - Returns true if the package name match the given Glob pattern.
 * `p.match_label(globstring) bool` - Returns true if one of the package labels match the given Glob pattern.
+* `p.match_label(globstring, globstring) bool` - Returns true if one of the package labels and the associated value match the given Glob patterns.
 * `p.match_annotation(globstring) bool` - Returns true if one of the package annotations match the given Glob pattern.
+* `p.match_annotation(globstring, globstring) bool` - Returns true if one of the package annotations and the associated value match the given Glob patterns.
 * `p.match_secret(globstring) bool` - Returns true if the package has a secret with given pattern.
 * `p.has_secret(string) bool` - Returns true if the package has a secret with given key.
 * `p.has_all_secrets(...string) bool` - Returns true if the package has all secrets with given keys.

pkg/bundle/patch/executor.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"regexp"
 
 	"github.com/imdario/mergo"
@@ -188,6 +189,17 @@ func compileSelector(s *bundlev1.PatchSelector, values map[string]interface{}) (
 		}
 	}
 
+	if s.RegoFile != "" {
+		// Read policy file
+		policyFile, err := os.ReadFile(s.RegoFile)
+		if err != nil {
+			return nil, fmt.Errorf("unable to open rego policy file: %w", err)
+		}
+
+		// Build the specification
+		return selector.MatchRego(context.Background(), string(policyFile))
+	}
+
 	// Has rego policy
 	if s.Rego != "" {
 		// Return specification

pkg/bundle/patch/reader_test.go

@@ -49,6 +49,9 @@ func generateReaderTests(t *testing.T, rootPath, state string, wantErr bool) []r
 		if info.IsDir() {
 			return nil
 		}
+		if filepath.Ext(path) != "yaml" {
+			return nil
+		}
 
 		tests = append(tests, readerTestCase{
 			name:    fmt.Sprintf("%s-%s", state, filepath.Base(info.Name())),

pkg/bundle/ruleset/linter/engine/cel/engine_test.go

@@ -47,6 +47,10 @@ func TestNew(t *testing.T) {
 			name: "funcs",
 			args: args{
 				expressions: []string{
+					`p.match_label("test")`,
+					`p.match_label("test", "true")`,
+					`p.match_annotation("namespace/v1/test")`,
+					`p.match_annotation("namespace/v1/test", "testing")`,
 					`p.match_path("app/production/test")`,
 					`p.has_secret("test") && p.secret("test").is_base64()`,
 					`p.has_all_secrets(["test","test2"])`,

pkg/bundle/ruleset/linter/engine/cel/ext/package.go

@@ -51,12 +51,20 @@ func (packageLib) CompileOptions() []cel.EnvOption {
 					[]*exprpb.Type{harpPackageObjectType, decls.String},
 					decls.Bool,
 				),
+				decls.NewInstanceOverload("package_match_label_string_string",
+					[]*exprpb.Type{harpPackageObjectType, decls.String, decls.String},
+					decls.Bool,
+				),
 			),
 			decls.NewFunction("match_annotation",
 				decls.NewInstanceOverload("package_match_annotation_string",
 					[]*exprpb.Type{harpPackageObjectType, decls.String},
 					decls.Bool,
 				),
+				decls.NewInstanceOverload("package_match_annotation_string_string",
+					[]*exprpb.Type{harpPackageObjectType, decls.String, decls.String},
+					decls.Bool,
+				),
 			),
 			decls.NewFunction("match_path",
 				decls.NewInstanceOverload("package_match_path_string",
@@ -113,10 +121,18 @@ func (packageLib) ProgramOptions() []cel.ProgramOption {
 				Operator: "package_match_label_string",
 				Binary:   celPackageMatchLabel,
 			},
+			&functions.Overload{
+				Operator: "package_match_label_string_string",
+				Function: celPackageMatchLabelValue,
+			},
 			&functions.Overload{
 				Operator: "package_match_annotation_string",
 				Binary:   celPackageMatchAnnotation,
 			},
+			&functions.Overload{
+				Operator: "package_match_annotation_string_string",
+				Function: celPackageMatchAnnotationValue,
+			},
 			&functions.Overload{
 				Operator: "package_match_path_string",
 				Binary:   celPackageMatchPath,
@@ -174,6 +190,50 @@ func celPackageMatchLabel(lhs, rhs ref.Val) ref.Val {
 	return types.Bool(false)
 }
 
+func celPackageMatchLabelValue(values ...ref.Val) ref.Val {
+	if len(values) != 3 {
+		return types.Bool(false)
+	}
+
+	lhs := values[0]
+	x, _ := lhs.ConvertToNative(reflect.TypeOf(&bundlev1.Package{}))
+	p, ok := x.(*bundlev1.Package)
+	if !ok {
+		return types.Bool(false)
+	}
+
+	keyPatternTyped, ok := values[1].(types.String)
+	if !ok {
+		return types.Bool(false)
+	}
+
+	keyPattern, ok := keyPatternTyped.Value().(string)
+	if !ok {
+		return types.Bool(false)
+	}
+
+	valuePatternTyped, ok := values[2].(types.String)
+	if !ok {
+		return types.Bool(false)
+	}
+
+	valuePattern, ok := valuePatternTyped.Value().(string)
+	if !ok {
+		return types.Bool(false)
+	}
+
+	km := glob.MustCompile(keyPattern)
+	vm := glob.MustCompile(valuePattern)
+
+	for k, v := range p.Labels {
+		if km.Match(k) && vm.Match(v) {
+			return types.Bool(true)
+		}
+	}
+
+	return types.Bool(false)
+}
+
 func celPackageMatchAnnotation(lhs, rhs ref.Val) ref.Val {
 	x, _ := lhs.ConvertToNative(reflect.TypeOf(&bundlev1.Package{}))
 	p, ok := x.(*bundlev1.Package)
@@ -201,6 +261,50 @@ func celPackageMatchAnnotation(lhs, rhs ref.Val) ref.Val {
 	return types.Bool(false)
 }
 
+func celPackageMatchAnnotationValue(values ...ref.Val) ref.Val {
+	if len(values) != 3 {
+		return types.Bool(false)
+	}
+
+	lhs := values[0]
+	x, _ := lhs.ConvertToNative(reflect.TypeOf(&bundlev1.Package{}))
+	p, ok := x.(*bundlev1.Package)
+	if !ok {
+		return types.Bool(false)
+	}
+
+	keyPatternTyped, ok := values[1].(types.String)
+	if !ok {
+		return types.Bool(false)
+	}
+
+	keyPattern, ok := keyPatternTyped.Value().(string)
+	if !ok {
+		return types.Bool(false)
+	}
+
+	valuePatternTyped, ok := values[2].(types.String)
+	if !ok {
+		return types.Bool(false)
+	}
+
+	valuePattern, ok := valuePatternTyped.Value().(string)
+	if !ok {
+		return types.Bool(false)
+	}
+
+	km := glob.MustCompile(keyPattern)
+	vm := glob.MustCompile(valuePattern)
+
+	for k, v := range p.Annotations {
+		if km.Match(k) && vm.Match(v) {
+			return types.Bool(true)
+		}
+	}
+
+	return types.Bool(false)
+}
+
 func celPackageMatchPath(lhs, rhs ref.Val) ref.Val {
 	x, _ := lhs.ConvertToNative(reflect.TypeOf(&bundlev1.Package{}))
 	p, ok := x.(*bundlev1.Package)

pkg/bundle/selector/match_secret.go

@@ -21,8 +21,9 @@ import (
 	"regexp"
 	"strings"
 
-	bundlev1 "github.com/elastic/harp/api/gen/go/harp/bundle/v1"
 	"github.com/gobwas/glob"
+
+	bundlev1 "github.com/elastic/harp/api/gen/go/harp/bundle/v1"
 )
 
 // MatchSecretStrict returns a secret key matcher specification with strict profile.

test/fixtures/patch/valid/cel-remove-packages.yaml

@@ -0,0 +1,15 @@
+apiVersion: harp.elastic.co/v1
+kind: BundlePatch
+meta:
+  name: "cel-package-remover"
+  owner: [email protected]
+  description: "Remove a targeted package with CEL"
+spec:
+  rules:
+    - selector:
+        cel:
+          - p.match_label("to-remove")
+
+      package:
+        # Flag to be removed
+        remove: true

test/fixtures/patch/valid/regofile-remove-packages.yaml

@@ -0,0 +1,13 @@
+apiVersion: harp.elastic.co/v1
+kind: BundlePatch
+meta:
+  name: "regofile-package-remover"
+  owner: [email protected]
+  description: "Remove a targeted package with Rego file"
+spec:
+  rules:
+    - selector:
+        regoFile: remover.rego
+      package:
+        # Flag to be removed
+        remove: true

test/fixtures/patch/valid/remover.rego

@@ -0,0 +1,7 @@
+package harp
+
+default matched = false
+
+matched {
+    input.labels["to-remove"]
+}