Added kuttl tests and cli tests for kyverno 1.10 (#80) (#85)

* Added kuttl tests for kyverno 1.10

* updated e2e workflow yaml

Updated the license key in the helm command

Added Kuttl e2e tests for best practices policy

Updated the kuttl test yaml files

Updated the resource yaml

Kyverno 1.10 policy updates (#79)

* Update policies to use Kyverno 1.10

* Update Kyverno version annotation

* Update Kyverno annotation and e2e tests

Co-authored-by: sathyaseelan <[email protected]>

Makefile

@@ -60,6 +60,9 @@ kind-delete-cluster: $(KIND)
 kind-deploy-kyverno: $(HELM) 
 	@echo Install kyverno chart... >&2
 	@echo $(N4K_LICENSE_KEY) >&2
+
+##	@$(HELM) repo add nirmata https://nirmata.github.io/kyverno-charts
+##	@$(HELM) install kyverno --namespace kyverno --create-namespace nirmata/kyverno --set image.tag=v1.10.0-n4k.nirmata.1 --set initImage.tag=v1.10.0-n4k.nirmata.1 --set cleanupController.image.tag=v1.10.0-n4k.nirmata.1
 
     ### Adding temporary  installation command for the kyverno n4k 1.10
 	git clone -b kyverno-1.10-beta1 https://github.com/nirmata/kyverno-charts.git

charts/best-practices-workload-security/pols/disallow_cri_sock_mount.yaml

@@ -8,6 +8,7 @@ metadata:
     policies.kyverno.io/severity: medium
     policies.kyverno.io/subject: Pod
     policies.kyverno.io/minversion: 1.6.0
+    kyverno.io/kyverno-version: 1.10.0
     policies.kyverno.io/description: >-
       Container daemon socket bind mounts allows access to the container engine on the
       node. This access can be used for privilege escalation and to manage containers

charts/best-practices-workload-security/pols/disallow_default_namespace.yaml

@@ -6,6 +6,7 @@ metadata:
     pod-policies.kyverno.io/autogen-controllers: none
     policies.kyverno.io/title: Disallow Default Namespace
     policies.kyverno.io/minversion: 1.6.0
+    kyverno.io/kyverno-version: 1.10.0
     policies.kyverno.io/category: Multi-Tenancy
     policies.kyverno.io/severity: medium
     policies.kyverno.io/subject: Pod

charts/best-practices-workload-security/pols/disallow_latest_tag.yaml

@@ -6,6 +6,7 @@ metadata:
     policies.kyverno.io/title: Disallow Latest Tag
     policies.kyverno.io/category: Best Practices
     policies.kyverno.io/severity: medium
+    kyverno.io/kyverno-version: 1.10.0
     policies.kyverno.io/subject: Pod
     policies.kyverno.io/description: >-
       The ':latest' tag is mutable and can lead to unexpected errors if the

charts/best-practices-workload-security/pols/require_drop_cap_net_raw.yaml

@@ -6,6 +6,7 @@ metadata:
     policies.kyverno.io/title: Drop CAP_NET_RAW
     policies.kyverno.io/category: Best Practices
     policies.kyverno.io/minversion: 1.6.0
+    kyverno.io/kyverno-version: 1.10.0
     policies.kyverno.io/severity: medium
     policies.kyverno.io/subject: Pod
     policies.kyverno.io/description: >-

charts/best-practices-workload-security/pols/require_labels.yaml

@@ -6,6 +6,7 @@ metadata:
     policies.kyverno.io/title: Require Labels
     policies.kyverno.io/category: Best Practices
     policies.kyverno.io/severity: medium
+    kyverno.io/kyverno-version: 1.10.0
     policies.kyverno.io/subject: Pod, Label
     policies.kyverno.io/description: >-
       Define and use labels that identify semantic attributes of your application or Deployment.

charts/best-practices-workload-security/pols/require_pod_requests_limits.yaml

@@ -8,6 +8,7 @@ metadata:
     policies.kyverno.io/severity: medium
     policies.kyverno.io/subject: Pod
     policies.kyverno.io/minversion: 1.6.0
+    kyverno.io/kyverno-version: 1.10.0
     policies.kyverno.io/description: >-
       As application workloads share cluster resources, it is important to limit resources
       requested and consumed by each Pod. It is recommended to require resource requests and

charts/best-practices-workload-security/pols/require_ro_rootfs.yaml

@@ -8,6 +8,7 @@ metadata:
     policies.kyverno.io/severity: medium
     policies.kyverno.io/subject: Pod
     policies.kyverno.io/minversion: 1.6.0
+    kyverno.io/kyverno-version: 1.10.0
     policies.kyverno.io/description: >-
       A read-only root file system helps to enforce an immutable infrastructure strategy;
       the container only needs to write on the mounted volume that persists the state.

charts/best-practices-workload-security/pols/restrict-service-external-ips.yaml

@@ -6,6 +6,7 @@ metadata:
     policies.kyverno.io/title: Restrict External IPs
     policies.kyverno.io/category: Best Practices
     policies.kyverno.io/severity: medium
+    kyverno.io/kyverno-version: 1.10.0
     policies.kyverno.io/subject: Service
     policies.kyverno.io/description: >-
       Service externalIPs can be used for a MITM attack (CVE-2020-8554).

charts/best-practices-workload-security/pols/restrict_image_registries.yaml

@@ -7,6 +7,7 @@ metadata:
     policies.kyverno.io/category: Best Practices, EKS Best Practices
     policies.kyverno.io/severity: medium
     policies.kyverno.io/minversion: 1.6.0
+    kyverno.io/kubernetes-version: "1.26"
     policies.kyverno.io/subject: Pod
     policies.kyverno.io/description: >-
       Images from unknown, public registries can be of dubious quality and may not be
@@ -28,5 +29,9 @@ spec:
       message: "Unknown image registry."
       pattern:
         spec:
+          =(ephemeralContainers):
+          - image: "eu.foo.io/* | bar.io/*"
+          =(initContainers):
+          - image: "eu.foo.io/* | bar.io/*"
           containers:
           - image: "eu.foo.io/* | bar.io/*"

charts/best-practices-workload-security/pols/restrict_node_port.yaml

@@ -6,6 +6,7 @@ metadata:
     policies.kyverno.io/title: Disallow NodePort
     policies.kyverno.io/category: Best Practices
     policies.kyverno.io/severity: medium
+    kyverno.io/kyverno-version: 1.10.0
     policies.kyverno.io/subject: Service
     policies.kyverno.io/description: >-
       A Kubernetes Service of type NodePort uses a host port to receive traffic from

charts/best-practices-workload-security/templates/e2e/01-policy.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- ../restrict_image_registries.yaml
+assert:
+- policy-assert.yaml

charts/best-practices-workload-security/templates/e2e/02-enforce.yaml

@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+- script: |
+    sed 's/validationFailureAction: audit/validationFailureAction: Enforce/' ../restrict_image_registries.yaml | kubectl apply -f - 

charts/best-practices-workload-security/templates/e2e/03-enforce-policy-assert.yaml

@@ -0,0 +1,11 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: restrict-image-registries
+spec:
+  validationFailureAction: enforce
+status:
+  conditions:
+  - reason: Succeeded
+    status: "True"
+    type: Ready

charts/best-practices-workload-security/templates/e2e/04-manifests.yaml

@@ -0,0 +1,15 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+apply:
+- file: good-pods.yaml
+  shouldFail: false
+- file: good-podcontrollers.yaml
+  shouldFail: false
+- file: bad-pod-noregistry.yaml
+  shouldFail: true
+- file: bad-pod-notall.yaml
+  shouldFail: true
+- file: bad-pod-false.yaml
+  shouldFail: true
+- file: bad-podcontrollers.yaml
+  shouldFail: true

charts/best-practices-workload-security/templates/e2e/05-ephemeral.yaml

@@ -0,0 +1,4 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: if kubectl debug -it goodpod02-registry --image=busybox:1.35 --target=k8s-nginx -n ir-pods-namespace; then exit 1; else exit 0; fi;

charts/best-practices-workload-security/templates/e2e/98-delete.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: kubectl delete deployments --all --force --grace-period=0 -n ir-pods-namespace
+  - command: kubectl delete pods --all --force --grace-period=0 -n ir-pods-namespace
+  - command: kubectl delete cronjobs --all --force --grace-period=0 -n ir-pods-namespace

charts/best-practices-workload-security/templates/e2e/99-delete.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+- apiVersion: kyverno.io/v1
+  kind: ClusterPolicy
+  name: restrict-image-registries

charts/best-practices-workload-security/templates/e2e/bad-pod-false.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: badpod01-registry
+spec:
+  containers:
+  - name: k8s-nginx
+    image: registry.k8s.io/nginx:1.7.9

charts/best-practices-workload-security/templates/e2e/bad-pod-noregistry.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: badpod04-registry
+spec:
+  containers:
+  - name: k8s-nginx
+    image: nginx

charts/best-practices-workload-security/templates/e2e/bad-pod-notall.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: badpod02-registry
+spec:
+  containers:
+  - name: k8s-nginx
+    image: registry.k8s.io/nginx:1.7.9
+  - name: busybox
+    image: bar.io/busybox
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: badpod03-registry
+spec:
+  containers:
+  - name: busybox
+    image: eu.foo.io/busybox
+  - name: k8s-nginx
+    image: registry.k8s.io/nginx:1.7.9

charts/best-practices-workload-security/templates/e2e/bad-podcontrollers.yaml

@@ -0,0 +1,140 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: reqro-baddeployment01
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: app
+  template:
+    metadata:
+      labels:
+        foo: bar
+    spec:
+      initContainers:
+      - name: k8s-nginx-init
+        image: bar.io/nginx
+      - name: busybox-init
+        image: busybox
+      containers:
+      - name: busybox
+        image: busybox:1.35
+      - name: k8s-nginx
+        image: bar.io/nginx
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: reqro-baddeployment02
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: app
+  template:
+    metadata:
+      labels:
+        foo: bar
+    spec:
+      initContainers:
+      - name: k8s-nginx-init
+        image: bar.io/nginx
+      - name: nginx-init
+        image: eu.foo.io/nginx
+      containers:
+      - name: k8s-nginx
+        image: bar.io/nginx
+      - name: busybox
+        image: busybox:1.35
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: reqro-baddeployment03
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: app
+  template:
+    metadata:
+      labels:
+        foo: bar
+    spec:
+      initContainers:
+      - name: k8s-nginx-init
+        image: bar.io/nginx
+      - name: busybox-init
+        image: busybox:1.35
+      containers:
+      - name: k8s-nginx
+        image: bar.io/nginx
+      - name: nginx
+        image: eu.foo.io/nginx
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: reqro-badcronjob01
+spec:
+  schedule: "*/1 * * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          initContainers:
+          - name: k8s-nginx-init
+            image: bar.io/nginx
+          - name: busybox-init
+            image: busybox
+          containers:
+          - name: busybox
+            image: busybox:1.35
+          - name: k8s-nginx
+            image: bar.io/nginx
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: reqro-badcronjob02
+spec:
+  schedule: "*/1 * * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          initContainers:
+          - name: k8s-nginx-init
+            image: bar.io/nginx
+          - name: nginx-init
+            image: eu.foo.io/nginx
+          containers:
+          - name: k8s-nginx
+            image: bar.io/nginx
+          - name: busybox
+            image: busybox:1.35
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: reqro-badcronjob03
+spec:
+  schedule: "*/1 * * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          initContainers:
+          - name: k8s-nginx-init
+            image: bar.io/nginx
+          - name: busybox-init
+            image: busybox:1.35
+          containers:
+          - name: k8s-nginx
+            image: bar.io/nginx
+          - name: nginx
+            image: eu.foo.io/nginx

charts/best-practices-workload-security/templates/e2e/good-podcontrollers.yaml

@@ -0,0 +1,48 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: reqro-gooddeployment01
+  namespace: ir-pods-namespace
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+        foo: bar
+  template:
+    metadata:
+      labels:
+        foo: bar
+    spec:
+      initContainers:
+      - name: k8s-nginx-init
+        image: bar.io/nginx
+      - name: busybox-init
+        image: eu.foo.io/busybox
+      containers:
+      - name: busybox
+        image: eu.foo.io/nginx
+      - name: k8s-nginx
+        image: bar.io/nginx
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: reqprobes-goodcronjob01
+  namespace: ir-pods-namespace
+spec:
+  schedule: "*/1 * * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          initContainers:
+          - name: k8s-nginx-init
+            image: bar.io/nginx
+          - name: busybox-init
+            image: eu.foo.io/busybox
+          containers:
+          - name: busybox
+            image: eu.foo.io/nginx
+          - name: k8s-nginx
+            image: bar.io/nginx