Support KubeSecondaryDNS plugin

Makefile

@@ -159,6 +159,8 @@ gen-manifests: manifest-templator
 	KUBEMACPOOL_IMAGE=$(KUBEMACPOOL_IMAGE) \
 	MACVTAP_CNI_IMAGE=$(MACVTAP_CNI_IMAGE) \
 	MULTUS_DYNAMIC_NETWORKS_CONTROLLER_IMAGE=$(MULTUS_DYNAMIC_NETWORKS_CONTROLLER_IMAGE) \
+	KUBE_SECONDARY_DNS_IMAGE=$(KUBE_SECONDARY_DNS_IMAGE) \
+	CORE_DNS_IMAGE=$(CORE_DNS_IMAGE) \
 	KUBE_RBAC_PROXY_IMAGE=$(KUBE_RBAC_PROXY_IMAGE) \
 		./hack/generate-manifests.sh
 

README.md

@@ -19,6 +19,7 @@ spec:
   kubeMacPool: {}
   ovs: {}
   macvtap: {}
+  kubeSecondaryDNS: {}
   imagePullPolicy: Always
 ```
 
@@ -159,6 +160,26 @@ A simple example on how to do so, the user must deploy a `ConfigMap`, such as in
 
 Currently, this configuration is not dynamic.
 
+## KubeSecondaryDNS
+
+[This controller](https://github.com/kubevirt/kubesecondarydns)
+allows to support FQDN for VMI's secondary networks.
+
+```yaml
+apiVersion: networkaddonsoperator.network.kubevirt.io/v1
+kind: NetworkAddonsConfig
+metadata:
+  name: cluster
+spec:
+  kubeSecondaryDNS:
+    DOMAIN: ""
+    NAME_SERVER_IP: ""
+```
+
+Additionally, container image used to deliver this plugin can be set using
+`KUBE_SECONDARY_DNS_IMAGE` environment variable in operator
+deployment manifest.
+
 ## Image Pull Policy
 
 Administrator can specify [image pull policy](https://kubernetes.io/docs/concepts/containers/images/)

automation/check-patch.e2e-kube-secondary-dns-functests.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -xeuE
+
+# This script should be able to execute kube secondary dns
+# functional tests against Kubernetes cluster with
+# CNAO built with latest changes, on any
+# environment with basic dependencies listed in
+# check-patch.packages installed and docker running.
+#
+# yum -y install automation/check-patch.packages
+# automation/check-patch.e2e-kube-secondary-dns-functests.sh
+
+teardown() {
+    cp $(find . -name "*junit*.xml") $ARTIFACTS || true
+    rm -rf "${TMP_COMPONENT_PATH}"
+    cd ${TMP_PROJECT_PATH}
+    make cluster-down
+}
+
+main() {
+    # Setup CNAO and artifacts temp directory
+    source automation/check-patch.setup.sh
+    cd ${TMP_PROJECT_PATH}
+
+    # Spin-up ephemeral cluster with latest CNAO
+    # this script also exports KUBECONFIG, and fetch $COMPONENT repository
+    COMPONENT="kube-secondary-dns" source automation/components-functests.setup.sh
+
+    trap teardown EXIT
+
+    cd ${TMP_COMPONENT_PATH}
+    make create-nodeport
+    echo "Run kube-secondary-dns functional tests"
+    make functest
+}
+
+[[ "${BASH_SOURCE[0]}" == "$0" ]] && main "$@"

automation/components-functests.setup.sh

@@ -48,6 +48,7 @@ spec:
    rangeEnd: "02:00:00:00:00:0F"
   ovs: {}
   macvtap: {}
+  kubeSecondaryDNS: {}
   imagePullPolicy: Always
 EOF
 

components.yaml

@@ -41,3 +41,9 @@ components:
     branch: main
     update-policy: tagged
     metadata: v0.29.1
+  kube-secondary-dns:
+    url: https://github.com/kubevirt/kubesecondarydns
+    commit: a7779d99e0b196119f8bf9337186f091aea54df0
+    branch: main
+    update-policy: tagged
+    metadata: v0.0.5

data/kube-secondary-dns/secondarydns.yaml

@@ -0,0 +1,131 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: {{ .Namespace }}
+---
+apiVersion: v1
+data:
+  DOMAIN: {{ .Domain }}
+  NAME_SERVER_IP: {{ .NameServerIp }}
+  Corefile: |
+    .:53 {
+        auto {
+          directory /zones db\.(.*) {1}
+          reload 45s
+        }
+        reload
+        log
+    }
+kind: ConfigMap
+metadata:
+  name: secondary-dns
+  namespace: {{ .Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: secondary
+rules:
+- apiGroups:
+  - kubevirt.io
+  resources:
+  - virtualmachineinstances
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: secondary
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: secondary
+subjects:
+  - kind: ServiceAccount
+    name: secondary
+    namespace: {{ .Namespace }}
+---
+kind: ServiceAccount
+apiVersion: v1
+metadata:
+  name: secondary
+  namespace: {{ .Namespace }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    k8s-app: secondary-dns
+  name: secondary-dns
+  namespace: {{ .Namespace }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: secondary-dns
+  template:
+    metadata:
+      labels:
+        k8s-app: secondary-dns
+      annotations:
+        kubectl.kubernetes.io/default-container: status-monitor
+    spec:
+      serviceAccountName: secondary
+      containers:
+        - args:
+            - -conf
+            - /etc/coredns/Corefile
+          image: {{ .CoreDNSImage }}
+          imagePullPolicy: {{ .ImagePullPolicy }}
+          name: secondary-dns
+          ports:
+            - containerPort: 53
+              name: dns
+              protocol: UDP
+          resources:
+            limits:
+              memory: 170Mi
+            requests:
+              cpu: 100m
+              memory: 70Mi
+          volumeMounts:
+            - name: config-volume
+              mountPath: /etc/coredns
+              readOnly: true
+            - name: secdns-zones
+              mountPath: /zones
+              readOnly: true
+        - name: status-monitor
+          image: {{ .KubeSecondaryDNSImage }}
+          volumeMounts:
+            - name: secdns-zones
+              mountPath: /zones
+          env:
+            - name: DOMAIN
+              valueFrom:
+                configMapKeyRef:
+                  name: secondary-dns
+                  key: DOMAIN
+            - name: NAME_SERVER_IP
+              valueFrom:
+                configMapKeyRef:
+                  name: secondary-dns
+                  key: NAME_SERVER_IP
+          imagePullPolicy: {{ .ImagePullPolicy }}
+      priorityClassName: system-cluster-critical
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 1
+      volumes:
+        - name: config-volume
+          configMap:
+            defaultMode: 420
+            items:
+              - key: Corefile
+                path: Corefile
+            name: secondary-dns
+        - name: secdns-zones
+          emptyDir: {}

hack/components/bump-kube-secondary-dns.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+set -xeo pipefail
+
+source hack/components/yaml-utils.sh
+source hack/components/git-utils.sh
+source hack/components/docker-utils.sh
+
+function __parametize_by_object() {
+  for f in ./*; do
+    case "${f}" in
+      ./Namespace_secondary.yaml)
+        yaml-utils::update_param ${f} metadata.name '{{ .Namespace }}'
+        yaml-utils::remove_single_quotes_from_yaml ${f}
+        ;;
+      ./ConfigMap_secondary-dns.yaml)
+        yaml-utils::update_param ${f} metadata.namespace '{{ .Namespace }}'
+        yaml-utils::update_param ${f} data.DOMAIN '{{ .Domain }}'
+        yaml-utils::update_param ${f} data.NAME_SERVER_IP '{{ .NameServerIp }}'
+        yaml-utils::remove_single_quotes_from_yaml ${f}
+        ;;
+      ./ClusterRoleBinding_secondary.yaml)
+        yaml-utils::update_param ${f} subjects[0].namespace '{{ .Namespace }}'
+        yaml-utils::remove_single_quotes_from_yaml ${f}
+        ;;
+      ./Deployment_secondary-dns.yaml)
+        yaml-utils::update_param ${f} metadata.namespace '{{ .Namespace }}'
+        yaml-utils::update_param ${f} spec.template.spec.containers[0].image '{{ .CoreDNSImage }}'
+        yaml-utils::update_param ${f} spec.template.spec.containers[1].image '{{ .KubeSecondaryDNSImage }}'
+        yaml-utils::set_param ${f} spec.template.spec.containers[0].imagePullPolicy '{{ .ImagePullPolicy }}'
+        yaml-utils::set_param ${f} spec.template.spec.containers[1].imagePullPolicy '{{ .ImagePullPolicy }}'
+        yaml-utils::remove_single_quotes_from_yaml ${f}
+        ;;
+      ./ServiceAccount_secondary.yaml)
+        yaml-utils::update_param ${f} metadata.namespace '{{ .Namespace }}'
+        yaml-utils::remove_single_quotes_from_yaml ${f}
+        ;;
+    esac
+  done
+}
+
+echo 'Bumping kube-secondary-dns'
+KUBE_SECONDARY_DNS_URL=$(yaml-utils::get_component_url kube-secondary-dns)
+KUBE_SECONDARY_DNS_COMMIT=$(yaml-utils::get_component_commit kube-secondary-dns)
+KUBE_SECONDARY_DNS_REPO=$(yaml-utils::get_component_repo ${KUBE_SECONDARY_DNS_URL})
+
+TEMP_DIR=$(git-utils::create_temp_path kube-secondary-dns)
+trap "rm -rf ${TEMP_DIR}" EXIT
+KUBE_SECONDARY_DNS_PATH=${TEMP_DIR}/${KUBE_SECONDARY_DNS_REPO}
+
+echo 'Fetch kube-secondary-dns sources'
+git-utils::fetch_component ${KUBE_SECONDARY_DNS_PATH} ${KUBE_SECONDARY_DNS_URL} ${KUBE_SECONDARY_DNS_COMMIT}
+
+echo 'Adjust kube-secondary-dns to CNAO'
+(
+  cd ${KUBE_SECONDARY_DNS_PATH}
+  mkdir -p config/cnao
+  cp manifests/secondarydns.yaml config/cnao
+
+  echo 'Split manifest per object'
+  cd config/cnao
+  $(yaml-utils::split_yaml_by_seperator . secondarydns.yaml)
+
+  rm secondarydns.yaml
+  $(yaml-utils::rename_files_by_object .)
+
+  echo 'parametize manifests by object'
+  __parametize_by_object
+
+  echo 'rejoin sub-manifests to a final manifest'
+  cat Namespace_secondary.yaml \
+      ConfigMap_secondary-dns.yaml \
+      ClusterRole_secondary.yaml \
+      ClusterRoleBinding_secondary.yaml \
+      ServiceAccount_secondary.yaml \
+      Deployment_secondary-dns.yaml > secondarydns.yaml
+)
+
+echo 'copy manifests'
+rm -rf data/kube-secondary-dns/*
+cp ${KUBE_SECONDARY_DNS_PATH}/config/cnao/secondarydns.yaml data/kube-secondary-dns
+
+echo 'Get kube-secondary-dns image name and update it under CNAO'
+KUBE_SECONDARY_DNS_TAG=$(git-utils::get_component_tag ${KUBE_SECONDARY_DNS_PATH})
+KUBE_SECONDARY_DNS_IMAGE=ghcr.io/kubevirt/kubesecondarydns
+KUBE_SECONDARY_DNS_IMAGE_TAGGED=${KUBE_SECONDARY_DNS_IMAGE}:${KUBE_SECONDARY_DNS_TAG}
+KUBE_SECONDARY_DNS_IMAGE_DIGEST="$(docker-utils::get_image_digest "${KUBE_SECONDARY_DNS_IMAGE_TAGGED}" "${KUBE_SECONDARY_DNS_IMAGE}")"
+
+sed -i -r "s#\"${KUBE_SECONDARY_DNS_IMAGE}(@sha256)?:.*\"#\"${KUBE_SECONDARY_DNS_IMAGE_DIGEST}\"#" pkg/components/components.go
+sed -i -r "s#\"${KUBE_SECONDARY_DNS_IMAGE}(@sha256)?:.*\"#\"${KUBE_SECONDARY_DNS_IMAGE_DIGEST}\"#" test/releases/${CNAO_VERSION}.go

hack/generate-manifests.sh

@@ -26,6 +26,7 @@ for template in $templates; do
 		--container-tag=${CONTAINER_TAG} \
 		--image-pull-policy=${IMAGE_PULL_POLICY} \
 		--kube-rbac-proxy-image=${KUBE_RBAC_PROXY_IMAGE} \
+		--core-dns-image=${CORE_DNS_IMAGE} \
 		--input-file=${infile} \
 	)
 	if [[ ! -z "$rendered" ]]; then

pkg/apis/networkaddonsoperator/shared/networkaddonsconfig_types.go

@@ -16,6 +16,7 @@ type NetworkAddonsConfigSpec struct {
 	KubeMacPool            *KubeMacPool              `json:"kubeMacPool,omitempty"`
 	ImagePullPolicy        corev1.PullPolicy         `json:"imagePullPolicy,omitempty"`
 	NMState                *NMState                  `json:"nmstate,omitempty"`
+	KubeSecondaryDNS       *KubeSecondaryDNS         `json:"kubeSecondaryDNS,omitempty"`
 	MacvtapCni             *MacvtapCni               `json:"macvtap,omitempty"`
 	SelfSignConfiguration  *SelfSignConfiguration    `json:"selfSignConfiguration,omitempty"`
 	PlacementConfiguration *PlacementConfiguration   `json:"placementConfiguration,omitempty"`
@@ -63,6 +64,14 @@ type Ovs struct{}
 // NMState is a declarative node network configuration driven through Kubernetes API
 type NMState struct{}
 
+// KubeSecondaryDNS plugin allows to support FQDN for VMI's secondary networks
+type KubeSecondaryDNS struct {
+	// Domain defines the FQDN domain
+	Domain string `json:"domain,omitempty"`
+	// NameServerIp defines the name server IP
+	NameServerIP string `json:"nameServerIP,omitempty"`
+}
+
 // KubeMacPool plugin manages MAC allocation to Pods and VMs in Kubernetes
 type KubeMacPool struct {
 	// RangeStart defines the first mac in range