ci: Fix disconnected tests

examples/no_outbound.json

@@ -33,7 +33,8 @@
                 "count": 1,
                 "vmSize": "Standard_D2_v3",
                 "availabilityProfile": "AvailabilitySet",
-                "osType": "Windows"
+                "osType": "Windows",
+                "osDiskSizeGB": 50
             }
         ],
         "linuxProfile": {

test/e2e/kubernetes/kubernetes_test.go

@@ -769,7 +769,14 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
 		It("should be able to schedule a pod to a control plane node", func() {
 			By("Creating a Job with control plane nodeSelector")
 			for i := 1; i <= 3; i++ {
-				j, err := job.CreateJobFromFileWithRetry(filepath.Join(WorkloadDir, "busybox-master.yaml"), "busybox-master", "default", 3*time.Second, 3*time.Minute)
+				var jobFile string
+				if cfg.BlockOutboundInternet {
+					jobFile = "busybox-master-no-egress.yaml"
+				} else {
+					jobFile = "busybox-master.yaml"
+				}
+
+				j, err := job.CreateJobFromFileWithRetry(filepath.Join(WorkloadDir, jobFile), "busybox-master", "default", 3*time.Second, 3*time.Minute)
 				if err != nil {
 					fmt.Printf("unable to create job: %s\n", err)
 					continue
@@ -789,7 +796,14 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
 			if eng.AnyAgentIsLinux() {
 				By("Creating a Job with agent nodeSelector")
 				for i := 1; i <= 3; i++ {
-					j, err := job.CreateJobFromFileWithRetry(filepath.Join(WorkloadDir, "busybox-agent.yaml"), "busybox-agent", "default", 3*time.Second, 3*time.Minute)
+					var agentJobFile string
+					if cfg.BlockOutboundInternet {
+						agentJobFile = "busybox-agent-no-egress.yaml"
+					} else {
+						agentJobFile = "busybox-agent.yaml"
+					}
+
+					j, err := job.CreateJobFromFileWithRetry(filepath.Join(WorkloadDir, agentJobFile), "busybox-agent", "default", 3*time.Second, 3*time.Minute)
 					if err != nil {
 						fmt.Printf("unable to create job: %s\n", err)
 						continue
@@ -960,15 +974,13 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
 
 		It("should not have outbound access if specified", func() {
 			if cfg.BlockOutboundInternet {
-				nodes, err := node.GetReadyWithRetry(1*time.Second, cfg.Timeout)
-				Expect(err).NotTo(HaveOccurred())
-				outboundInternetCommand := fmt.Sprintf("nc -vz microsoft.com 80")
-				for _, n := range nodes {
-					if n.IsLinux() {
-						err = sshConn.ExecuteRemoteWithRetry(n.Metadata.Name, outboundInternetCommand, true, 1*time.Minute, timeoutWhenWaitingForPodOutboundAccess)
-						Expect(err).To(HaveOccurred())
-					}
+				j, err := job.CreateJobFromFileWithRetry(filepath.Join(WorkloadDir, "busybox-validate-no-egress.yaml"), "busybox-validate-no-egress", "default", 3*time.Second, 3*time.Minute)
+				if err != nil {
+					fmt.Printf("unable to create job: %s\n", err)
 				}
+				ready, err := j.WaitOnSucceeded(1*time.Minute, timeoutWhenWaitingForPodOutboundAccess)
+				Expect(err).To(HaveOccurred())
+				Expect(ready).To(Equal(false))
 			} else {
 				Skip("Outbound access is allowed")
 			}

test/e2e/kubernetes/workloads/busybox-agent-no-egress.yaml

@@ -0,0 +1,26 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: busybox-agent
+spec:
+  template:
+    spec:
+      initContainers:
+      - name: init-resolver
+        image: mcr.microsoft.com/oss/busybox/busybox:1.33.1
+        command: ['sh', '-c', 'until nc -nvz $(grep nameserver /etc/resolv.conf | cut -d" " -f2) 53; do echo waiting for container DNS resolver access; sleep 3; done;']
+      - name: init-dns
+        image: mcr.microsoft.com/oss/busybox/busybox:1.33.1
+        command: ['sh', '-c', 'until nslookup mcr.microsoft.com; do echo waiting for successful container DNS resolution; sleep 3; done;']
+      - name: init-kubernetes-net
+        image: mcr.microsoft.com/oss/busybox/busybox:1.33.1
+        command: ['sh', '-c', 'until nc -vz kubernetes 443 && nc -vz kubernetes.default.svc 443 && nc -vz kubernetes.default.svc.cluster.local 443; do echo waiting for successful connection to apiserver endpoint; sleep 3; done;']
+      containers:
+      - image: mcr.microsoft.com/oss/busybox/busybox:1.33.1
+        name: busybox-agent
+        command: ['sh', '-c', '[ $(echo "Hello, World!" | sha256sum | cut -d" " -f1) = "c98c24b677eff44860afea6f493bbaec5bb1c4cbb209c6fc2bbb47f66ff2ad31" ]']
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/role: agent
+        kubernetes.io/os: linux
+  backoffLimit: 0

test/e2e/kubernetes/workloads/busybox-master-no-egress.yaml

@@ -0,0 +1,39 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: busybox-master
+spec:
+  template:
+    spec:
+      initContainers:
+      - name: init-resolver
+        image: mcr.microsoft.com/oss/busybox/busybox:1.33.1
+        command: ['sh', '-c', 'until nc -nvz $(grep nameserver /etc/resolv.conf | cut -d" " -f2) 53; do echo waiting for container DNS resolver access; sleep 3; done;']
+      - name: init-kubernetes-net
+        image: mcr.microsoft.com/oss/busybox/busybox:1.33.1
+        command: ['sh', '-c', 'until nc -vz kubernetes 443 && nc -vz kubernetes.default.svc 443 && nc -vz kubernetes.default.svc.cluster.local 443; do echo waiting for successful connection to apiserver endpoint; sleep 3; done;']
+      - name: init-dns
+        image: mcr.microsoft.com/oss/busybox/busybox:1.33.1
+        command: ['sh', '-c', 'until nslookup mcr.microsoft.com; do echo waiting for successful container DNS resolution; sleep 3; done;']
+      containers:
+      - image: mcr.microsoft.com/oss/busybox/busybox:1.33.1
+        name: busybox-master
+        command: ['sh', '-c', '[ $(echo "Hello, World!" | sha256sum | cut -d" " -f1) = "c98c24b677eff44860afea6f493bbaec5bb1c4cbb209c6fc2bbb47f66ff2ad31" ]']
+      restartPolicy: Never
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/control-plane
+        operator: Equal
+        value: "true"
+        effect: NoSchedule
+      - operator: "Exists"
+        effect: NoExecute
+      - operator: "Exists"
+        effect: NoSchedule
+      nodeSelector:
+        kubernetes.io/role: master
+        kubernetes.io/os: linux
+  backoffLimit: 0

test/e2e/kubernetes/workloads/busybox-validate-no-egress.yaml

@@ -0,0 +1,14 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: busybox-validate-no-egress
+spec:
+  template:
+    spec:
+      containers:
+      - image: mcr.microsoft.com/oss/busybox/busybox:1.33.1
+        name: busybox-validate-no-egress
+        command: ['sh', '-c', 'nc -vz microsoft.com 80']
+      restartPolicy: Never
+      nodeSelector:
+        kubernetes.io/os: linux