From 9b9945790bb0b62702dd52f0a3cd1a0d562cd515 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Fri, 19 Apr 2024 08:48:28 +0200
Subject: [PATCH 01/27] Use frr container for internet vrf

---
 deploy_partition.yaml                     |   2 -
 inventories/group_vars/cumulus/main.yaml  |   3 +-
 mini-lab.cumulus.yaml                     |  15 ++-
 roles/internet/files/frr.tpl              | 151 ----------------------
 roles/internet/tasks/main.yaml            |  35 -----
 roles/internet/templates/internet.intf.j2 |  20 ---
 6 files changed, 15 insertions(+), 211 deletions(-)
 delete mode 100644 roles/internet/files/frr.tpl
 delete mode 100644 roles/internet/tasks/main.yaml
 delete mode 100644 roles/internet/templates/internet.intf.j2

diff --git a/deploy_partition.yaml b/deploy_partition.yaml
index c885de9d..2dda5afb 100644
--- a/deploy_partition.yaml
+++ b/deploy_partition.yaml
@@ -20,8 +20,6 @@
   roles:
     - name: metal-roles/partition/roles/leaf
       tags: leaf
-    - name: internet
-      tags: internet
 
 - name: Deploy docker on Cumulus switches
   hosts: leaves:&cumulus
diff --git a/inventories/group_vars/cumulus/main.yaml b/inventories/group_vars/cumulus/main.yaml
index 15d599f0..bd9f059f 100644
--- a/inventories/group_vars/cumulus/main.yaml
+++ b/inventories/group_vars/cumulus/main.yaml
@@ -3,7 +3,8 @@ ports:
   1: 100G
 interfaces:
 - name: swp1
-uplinks: []
+uplinks:
+- name: swp31
 
 # The best practice recommendation is to set an MTU of 9,216 for the inter-switch links,
 # and an MTU of 9,000 for the server-facing ports, which don’t carry the VXLAN header.
diff --git a/mini-lab.cumulus.yaml b/mini-lab.cumulus.yaml
index a686ea3d..5791ee55 100644
--- a/mini-lab.cumulus.yaml
+++ b/mini-lab.cumulus.yaml
@@ -14,8 +14,6 @@ topology:
         ansible-group: cumulus
       binds:
         - files/ssh/id_rsa.pub:/root/.ssh/authorized_keys
-    linux:
-      image: ${MINI_LAB_VM_IMAGE}
 
   nodes:
     leaf01:
@@ -26,8 +24,19 @@ topology:
       kind: cvx
       binds:
         - apt-transport-https.tar.gz:/root/jessie-apt-transport-fix.tar.gz
+    inet:
+      kind: linux
+      image: quay.io/frrouting/frr:9.1.0
+      binds:
+        - files/inet/daemons:/etc/frr/daemons
+        - files/inet/frr.conf:/etc/frr/frr.conf
+        - files/inet/vtysh.conf:/etc/frr/vtysh.conf
+        - files/inet/network.sh:/root/network.sh
+      exec:
+        - sh /root/network.sh
     vms:
       kind: linux
+      image: ${MINI_LAB_VM_IMAGE}
       binds:
         - /dev:/dev
         - scripts:/mini-lab
@@ -39,3 +48,5 @@ topology:
     - endpoints: ["leaf02:swp2", "vms:lan3"]
     - endpoints: ["leaf01:swp3", "vms:lan4"]
     - endpoints: ["leaf02:swp3", "vms:lan5"]
+    - endpoints: ["leaf01:swp31", "inet:eth1"]
+    - endpoints: ["leaf02:swp31", "inet:eth2"]
diff --git a/roles/internet/files/frr.tpl b/roles/internet/files/frr.tpl
deleted file mode 100644
index 1dfef936..00000000
--- a/roles/internet/files/frr.tpl
+++ /dev/null
@@ -1,151 +0,0 @@
-{{- $ASN := .ASN -}}{{- $RouterId := .Loopback -}}! The frr version is not rendered since it seems to be optional.
-frr defaults datacenter
-hostname {{ .Name }}
-username cumulus nopassword
-service integrated-vtysh-config
-!
-log syslog {{ .LogLevel }}
-debug bgp updates
-debug bgp nht
-debug bgp update-groups
-debug bgp zebra
-!
-vrf vrfInternet
- vni 104009
- ip route 0.0.0.0/0 172.17.0.1 nexthop-vrf mgmt
-exit-vrf
-{{- range $vrf, $t := .Ports.Vrfs }}
-!
-vrf vrf{{ $t.VNI }}
- vni {{ $t.VNI }}
- exit-vrf
-{{- end }}
-{{- range .Ports.Underlay }}
-!
-interface {{ . }}
- ipv6 nd ra-interval 6
- no ipv6 nd suppress-ra
-{{- end }}
-{{- range .Ports.Firewalls }}
-!
-interface {{ .Port }}
- ipv6 nd ra-interval 6
- no ipv6 nd suppress-ra
-{{- end }}
-{{- range $vrf, $t := .Ports.Vrfs }}
-{{- range $t.Neighbors }}
-!
-interface {{ . }} vrf {{ $vrf }}
- ipv6 nd ra-interval 6
- no ipv6 nd suppress-ra
-{{- end }}
-{{- end }}
-!
-router bgp {{ $ASN }}
- bgp router-id {{ $RouterId }}
- bgp bestpath as-path multipath-relax
- neighbor FABRIC peer-group
- neighbor FABRIC remote-as external
- neighbor FABRIC timers 1 3
- {{- range .Ports.Underlay }}
- neighbor {{ . }} interface peer-group FABRIC
- {{- end }}
- neighbor FIREWALL peer-group
- neighbor FIREWALL remote-as external
- neighbor FIREWALL timers 1 3
- {{- range .Ports.Firewalls }}
- neighbor {{ .Port }} interface peer-group FIREWALL
- {{- end }}
- !
- address-family ipv4 unicast
-  redistribute connected route-map LOOPBACKS
-  neighbor FIREWALL allowas-in 1
-  {{- range $k, $f := .Ports.Firewalls }}
-  neighbor {{ $f.Port }} route-map fw-{{ $k }}-in in
-  {{- end }}
- exit-address-family
- !
- address-family l2vpn evpn
-  advertise-all-vni
-  neighbor FABRIC activate
-  neighbor FIREWALL activate
-  neighbor FIREWALL allowas-in 1
-  {{- range $k, $f := .Ports.Firewalls }}
-  neighbor {{ $f.Port }} route-map fw-{{ $k }}-vni out
-  {{- end }}
- exit-address-family
-!
-route-map LOOPBACKS permit 10
- match interface lo
-!
-{{- range $k, $f := .Ports.Firewalls }}
-# route-maps for firewall@{{ $k }}
-        {{- range $f.IPPrefixLists }}
-ip prefix-list {{ .Name }} {{ .Spec }}
-        {{- end}}
-        {{- range $f.RouteMaps }}
-route-map {{ .Name }} {{ .Policy }} {{ .Order }}
-                {{- range .Entries }}
- {{ . }}
-                {{- end }}
-        {{- end }}
-!
-{{- end }}
-ip route 0.0.0.0/0 {{ .Ports.Eth0.Gateway }} nexthop-vrf mgmt
-!
-{{- range $vrf, $t := .Ports.Vrfs }}
-router bgp {{ $ASN }} vrf {{ $vrf }}
- bgp router-id {{ $RouterId }}
- bgp bestpath as-path multipath-relax
- neighbor MACHINE peer-group
- neighbor MACHINE remote-as external
- neighbor MACHINE timers 1 3
- {{- range $t.Neighbors }}
- neighbor {{ . }} interface peer-group MACHINE
- {{- end }}
- !
- address-family ipv4 unicast
-  redistribute connected
-  neighbor MACHINE maximum-prefix 24000
-  {{- if gt (len $t.IPPrefixLists) 0 }}
-  neighbor MACHINE route-map {{ $vrf }}-in in
-  {{- end }}
- exit-address-family
- !
- address-family l2vpn evpn
-  advertise ipv4 unicast
- exit-address-family
-!
-{{- if gt (len $t.IPPrefixLists) 0 }}
-# route-maps for {{ $vrf }}
-        {{- range $t.IPPrefixLists }}
-ip prefix-list {{ .Name }} {{ .Spec }}
-        {{- end}}
-        {{- range $t.RouteMaps }}
-route-map {{ .Name }} {{ .Policy }} {{ .Order }}
-                {{- range .Entries }}
- {{ . }}
-                {{- end }}
-        {{- end }}
-!{{- end }}{{- end }}
-!
-router bgp {{ $ASN }} vrf vrfInternet
- bgp router-id {{ $RouterId }}
- bgp bestpath as-path multipath-relax
- !
- address-family ipv4 unicast
-  import vrf mgmt
-  network 0.0.0.0/0
- exit-address-family
- !
- address-family l2vpn evpn
-  advertise ipv4 unicast
- exit-address-family
-!
-vrf mgmt
- ip route 10.0.1.0/24 {{ .Loopback }} nexthop-vrf default
- ip route 100.255.254.0/24 vrfInternet nexthop-vrf vrfInternet
-exit-vrf
-!
-line vty
-!
\ No newline at end of file
diff --git a/roles/internet/tasks/main.yaml b/roles/internet/tasks/main.yaml
deleted file mode 100644
index 9503f15c..00000000
--- a/roles/internet/tasks/main.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
----
-- name: place customized frr template file for mini-lab
-  copy:
-    src: frr.tpl
-    dest: /etc/frr/frr.tpl
-
-# add additional interfaces vniInternet, vlanInternet and vrfInternet
-# this is done with a dedicated file that is included by /etc/network/interfaces
-# note: metal-core has no knowledge of this interfaces and would otherwise simply overwrite them if they are present in /etc/network/interfaces
-- name: add internet interfaces configuration to /etc/network/interfaces.d
-  template:
-    src: internet.intf.j2
-    dest: /etc/network/interfaces.d/internet.intf
-
-# add the vniInternet inferface to the bridge and configure the new vid at the bridge
-# otherwise ifreload will not work
-- name: add internet vni to bridge
-  command: net add bridge bridge ports vniInternet
-
-- name: add internet vlan id to bridge
-  command: net add bridge bridge vids 104009
-
-- name: commit
-  command: net commit
-
-- name: set envs for metal-core
-  set_fact:
-    metal_core_frr_tpl_file: "/etc/frr/frr.tpl"
-    metal_core_additional_bridge_ports:
-      - vniInternet
-    metal_core_additional_bridge_vids:
-      - 4009
-
-- name: add masquerade rule that skips virtual internet network
-  command: iptables -t nat -A POSTROUTING ! -s 100.255.254.0/24 -o eth0 -j MASQUERADE
diff --git a/roles/internet/templates/internet.intf.j2 b/roles/internet/templates/internet.intf.j2
deleted file mode 100644
index fd99ce96..00000000
--- a/roles/internet/templates/internet.intf.j2
+++ /dev/null
@@ -1,20 +0,0 @@
-auto vrfInternet
-iface vrfInternet
-    vrf-table auto
-
-auto vlanInternet
-iface vlanInternet
-    mtu 9000
-    vlan-id 4009
-    vlan-raw-device bridge
-    vrf vrfInternet
-
-auto vniInternet
-iface vniInternet
-    mtu 9000
-    bridge-access 4009
-    bridge-learning off
-    mstpctl-bpduguard yes
-    mstpctl-portbpdufilter yes
-    vxlan-id 104009
-    vxlan-local-tunnelip {{ lo }}

From 2f64ff965ee5a22e0dda5526206bd3127be760f2 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Fri, 19 Apr 2024 08:49:35 +0200
Subject: [PATCH 02/27] Allow ssh into firewall and machine

---
 Makefile         | 17 +++++++++++++++++
 files/ssh/config | 17 +++++++++++++++++
 2 files changed, 34 insertions(+)
 create mode 100644 files/ssh/config

diff --git a/Makefile b/Makefile
index 784ac0bf..fe0de2e2 100644
--- a/Makefile
+++ b/Makefile
@@ -26,12 +26,15 @@ SONIC_REMOTE_IMG := https://sonic-build.azurewebsites.net/api/sonic/artifacts?br
 ifeq ($(MINI_LAB_FLAVOR),default)
 LAB_MACHINES=machine01,machine02
 LAB_TOPOLOGY=mini-lab.cumulus.yaml
+VRF=vrf20
 else ifeq ($(MINI_LAB_FLAVOR),cluster-api)
 LAB_MACHINES=machine01,machine02,machine03
 LAB_TOPOLOGY=mini-lab.cumulus.yaml
+VRF=vrf20
 else ifeq ($(MINI_LAB_FLAVOR),sonic)
 LAB_MACHINES=machine01,machine02
 LAB_TOPOLOGY=mini-lab.sonic.yaml
+VRF=Vrf20
 else
 $(error Unknown flavor $(MINI_LAB_FLAVOR))
 endif
@@ -211,6 +214,20 @@ console-machine02:
 console-machine03:
 	@$(MAKE) --no-print-directory _console-machine	CONSOLE_PORT=4002
 
+.PHONY: ssh-fw
+ssh-fw:
+	$(eval fw = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp neighbors fw json' | \
+		python3 -c 'import sys, json; data = json.load(sys.stdin); key = next(iter(data)); print(data[key][\"bgpNeighborAddr\"] + \"%\" + key)'" \
+	))
+	ssh -F files/ssh/config $(fw) $(COMMAND)
+
+.PHONY: ssh-machine
+ssh-machine:
+	$(eval machine = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp vrf $(VRF) neighbors test json' | \
+		python3 -c 'import sys, json; data = json.load(sys.stdin); key = next(iter(data)); print(data[key][\"bgpNeighborAddr\"] + \"%\" + key)'" \
+	))
+	ssh -F files/ssh/config $(machine) $(COMMAND)
+
 ## DEV TARGETS ##
 
 .PHONY: dev-env
diff --git a/files/ssh/config b/files/ssh/config
new file mode 100644
index 00000000..d45f019a
--- /dev/null
+++ b/files/ssh/config
@@ -0,0 +1,17 @@
+Host leaf01
+    HostName leaf01
+    User root
+    IdentityFile files/ssh/id_rsa
+    PasswordAuthentication no
+    ProxyJump none
+    StrictHostKeyChecking no
+    UserKnownHostsFile /dev/null
+    PubkeyAcceptedKeyTypes +ssh-rsa
+
+Host * !leaf01
+    User metal
+    IdentityFile files/ssh/id_rsa
+    PasswordAuthentication no
+    StrictHostKeyChecking no
+    UserKnownHostsFile /dev/null
+    ProxyCommand ssh -q -F files/ssh/config leaf01 'sudo ip vrf exec default bash -c "exec 3<>/dev/tcp/%h/%p; cat<&0 >&3 & cat<&3 >&1"'

From 44beae2cd420de2e745963933d23b885147d2512 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Fri, 19 Apr 2024 08:50:21 +0200
Subject: [PATCH 03/27] Add pink test

---
 Makefile            | 20 ++++++++++++++++++++
 test/integration.sh |  3 +++
 2 files changed, 23 insertions(+)

diff --git a/Makefile b/Makefile
index fe0de2e2..d96d2b4c 100644
--- a/Makefile
+++ b/Makefile
@@ -18,6 +18,7 @@ MINI_LAB_FLAVOR := $(or $(MINI_LAB_FLAVOR),default)
 MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:latest)
 
 MACHINE_OS=ubuntu-22.04
+MAX_RETRIES := 10
 
 # Check: https://sonic-build.azurewebsites.net/ui/sonic/pipelines
 SONIC_REMOTE_IMG := https://sonic-build.azurewebsites.net/api/sonic/artifacts?branchName=202211&platform=vs&target=target%2Fsonic-vs.img.gz
@@ -228,6 +229,25 @@ ssh-machine:
 	))
 	ssh -F files/ssh/config $(machine) $(COMMAND)
 
+.PHONY: ping-cloudflare
+ping-cloudflare:
+	@echo "Attempting to ping 1.1.1.1..."
+	@for i in $$(seq 1 $(MAX_RETRIES)); do \
+		if $(MAKE) ssh-machine COMMAND="ping -c 1 1.1.1.1" > /dev/null 2>&1; then \
+			echo "Ping successful"; \
+			exit 0; \
+		else \
+			echo "Ping failed"; \
+			if [ $$i -lt $(MAX_RETRIES) ]; then \
+				echo "Retrying in 1 seconds..."; \
+				sleep 1; \
+			else \
+				echo "Max retries reached"; \
+				exit 1; \
+			fi; \
+		fi; \
+	done
+
 ## DEV TARGETS ##
 
 .PHONY: dev-env
diff --git a/test/integration.sh b/test/integration.sh
index ab08f030..bf683a8a 100755
--- a/test/integration.sh
+++ b/test/integration.sh
@@ -44,6 +44,9 @@ echo "$phoned/$minPhoned machines have phoned home"
 
 sleep 10
 
+echo "Test connectivity to outside"
+make ping-cloudflare
+
 echo "Adding route to leaf01 and leaf02"
 make route
 

From 98f84c9cd20b92696455b51c92df54a0fc236239 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Fri, 19 Apr 2024 10:23:52 +0200
Subject: [PATCH 04/27] Add comments

---
 Makefile         | 2 ++
 files/ssh/config | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index cbc7c476..c8f9a4c6 100644
--- a/Makefile
+++ b/Makefile
@@ -216,6 +216,8 @@ console-machine02:
 console-machine03:
 	@$(MAKE) --no-print-directory _console-machine	CONSOLE_PORT=4002
 
+## SSH TARGETS FOR MACHINES ##
+# Python code could be replaced by jq, but it is not preinstalled on Cumulus
 .PHONY: ssh-fw
 ssh-fw:
 	$(eval fw = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp neighbors fw json' | \
diff --git a/files/ssh/config b/files/ssh/config
index d45f019a..3bfcfc11 100644
--- a/files/ssh/config
+++ b/files/ssh/config
@@ -3,7 +3,6 @@ Host leaf01
     User root
     IdentityFile files/ssh/id_rsa
     PasswordAuthentication no
-    ProxyJump none
     StrictHostKeyChecking no
     UserKnownHostsFile /dev/null
     PubkeyAcceptedKeyTypes +ssh-rsa
@@ -14,4 +13,5 @@ Host * !leaf01
     PasswordAuthentication no
     StrictHostKeyChecking no
     UserKnownHostsFile /dev/null
+    # bash could be replaced with ncat, but it is not preinstalled on Cumulus
     ProxyCommand ssh -q -F files/ssh/config leaf01 'sudo ip vrf exec default bash -c "exec 3<>/dev/tcp/%h/%p; cat<&0 >&3 & cat<&3 >&1"'

From 8e94a0e1675ef2b04bd57b2c06c3a550008cb897 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Fri, 19 Apr 2024 10:35:23 +0200
Subject: [PATCH 05/27] Increase retry time

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index c8f9a4c6..a2259f2c 100644
--- a/Makefile
+++ b/Makefile
@@ -242,8 +242,8 @@ ping-cloudflare:
 		else \
 			echo "Ping failed"; \
 			if [ $$i -lt $(MAX_RETRIES) ]; then \
-				echo "Retrying in 1 seconds..."; \
-				sleep 1; \
+				echo "Retrying in 3 seconds..."; \
+				sleep 3; \
 			else \
 				echo "Max retries reached"; \
 				exit 1; \

From de5854d90615e0692edda6400bbc55bb16b7fb49 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Fri, 7 Jun 2024 13:49:00 +0200
Subject: [PATCH 06/27] Allow ssh over link local addresses into the machines

---
 Makefile            |  4 ++--
 compose.yaml        |  1 +
 files/ignition.json | 25 +++++++++++++++++++++++++
 3 files changed, 28 insertions(+), 2 deletions(-)
 create mode 100644 files/ignition.json

diff --git a/Makefile b/Makefile
index 4313a38a..75e05d36 100644
--- a/Makefile
+++ b/Makefile
@@ -136,11 +136,11 @@ _privatenet: env
 
 .PHONY: machine
 machine: _privatenet
-	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --networks $(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --userdata "@/tmp/ignition.json" --networks $(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: firewall
 firewall: _ips _privatenet
-	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --userdata "@/tmp/ignition.json" --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: ls
 ls: env
diff --git a/compose.yaml b/compose.yaml
index 02ca5dc5..3e700c21 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -79,6 +79,7 @@ services:
       - METALCTL_API_URL=http://api.172.17.0.1.nip.io:8080/metal
     volumes:
       - ./files/ssh:/root/.ssh:ro
+      - ./files/ignition.json:/tmp/ignition.json
     network_mode: host
     dns:
       - 172.17.0.1
diff --git a/files/ignition.json b/files/ignition.json
new file mode 100644
index 00000000..3c819a19
--- /dev/null
+++ b/files/ignition.json
@@ -0,0 +1,25 @@
+{
+  "ignition": {
+    "config": {},
+    "security": {},
+    "timeouts": {},
+    "version": "2.3.0"
+  },
+  "networkd": {},
+  "passwd": {},
+  "storage": {
+    "files": [
+      {
+        "filesystem": "root",
+        "path": "/etc/hosts.allow",
+        "append": true,
+        "contents": {
+          "source": "data:,ALL%3A%20%5Bfe80%3A%3A%5D%2F10%0D%0A",
+          "verification": {}
+        },
+        "mode": 644
+      }
+    ]
+  },
+  "systemd": {}
+}

From 259e9e4b3d0bd38abd2ec9e688bf4b26178a2f60 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <20912167+robertvolkmann@users.noreply.github.com>
Date: Fri, 12 Jul 2024 23:31:38 +0200
Subject: [PATCH 07/27] Try older firewall image

---
 inventories/group_vars/control-plane/metal.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/inventories/group_vars/control-plane/metal.yml b/inventories/group_vars/control-plane/metal.yml
index 6cf4cb36..21b6e813 100644
--- a/inventories/group_vars/control-plane/metal.yml
+++ b/inventories/group_vars/control-plane/metal.yml
@@ -16,8 +16,8 @@ metal_api_latest_os_image_release_name: "{{ (lookup('url', 'https://api.github.c
 metal_api_images:
 - id: firewall-ubuntu-3.0
   name: Firewall 3 Ubuntu
-  description: Firewall 3 Ubuntu Latest Release
-  url: https://images.metal-stack.io/metal-os/{{ metal_api_latest_os_image_release_name }}/firewall/3.0-ubuntu/img.tar.lz4
+  description: Firewall 3 Ubuntu 20230710
+  url: https://images.metal-stack.io/metal-os/20230710/firewall/3.0-ubuntu/img.tar.lz4
   features:
     - firewall
 - id: ubuntu-24.04

From de2f28261a68b8aa5fb0d908ccddc7718f868917 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Sat, 13 Jul 2024 00:17:12 +0200
Subject: [PATCH 08/27] sudo is required for ping

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 92d1c428..7f6a1832 100644
--- a/Makefile
+++ b/Makefile
@@ -19,7 +19,7 @@ MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:
 MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
 
 MACHINE_OS=ubuntu-24.04
-MAX_RETRIES := 10
+MAX_RETRIES := 20
 
 # Machine flavors
 ifeq ($(MINI_LAB_FLAVOR),cumulus)
@@ -235,7 +235,7 @@ ssh-machine:
 ping-cloudflare:
 	@echo "Attempting to ping 1.1.1.1..."
 	@for i in $$(seq 1 $(MAX_RETRIES)); do \
-		if $(MAKE) ssh-machine COMMAND="ping -c 1 1.1.1.1" > /dev/null 2>&1; then \
+		if $(MAKE) ssh-machine COMMAND="sudo ping -c 1 1.1.1.1" > /dev/null 2>&1; then \
 			echo "Ping successful"; \
 			exit 0; \
 		else \

From 9ee5a009a1310b0e28daad0cd470409170a587e1 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Sat, 13 Jul 2024 00:17:25 +0200
Subject: [PATCH 09/27] Use last working firewall image

---
 inventories/group_vars/control-plane/metal.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/inventories/group_vars/control-plane/metal.yml b/inventories/group_vars/control-plane/metal.yml
index 21b6e813..e0ca86c6 100644
--- a/inventories/group_vars/control-plane/metal.yml
+++ b/inventories/group_vars/control-plane/metal.yml
@@ -16,8 +16,8 @@ metal_api_latest_os_image_release_name: "{{ (lookup('url', 'https://api.github.c
 metal_api_images:
 - id: firewall-ubuntu-3.0
   name: Firewall 3 Ubuntu
-  description: Firewall 3 Ubuntu 20230710
-  url: https://images.metal-stack.io/metal-os/20230710/firewall/3.0-ubuntu/img.tar.lz4
+  description: Firewall 3 Ubuntu 20231206
+  url: https://images.metal-stack.io/metal-os/20231206/firewall/3.0-ubuntu/img.tar.lz4
   features:
     - firewall
 - id: ubuntu-24.04

From 2cd0967a96510da51f6afab31b091e76e272894b Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Mon, 15 Jul 2024 14:59:07 +0200
Subject: [PATCH 10/27] Define firewall rules

---
 Makefile         |  2 +-
 compose.yaml     |  1 +
 files/rules.yaml | 21 +++++++++++++++++++++
 3 files changed, 23 insertions(+), 1 deletion(-)
 create mode 100644 files/rules.yaml

diff --git a/Makefile b/Makefile
index 7f6a1832..3439460f 100644
--- a/Makefile
+++ b/Makefile
@@ -129,7 +129,7 @@ machine: _privatenet
 
 .PHONY: firewall
 firewall: _ips _privatenet
-	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --userdata "@/tmp/ignition.json" --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --userdata "@/tmp/ignition.json" --firewall-rules-file=/tmp/rules.yaml --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: ls
 ls: env
diff --git a/compose.yaml b/compose.yaml
index 3e700c21..7389d839 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -80,6 +80,7 @@ services:
     volumes:
       - ./files/ssh:/root/.ssh:ro
       - ./files/ignition.json:/tmp/ignition.json
+      - ./files/rules.yaml:/tmp/rules.yaml
     network_mode: host
     dns:
       - 172.17.0.1
diff --git a/files/rules.yaml b/files/rules.yaml
new file mode 100644
index 00000000..e3792004
--- /dev/null
+++ b/files/rules.yaml
@@ -0,0 +1,21 @@
+---
+egress:
+  - comment: allow outgoing https
+    ports:
+      - 443
+    protocol: TCP
+    to:
+      - 0.0.0.0/0
+  - comment: allow outgoing dns via tcp
+    ports:
+      - 53
+    protocol: TCP
+    to:
+      - 0.0.0.0/0
+  - comment: allow outgoing dns and ntp via udp
+    ports:
+      - 53
+      - 123
+    protocol: UDP
+    to:
+      - 0.0.0.0/0

From 61f22e37eadf7791d1f36fd4343962a7557917df Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Mon, 15 Jul 2024 14:59:33 +0200
Subject: [PATCH 11/27] Use curl instead of ping because CI env doesn't allow
 ICMP

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 3439460f..32b7e878 100644
--- a/Makefile
+++ b/Makefile
@@ -235,7 +235,7 @@ ssh-machine:
 ping-cloudflare:
 	@echo "Attempting to ping 1.1.1.1..."
 	@for i in $$(seq 1 $(MAX_RETRIES)); do \
-		if $(MAKE) ssh-machine COMMAND="sudo ping -c 1 1.1.1.1" > /dev/null 2>&1; then \
+		if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent https://1.1.1.1"; then \
 			echo "Ping successful"; \
 			exit 0; \
 		else \

From 00392407100021a58242bf5da2f6bbdf1c795c11 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Mon, 15 Jul 2024 14:59:41 +0200
Subject: [PATCH 12/27] Revert "Use last working firewall image"

This reverts commit 9ee5a009a1310b0e28daad0cd470409170a587e1.
---
 inventories/group_vars/control-plane/metal.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/inventories/group_vars/control-plane/metal.yml b/inventories/group_vars/control-plane/metal.yml
index e0ca86c6..21b6e813 100644
--- a/inventories/group_vars/control-plane/metal.yml
+++ b/inventories/group_vars/control-plane/metal.yml
@@ -16,8 +16,8 @@ metal_api_latest_os_image_release_name: "{{ (lookup('url', 'https://api.github.c
 metal_api_images:
 - id: firewall-ubuntu-3.0
   name: Firewall 3 Ubuntu
-  description: Firewall 3 Ubuntu 20231206
-  url: https://images.metal-stack.io/metal-os/20231206/firewall/3.0-ubuntu/img.tar.lz4
+  description: Firewall 3 Ubuntu 20230710
+  url: https://images.metal-stack.io/metal-os/20230710/firewall/3.0-ubuntu/img.tar.lz4
   features:
     - firewall
 - id: ubuntu-24.04

From da38b4317b16e37a731f6acd3da52838f91a5dd4 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Mon, 15 Jul 2024 14:59:41 +0200
Subject: [PATCH 13/27] Revert "Try older firewall image"

This reverts commit 259e9e4b3d0bd38abd2ec9e688bf4b26178a2f60.
---
 inventories/group_vars/control-plane/metal.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/inventories/group_vars/control-plane/metal.yml b/inventories/group_vars/control-plane/metal.yml
index 21b6e813..6cf4cb36 100644
--- a/inventories/group_vars/control-plane/metal.yml
+++ b/inventories/group_vars/control-plane/metal.yml
@@ -16,8 +16,8 @@ metal_api_latest_os_image_release_name: "{{ (lookup('url', 'https://api.github.c
 metal_api_images:
 - id: firewall-ubuntu-3.0
   name: Firewall 3 Ubuntu
-  description: Firewall 3 Ubuntu 20230710
-  url: https://images.metal-stack.io/metal-os/20230710/firewall/3.0-ubuntu/img.tar.lz4
+  description: Firewall 3 Ubuntu Latest Release
+  url: https://images.metal-stack.io/metal-os/{{ metal_api_latest_os_image_release_name }}/firewall/3.0-ubuntu/img.tar.lz4
   features:
     - firewall
 - id: ubuntu-24.04

From 4e08b9081bcfccdd4f712b7d52b0bfc0c6fd0153 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Mon, 15 Jul 2024 16:45:38 +0200
Subject: [PATCH 14/27] rename make target to test connectivity

---
 Makefile            | 12 ++++++------
 test/integration.sh |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/Makefile b/Makefile
index 32b7e878..78f3342e 100644
--- a/Makefile
+++ b/Makefile
@@ -231,15 +231,15 @@ ssh-machine:
 	))
 	ssh -F files/ssh/config $(machine) $(COMMAND)
 
-.PHONY: ping-cloudflare
-ping-cloudflare:
-	@echo "Attempting to ping 1.1.1.1..."
+.PHONY: connect-to-cloudflare
+connect-to-cloudflare:
+	@echo "Attempting to connect to 1.1.1.1..."
 	@for i in $$(seq 1 $(MAX_RETRIES)); do \
-		if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent https://1.1.1.1"; then \
-			echo "Ping successful"; \
+		if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent https://1.1.1.1" > /dev/null 2>&1; then \
+			echo "Connected successfully"; \
 			exit 0; \
 		else \
-			echo "Ping failed"; \
+			echo "Connection failed"; \
 			if [ $$i -lt $(MAX_RETRIES) ]; then \
 				echo "Retrying in 3 seconds..."; \
 				sleep 3; \
diff --git a/test/integration.sh b/test/integration.sh
index bf683a8a..971941aa 100755
--- a/test/integration.sh
+++ b/test/integration.sh
@@ -45,7 +45,7 @@ echo "$phoned/$minPhoned machines have phoned home"
 sleep 10
 
 echo "Test connectivity to outside"
-make ping-cloudflare
+make connect-to-cloudflare
 
 echo "Adding route to leaf01 and leaf02"
 make route

From bc53bf1e54458c514dd32d0c762f7e80d49ff936 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Mon, 15 Jul 2024 17:01:11 +0200
Subject: [PATCH 15/27] adjust comment

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 78f3342e..9a652af6 100644
--- a/Makefile
+++ b/Makefile
@@ -233,7 +233,7 @@ ssh-machine:
 
 .PHONY: connect-to-cloudflare
 connect-to-cloudflare:
-	@echo "Attempting to connect to 1.1.1.1..."
+	@echo "Attempting to connect to Cloudflare..."
 	@for i in $$(seq 1 $(MAX_RETRIES)); do \
 		if $(MAKE) ssh-machine COMMAND="sudo curl --connect-timeout 1 --fail --silent https://1.1.1.1" > /dev/null 2>&1; then \
 			echo "Connected successfully"; \

From 20775b3c47ba29cb14aeadf6bdea8a75bcff32a0 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Tue, 16 Jul 2024 14:53:28 +0200
Subject: [PATCH 16/27] Remove Internet VRF leftovers on the cumulus switches

---
 roles/cumulus/files/frr.tpl              | 151 -----------------------
 roles/cumulus/tasks/internet.yaml        |  35 ------
 roles/cumulus/tasks/main.yaml            |   2 -
 roles/cumulus/templates/internet.intf.j2 |  20 ---
 4 files changed, 208 deletions(-)
 delete mode 100644 roles/cumulus/files/frr.tpl
 delete mode 100644 roles/cumulus/tasks/internet.yaml
 delete mode 100644 roles/cumulus/templates/internet.intf.j2

diff --git a/roles/cumulus/files/frr.tpl b/roles/cumulus/files/frr.tpl
deleted file mode 100644
index 1dfef936..00000000
--- a/roles/cumulus/files/frr.tpl
+++ /dev/null
@@ -1,151 +0,0 @@
-{{- $ASN := .ASN -}}{{- $RouterId := .Loopback -}}! The frr version is not rendered since it seems to be optional.
-frr defaults datacenter
-hostname {{ .Name }}
-username cumulus nopassword
-service integrated-vtysh-config
-!
-log syslog {{ .LogLevel }}
-debug bgp updates
-debug bgp nht
-debug bgp update-groups
-debug bgp zebra
-!
-vrf vrfInternet
- vni 104009
- ip route 0.0.0.0/0 172.17.0.1 nexthop-vrf mgmt
-exit-vrf
-{{- range $vrf, $t := .Ports.Vrfs }}
-!
-vrf vrf{{ $t.VNI }}
- vni {{ $t.VNI }}
- exit-vrf
-{{- end }}
-{{- range .Ports.Underlay }}
-!
-interface {{ . }}
- ipv6 nd ra-interval 6
- no ipv6 nd suppress-ra
-{{- end }}
-{{- range .Ports.Firewalls }}
-!
-interface {{ .Port }}
- ipv6 nd ra-interval 6
- no ipv6 nd suppress-ra
-{{- end }}
-{{- range $vrf, $t := .Ports.Vrfs }}
-{{- range $t.Neighbors }}
-!
-interface {{ . }} vrf {{ $vrf }}
- ipv6 nd ra-interval 6
- no ipv6 nd suppress-ra
-{{- end }}
-{{- end }}
-!
-router bgp {{ $ASN }}
- bgp router-id {{ $RouterId }}
- bgp bestpath as-path multipath-relax
- neighbor FABRIC peer-group
- neighbor FABRIC remote-as external
- neighbor FABRIC timers 1 3
- {{- range .Ports.Underlay }}
- neighbor {{ . }} interface peer-group FABRIC
- {{- end }}
- neighbor FIREWALL peer-group
- neighbor FIREWALL remote-as external
- neighbor FIREWALL timers 1 3
- {{- range .Ports.Firewalls }}
- neighbor {{ .Port }} interface peer-group FIREWALL
- {{- end }}
- !
- address-family ipv4 unicast
-  redistribute connected route-map LOOPBACKS
-  neighbor FIREWALL allowas-in 1
-  {{- range $k, $f := .Ports.Firewalls }}
-  neighbor {{ $f.Port }} route-map fw-{{ $k }}-in in
-  {{- end }}
- exit-address-family
- !
- address-family l2vpn evpn
-  advertise-all-vni
-  neighbor FABRIC activate
-  neighbor FIREWALL activate
-  neighbor FIREWALL allowas-in 1
-  {{- range $k, $f := .Ports.Firewalls }}
-  neighbor {{ $f.Port }} route-map fw-{{ $k }}-vni out
-  {{- end }}
- exit-address-family
-!
-route-map LOOPBACKS permit 10
- match interface lo
-!
-{{- range $k, $f := .Ports.Firewalls }}
-# route-maps for firewall@{{ $k }}
-        {{- range $f.IPPrefixLists }}
-ip prefix-list {{ .Name }} {{ .Spec }}
-        {{- end}}
-        {{- range $f.RouteMaps }}
-route-map {{ .Name }} {{ .Policy }} {{ .Order }}
-                {{- range .Entries }}
- {{ . }}
-                {{- end }}
-        {{- end }}
-!
-{{- end }}
-ip route 0.0.0.0/0 {{ .Ports.Eth0.Gateway }} nexthop-vrf mgmt
-!
-{{- range $vrf, $t := .Ports.Vrfs }}
-router bgp {{ $ASN }} vrf {{ $vrf }}
- bgp router-id {{ $RouterId }}
- bgp bestpath as-path multipath-relax
- neighbor MACHINE peer-group
- neighbor MACHINE remote-as external
- neighbor MACHINE timers 1 3
- {{- range $t.Neighbors }}
- neighbor {{ . }} interface peer-group MACHINE
- {{- end }}
- !
- address-family ipv4 unicast
-  redistribute connected
-  neighbor MACHINE maximum-prefix 24000
-  {{- if gt (len $t.IPPrefixLists) 0 }}
-  neighbor MACHINE route-map {{ $vrf }}-in in
-  {{- end }}
- exit-address-family
- !
- address-family l2vpn evpn
-  advertise ipv4 unicast
- exit-address-family
-!
-{{- if gt (len $t.IPPrefixLists) 0 }}
-# route-maps for {{ $vrf }}
-        {{- range $t.IPPrefixLists }}
-ip prefix-list {{ .Name }} {{ .Spec }}
-        {{- end}}
-        {{- range $t.RouteMaps }}
-route-map {{ .Name }} {{ .Policy }} {{ .Order }}
-                {{- range .Entries }}
- {{ . }}
-                {{- end }}
-        {{- end }}
-!{{- end }}{{- end }}
-!
-router bgp {{ $ASN }} vrf vrfInternet
- bgp router-id {{ $RouterId }}
- bgp bestpath as-path multipath-relax
- !
- address-family ipv4 unicast
-  import vrf mgmt
-  network 0.0.0.0/0
- exit-address-family
- !
- address-family l2vpn evpn
-  advertise ipv4 unicast
- exit-address-family
-!
-vrf mgmt
- ip route 10.0.1.0/24 {{ .Loopback }} nexthop-vrf default
- ip route 100.255.254.0/24 vrfInternet nexthop-vrf vrfInternet
-exit-vrf
-!
-line vty
-!
\ No newline at end of file
diff --git a/roles/cumulus/tasks/internet.yaml b/roles/cumulus/tasks/internet.yaml
deleted file mode 100644
index 0cab4702..00000000
--- a/roles/cumulus/tasks/internet.yaml
+++ /dev/null
@@ -1,35 +0,0 @@
----
-- name: place customized frr template file for mini-lab
-  copy:
-    src: frr.tpl
-    dest: /etc/frr/frr.tpl
-
-# add additional interfaces vniInternet, vlanInternet and vrfInternet
-# this is done with a dedicated file that is included by /etc/network/interfaces
-# note: metal-core has no knowledge of this interfaces and would otherwise simply overwrite them if they are present in /etc/network/interfaces
-- name: add internet interfaces configuration to /etc/network/interfaces.d
-  template:
-    src: internet.intf.j2
-    dest: /etc/network/interfaces.d/internet.intf
-
-# add the vniInternet interface to the bridge and configure the new vid at the bridge
-# otherwise ifreload will not work
-- name: add internet vni to bridge
-  command: net add bridge bridge ports vniInternet
-
-- name: add internet vlan id to bridge
-  command: net add bridge bridge vids 104009
-
-- name: commit
-  command: net commit
-
-- name: set envs for metal-core
-  set_fact:
-    metal_core_frr_tpl_file: "/etc/frr/frr.tpl"
-    metal_core_additional_bridge_ports:
-      - vniInternet
-    metal_core_additional_bridge_vids:
-      - 4009
-
-- name: add masquerade rule that skips virtual internet network
-  command: iptables -t nat -A POSTROUTING ! -s 100.255.254.0/24 -o eth0 -j MASQUERADE
diff --git a/roles/cumulus/tasks/main.yaml b/roles/cumulus/tasks/main.yaml
index c75a9fd1..b7042195 100644
--- a/roles/cumulus/tasks/main.yaml
+++ b/roles/cumulus/tasks/main.yaml
@@ -27,5 +27,3 @@
     src: bridgemac.json
     dest: /etc/network/ifupdown2/policy.d/bridgemac.json
   notify: reload interfaces
-
-- import_tasks: internet.yaml
diff --git a/roles/cumulus/templates/internet.intf.j2 b/roles/cumulus/templates/internet.intf.j2
deleted file mode 100644
index fd99ce96..00000000
--- a/roles/cumulus/templates/internet.intf.j2
+++ /dev/null
@@ -1,20 +0,0 @@
-auto vrfInternet
-iface vrfInternet
-    vrf-table auto
-
-auto vlanInternet
-iface vlanInternet
-    mtu 9000
-    vlan-id 4009
-    vlan-raw-device bridge
-    vrf vrfInternet
-
-auto vniInternet
-iface vniInternet
-    mtu 9000
-    bridge-access 4009
-    bridge-learning off
-    mstpctl-bpduguard yes
-    mstpctl-portbpdufilter yes
-    vxlan-id 104009
-    vxlan-local-tunnelip {{ lo }}

From 06fba334575b951a146395d922f117ee5e8eedad Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Tue, 16 Jul 2024 18:09:41 +0200
Subject: [PATCH 17/27] Use linux bridge to simulate external network

203.0.113.128/24 is a private network
---
 Makefile                                      | 35 ++++++++++---------
 README.md                                     | 12 +------
 files/ignition.json                           |  2 +-
 files/inet/frr.conf                           |  9 +++--
 files/inet/network.sh                         |  4 +--
 .../group_vars/control-plane/metal.yml        |  2 +-
 mini-lab.cumulus.yaml                         |  5 ++-
 mini-lab.sonic.yaml                           |  3 ++
 test/ci-cleanup.sh                            |  2 --
 test/integration.sh                           | 10 ++----
 10 files changed, 39 insertions(+), 45 deletions(-)

diff --git a/Makefile b/Makefile
index 9a652af6..33a47e79 100644
--- a/Makefile
+++ b/Makefile
@@ -81,7 +81,7 @@ partition: partition-bake
 	docker compose up --remove-orphans --force-recreate partition
 
 .PHONY: partition-bake
-partition-bake:
+partition-bake: external_network
 	docker pull $(MINI_LAB_VM_IMAGE)
 ifeq ($(MINI_LAB_FLAVOR),sonic)
 	docker pull $(MINI_LAB_SONIC_IMAGE)
@@ -90,20 +90,22 @@ endif
 		sudo --preserve-env $(CONTAINERLAB) deploy --topo $(LAB_TOPOLOGY) --reconfigure && \
 		./scripts/deactivate_offloading.sh; fi
 
+.PHONY: external_network
+external_network:
+	@if ! docker network ls | grep -q mini_lab_ext; then \
+  		docker network create mini_lab_ext \
+			--driver=bridge \
+			--gateway=203.0.113.1 \
+			--subnet=203.0.113.0/24 \
+			--opt "com.docker.network.driver.mtu=1500" \
+			--opt "com.docker.network.bridge.name=mini_lab_ext" \
+			--opt "com.docker.network.bridge.enable_ip_masquerade=true" && \
+		sudo ip route add 203.0.113.128/25 via 203.0.113.2 dev mini_lab_ext; fi
+
 .PHONY: env
 env:
 	@./env.sh
 
-.PHONY: _ips
-_ips:
-	$(eval ipL1 = $(shell ${YQ} --unwrapScalar=true '.nodes.leaf01."mgmt-ipv4-address"' clab-mini-lab/topology-data.json))
-	$(eval ipL2 = $(shell ${YQ} --unwrapScalar=true '.nodes.leaf02."mgmt-ipv4-address"' clab-mini-lab/topology-data.json))
-	$(eval staticR = "100.255.254.0/24 nexthop via $(ipL1) dev docker0 nexthop via $(ipL2) dev docker0")
-
-.PHONY: route
-route: _ips
-	eval "sudo ip r a ${staticR}"
-
 .PHONY: cleanup
 cleanup: cleanup-control-plane cleanup-partition
 
@@ -116,8 +118,9 @@ cleanup-control-plane:
 .PHONY: cleanup-partition
 cleanup-partition:
 	mkdir -p clab-mini-lab
-	sudo $(CONTAINERLAB) destroy --topo mini-lab.cumulus.yaml
-	sudo $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml
+	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.cumulus.yaml
+	sudo --preserve-env $(CONTAINERLAB) destroy --topo mini-lab.sonic.yaml
+	docker network rm --force mini_lab_ext
 
 .PHONY: _privatenet
 _privatenet: env
@@ -128,7 +131,7 @@ machine: _privatenet
 	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --userdata "@/tmp/ignition.json" --networks $(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: firewall
-firewall: _ips _privatenet
+firewall: _privatenet
 	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --userdata "@/tmp/ignition.json" --firewall-rules-file=/tmp/rules.yaml --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: ls
@@ -217,8 +220,8 @@ console-machine03:
 
 ## SSH TARGETS FOR MACHINES ##
 # Python code could be replaced by jq, but it is not preinstalled on Cumulus
-.PHONY: ssh-fw
-ssh-fw:
+.PHONY: ssh-firewall
+ssh-firewall:
 	$(eval fw = $(shell ssh -F files/ssh/config leaf01 "vtysh -c 'show bgp neighbors fw json' | \
 		python3 -c 'import sys, json; data = json.load(sys.stdin); key = next(iter(data)); print(data[key][\"bgpNeighborAddr\"] + \"%\" + key)'" \
 	))
diff --git a/README.md b/README.md
index cb0a08c1..ee704784 100644
--- a/README.md
+++ b/README.md
@@ -24,7 +24,7 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy
 - [docker](https://www.docker.com/) >= 24.x.y (for using kind and our deployment base image)
 - [kind](https://github.com/kubernetes-sigs/kind/releases) == v0.23.0 (for hosting the metal control plane)
 - [containerlab](https://containerlab.dev/install/) >= v0.55.0
-- the lab creates a docker network on your host machine (`172.17.0.1`), this hopefully does not overlap with other networks you have
+- the lab creates a docker network on your host machine (`203.0.113.1`), this hopefully does not overlap with other networks you have
 - (recommended) haveged to have enough random entropy (only needed if the PXE process does not work)
 
 Here is some code that should help you to set up most of the requirements:
@@ -167,16 +167,6 @@ Login with user name metal and the console password from
 docker compose run --rm metalctl machine consolepassword e0ab02d2-27cd-5a5e-8efc-080ba80cf258
 ```
 
-If you want to access the firewall with SSH or have internet connectivity from the firewall and machine, you'll need to have a static route configured that points to the leaf switches:
-
-```bash
-# Add the route to the network internet-mini-lab 100.255.254.0/24 via leaf01 and leaf02, whose IPs are dynamically allocated. Make sure there's no old route before execution.
-make route
-
-# Connect to the firewall
-ssh metal@100.255.254.1
-```
-
 To remove the kind cluster, the switches and machines, run:
 
 ```bash
diff --git a/files/ignition.json b/files/ignition.json
index 3c819a19..7c2eefe3 100644
--- a/files/ignition.json
+++ b/files/ignition.json
@@ -14,7 +14,7 @@
         "path": "/etc/hosts.allow",
         "append": true,
         "contents": {
-          "source": "data:,ALL%3A%20%5Bfe80%3A%3A%5D%2F10%0D%0A",
+          "source": "data:,ALL%3A%20%5Bfe80%3A%3A%5D%2F10%0D%0AALL%3A%20203.0.113.1%0D%0A",
           "verification": {}
         },
         "mode": 644
diff --git a/files/inet/frr.conf b/files/inet/frr.conf
index 0e32b873..813941d2 100644
--- a/files/inet/frr.conf
+++ b/files/inet/frr.conf
@@ -5,17 +5,20 @@ log syslog informational
 !
 vrf vrfInternet
  vni 104009
- ip route 0.0.0.0/0 172.17.0.1
+ ip route 0.0.0.0/0 203.0.113.1
 exit-vrf
 !
-interface eth2
+interface eth1
  ipv6 nd ra-interval 6
  no ipv6 nd suppress-ra
 !
-interface eth3
+interface eth2
  ipv6 nd ra-interval 6
  no ipv6 nd suppress-ra
 !
+interface ext
+ ip address 203.0.113.2/24
+!
 interface lo
  ip address 10.0.0.21/32
 !
diff --git a/files/inet/network.sh b/files/inet/network.sh
index 5b0313c2..72db1e85 100644
--- a/files/inet/network.sh
+++ b/files/inet/network.sh
@@ -3,7 +3,7 @@ set -o errexit -o xtrace
 
 ip link add vrfInternet type vrf table 1000
 ip link set dev vrfInternet up
-ip link set dev eth0 master vrfInternet
+ip link set dev ext master vrfInternet
 
 ip link add name bridge type bridge stp_state 0
 ip link set dev bridge type bridge vlan_filtering 1
@@ -25,5 +25,3 @@ bridge vlan del vid 1 untagged pvid dev vniInternet
 bridge vlan add vid 1000 dev vniInternet
 bridge vlan add vid 1000 untagged pvid dev vniInternet
 ip link set up dev vniInternet
-
-iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
diff --git a/inventories/group_vars/control-plane/metal.yml b/inventories/group_vars/control-plane/metal.yml
index 6cf4cb36..8a36552b 100644
--- a/inventories/group_vars/control-plane/metal.yml
+++ b/inventories/group_vars/control-plane/metal.yml
@@ -74,7 +74,7 @@ metal_api_networks:
   partitionid: "mini-lab"
   vrf: 104009
   prefixes:
-  - 100.255.254.0/24
+  - 203.0.113.128/25
   labels:
     network.metal-stack.io/default: ""
     network.metal-stack.io/default-external: ""
diff --git a/mini-lab.cumulus.yaml b/mini-lab.cumulus.yaml
index 5791ee55..b1bdb743 100644
--- a/mini-lab.cumulus.yaml
+++ b/mini-lab.cumulus.yaml
@@ -16,6 +16,8 @@ topology:
         - files/ssh/id_rsa.pub:/root/.ssh/authorized_keys
 
   nodes:
+    mini_lab_ext:
+      kind: bridge
     leaf01:
       kind: cvx
       binds:
@@ -26,7 +28,7 @@ topology:
         - apt-transport-https.tar.gz:/root/jessie-apt-transport-fix.tar.gz
     inet:
       kind: linux
-      image: quay.io/frrouting/frr:9.1.0
+      image: quay.io/frrouting/frr:10.0.1
       binds:
         - files/inet/daemons:/etc/frr/daemons
         - files/inet/frr.conf:/etc/frr/frr.conf
@@ -42,6 +44,7 @@ topology:
         - scripts:/mini-lab
 
   links:
+    - endpoints: ["inet:ext", "mini_lab_ext:inet"]
     - endpoints: ["leaf01:swp1", "vms:lan0"]
     - endpoints: ["leaf02:swp1", "vms:lan1"]
     - endpoints: ["leaf01:swp2", "vms:lan2"]
diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml
index 1e10b974..1a735070 100644
--- a/mini-lab.sonic.yaml
+++ b/mini-lab.sonic.yaml
@@ -6,6 +6,8 @@ mgmt:
 
 topology:
   nodes:
+    mini_lab_ext:
+      kind: bridge
     leaf01:
       kind: linux
       image: ${MINI_LAB_SONIC_IMAGE}
@@ -39,6 +41,7 @@ topology:
         - /dev:/dev
         - scripts:/mini-lab
   links:
+    - endpoints: ["inet:ext", "mini_lab_ext:inet"]
     - endpoints: ["leaf01:eth1", "inet:eth1"]
     - endpoints: ["leaf02:eth1", "inet:eth2"]
     - endpoints: ["leaf01:eth2", "vms:lan0"]
diff --git a/test/ci-cleanup.sh b/test/ci-cleanup.sh
index 7eb15589..a2160898 100755
--- a/test/ci-cleanup.sh
+++ b/test/ci-cleanup.sh
@@ -15,5 +15,3 @@ if [ ! -z "$running_containers" ]; then
 fi
 
 make cleanup
-
-sudo ip r d 100.255.254.0/24 || true
diff --git a/test/integration.sh b/test/integration.sh
index 971941aa..d2f4750d 100755
--- a/test/integration.sh
+++ b/test/integration.sh
@@ -44,14 +44,10 @@ echo "$phoned/$minPhoned machines have phoned home"
 
 sleep 10
 
+echo "Check if SSH login to firewall works"
+ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.129 -C exit
+
 echo "Test connectivity to outside"
 make connect-to-cloudflare
 
-echo "Adding route to leaf01 and leaf02"
-make route
-
-echo "Check if SSH login to firewall works"
-# FIXME: Again this is unstable in CI integration tests
-# ssh -o StrictHostKeyChecking=no metal@100.255.254.1 -C exit
-
 echo "Successfully started mini-lab"

From eb51cfc867344e36830d582141af4aa9215b5c54 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Tue, 16 Jul 2024 18:39:38 +0200
Subject: [PATCH 18/27] Test for internet connectivity before testing ssh
 access

---
 test/integration.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/integration.sh b/test/integration.sh
index d2f4750d..cb963b2e 100755
--- a/test/integration.sh
+++ b/test/integration.sh
@@ -44,10 +44,10 @@ echo "$phoned/$minPhoned machines have phoned home"
 
 sleep 10
 
-echo "Check if SSH login to firewall works"
-ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.129 -C exit
-
 echo "Test connectivity to outside"
 make connect-to-cloudflare
 
+echo "Check if SSH login to firewall works"
+ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.129 -C exit
+
 echo "Successfully started mini-lab"

From fc1b921a64c463631ee145c002e28e37c2a7d04f Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Tue, 16 Jul 2024 19:29:20 +0200
Subject: [PATCH 19/27] Check SSH login doesn't work on CI

---
 test/integration.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration.sh b/test/integration.sh
index cb963b2e..fd2ddd98 100755
--- a/test/integration.sh
+++ b/test/integration.sh
@@ -48,6 +48,6 @@ echo "Test connectivity to outside"
 make connect-to-cloudflare
 
 echo "Check if SSH login to firewall works"
-ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.129 -C exit
+#ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.129 -C exit
 
 echo "Successfully started mini-lab"

From 275517fc946e478081686233e924e189d3c1548a Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Wed, 17 Jul 2024 07:59:58 +0200
Subject: [PATCH 20/27] Add ingress rule to allow SSH access

---
 files/rules.yaml    | 9 +++++++++
 test/integration.sh | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/files/rules.yaml b/files/rules.yaml
index e3792004..2b20c744 100644
--- a/files/rules.yaml
+++ b/files/rules.yaml
@@ -19,3 +19,12 @@ egress:
     protocol: UDP
     to:
       - 0.0.0.0/0
+ingress:
+  - comment: allow incoming ssh
+    ports:
+      - 22
+    protocol: TCP
+    from:
+      - 203.0.113.1/25
+    to:
+      - 203.0.113.128/25
diff --git a/test/integration.sh b/test/integration.sh
index fd2ddd98..cb963b2e 100755
--- a/test/integration.sh
+++ b/test/integration.sh
@@ -48,6 +48,6 @@ echo "Test connectivity to outside"
 make connect-to-cloudflare
 
 echo "Check if SSH login to firewall works"
-#ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.129 -C exit
+ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.129 -C exit
 
 echo "Successfully started mini-lab"

From c4689839e1397f2545138f8fd715e03cfe32c074 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Wed, 17 Jul 2024 10:56:47 +0200
Subject: [PATCH 21/27] Revert "Add ingress rule to allow SSH access"

This reverts commit 275517fc946e478081686233e924e189d3c1548a.
---
 files/rules.yaml    | 9 ---------
 test/integration.sh | 2 +-
 2 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/files/rules.yaml b/files/rules.yaml
index 2b20c744..e3792004 100644
--- a/files/rules.yaml
+++ b/files/rules.yaml
@@ -19,12 +19,3 @@ egress:
     protocol: UDP
     to:
       - 0.0.0.0/0
-ingress:
-  - comment: allow incoming ssh
-    ports:
-      - 22
-    protocol: TCP
-    from:
-      - 203.0.113.1/25
-    to:
-      - 203.0.113.128/25
diff --git a/test/integration.sh b/test/integration.sh
index cb963b2e..fd2ddd98 100755
--- a/test/integration.sh
+++ b/test/integration.sh
@@ -48,6 +48,6 @@ echo "Test connectivity to outside"
 make connect-to-cloudflare
 
 echo "Check if SSH login to firewall works"
-ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.129 -C exit
+#ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.129 -C exit
 
 echo "Successfully started mini-lab"

From bd682dc7802c2d10ed9650f1c2904c320909c0f1 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Wed, 17 Jul 2024 10:57:42 +0200
Subject: [PATCH 22/27] Only test outgoing internet connectivity

---
 Makefile            | 6 +++---
 test/integration.sh | 5 -----
 2 files changed, 3 insertions(+), 8 deletions(-)

diff --git a/Makefile b/Makefile
index 33a47e79..434cfce2 100644
--- a/Makefile
+++ b/Makefile
@@ -19,7 +19,7 @@ MINI_LAB_VM_IMAGE := $(or $(MINI_LAB_VM_IMAGE),ghcr.io/metal-stack/mini-lab-vms:
 MINI_LAB_SONIC_IMAGE := $(or $(MINI_LAB_SONIC_IMAGE),ghcr.io/metal-stack/mini-lab-sonic:latest)
 
 MACHINE_OS=ubuntu-24.04
-MAX_RETRIES := 20
+MAX_RETRIES := 30
 
 # Machine flavors
 ifeq ($(MINI_LAB_FLAVOR),cumulus)
@@ -244,8 +244,8 @@ connect-to-cloudflare:
 		else \
 			echo "Connection failed"; \
 			if [ $$i -lt $(MAX_RETRIES) ]; then \
-				echo "Retrying in 3 seconds..."; \
-				sleep 3; \
+				echo "Retrying in 2 seconds..."; \
+				sleep 2; \
 			else \
 				echo "Max retries reached"; \
 				exit 1; \
diff --git a/test/integration.sh b/test/integration.sh
index fd2ddd98..eaa4a9b0 100755
--- a/test/integration.sh
+++ b/test/integration.sh
@@ -42,12 +42,7 @@ do
 done
 echo "$phoned/$minPhoned machines have phoned home"
 
-sleep 10
-
 echo "Test connectivity to outside"
 make connect-to-cloudflare
 
-echo "Check if SSH login to firewall works"
-#ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.129 -C exit
-
 echo "Successfully started mini-lab"

From b9c08e996bdb961f3ddf88a9169a6a1c741912d8 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Wed, 17 Jul 2024 13:01:41 +0200
Subject: [PATCH 23/27] Test SSH access to the machine

---
 Makefile            | 15 ++++++++++-----
 files/rules.yaml    |  9 +++++++++
 test/integration.sh |  3 +++
 3 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index 434cfce2..2c1293c3 100644
--- a/Makefile
+++ b/Makefile
@@ -124,15 +124,20 @@ cleanup-partition:
 
 .PHONY: _privatenet
 _privatenet: env
-	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network | grep user-private-network || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000000 --name user-private-network
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network | grep user-private-network || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network allocate --partition mini-lab --project 00000000-0000-0000-0000-000000000001 --name user-private-network
+
+.PHONY: _public_ips
+_public_ips: env
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip list --name firewall | grep firewall || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip create --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --ipaddress 203.0.113.129 --name firewall
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip list --name machine | grep machine || docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network ip create --network internet-mini-lab --project 00000000-0000-0000-0000-000000000001 --ipaddress 203.0.113.130 --name machine
 
 .PHONY: machine
-machine: _privatenet
-	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --userdata "@/tmp/ignition.json" --networks $(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
+machine: _privatenet _public_ips
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl machine create --description test --name test --hostname test --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image $(MACHINE_OS) --size v1-small-x86 --userdata "@/tmp/ignition.json" --ips 203.0.113.130 --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: firewall
-firewall: _privatenet
-	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000000 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --userdata "@/tmp/ignition.json" --firewall-rules-file=/tmp/rules.yaml --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
+firewall: _privatenet _public_ips
+	docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl firewall create --description fw --name fw --hostname fw --project 00000000-0000-0000-0000-000000000001 --partition mini-lab --image firewall-ubuntu-3.0 --size v1-small-x86 --userdata "@/tmp/ignition.json" --ips 203.0.113.129 --firewall-rules-file=/tmp/rules.yaml --networks internet-mini-lab,$(shell docker compose run $(DOCKER_COMPOSE_TTY_ARG) metalctl network list --name user-private-network -o template --template '{{ .id }}')
 
 .PHONY: ls
 ls: env
diff --git a/files/rules.yaml b/files/rules.yaml
index e3792004..2b20c744 100644
--- a/files/rules.yaml
+++ b/files/rules.yaml
@@ -19,3 +19,12 @@ egress:
     protocol: UDP
     to:
       - 0.0.0.0/0
+ingress:
+  - comment: allow incoming ssh
+    ports:
+      - 22
+    protocol: TCP
+    from:
+      - 203.0.113.1/25
+    to:
+      - 203.0.113.128/25
diff --git a/test/integration.sh b/test/integration.sh
index eaa4a9b0..bb6fae07 100755
--- a/test/integration.sh
+++ b/test/integration.sh
@@ -45,4 +45,7 @@ echo "$phoned/$minPhoned machines have phoned home"
 echo "Test connectivity to outside"
 make connect-to-cloudflare
 
+echo "Test connectivity from outside"
+ssh -o StrictHostKeyChecking=no -o "PubkeyAcceptedKeyTypes +ssh-rsa" -i files/ssh/id_rsa metal@203.0.113.130 -C exit
+
 echo "Successfully started mini-lab"

From 9e52625176b494f813c9ad56a937bc83e1ad561a Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Wed, 17 Jul 2024 13:05:20 +0200
Subject: [PATCH 24/27] Document that 203.0.113.0/24 is a reserved address
 block

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ee704784..0d194b07 100644
--- a/README.md
+++ b/README.md
@@ -24,7 +24,7 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy
 - [docker](https://www.docker.com/) >= 24.x.y (for using kind and our deployment base image)
 - [kind](https://github.com/kubernetes-sigs/kind/releases) == v0.23.0 (for hosting the metal control plane)
 - [containerlab](https://containerlab.dev/install/) >= v0.55.0
-- the lab creates a docker network on your host machine (`203.0.113.1`), this hopefully does not overlap with other networks you have
+- the lab creates a docker network on your host machine with the address block `203.0.113.0/24`, designated as TEST-NET-3 for documentation and examples.
 - (recommended) haveged to have enough random entropy (only needed if the PXE process does not work)
 
 Here is some code that should help you to set up most of the requirements:

From a3e4ea67c7f3591bf23b8adbdadc7e5d545c361c Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Thu, 18 Jul 2024 09:25:39 +0200
Subject: [PATCH 25/27] Use containerlab 0.56.0

---
 .github/workflows/integration.yaml | 2 +-
 README.md                          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index 80805ad5..b53a5201 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -33,7 +33,7 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         # we can remove desired version again after this fix was released: https://github.com/srl-labs/containerlab/pull/2000
-        DESIRED_VERSION: v0.52.0
+        DESIRED_VERSION: v0.56.0
 
     - name: Log in to the container registry
       uses: docker/login-action@v3
diff --git a/README.md b/README.md
index 0d194b07..c18c448e 100644
--- a/README.md
+++ b/README.md
@@ -23,7 +23,7 @@ The mini-lab is a small, virtual setup to locally run the metal-stack. It deploy
 - kvm as hypervisor for the VMs (you can check through the `kvm-ok` command)
 - [docker](https://www.docker.com/) >= 24.x.y (for using kind and our deployment base image)
 - [kind](https://github.com/kubernetes-sigs/kind/releases) == v0.23.0 (for hosting the metal control plane)
-- [containerlab](https://containerlab.dev/install/) >= v0.55.0
+- [containerlab](https://containerlab.dev/install/) >= v0.56.0
 - the lab creates a docker network on your host machine with the address block `203.0.113.0/24`, designated as TEST-NET-3 for documentation and examples.
 - (recommended) haveged to have enough random entropy (only needed if the PXE process does not work)
 

From 8a03d313af929e1ef9a6d9d4e8bff82275f4d5f8 Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Thu, 18 Jul 2024 09:55:05 +0200
Subject: [PATCH 26/27] Set MTU for interface on the mini_lab_ext bridge to fix
 MTU mismatches

---
 Makefile              | 2 +-
 mini-lab.cumulus.yaml | 1 +
 mini-lab.sonic.yaml   | 1 +
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 2c1293c3..8432bfa4 100644
--- a/Makefile
+++ b/Makefile
@@ -97,7 +97,7 @@ external_network:
 			--driver=bridge \
 			--gateway=203.0.113.1 \
 			--subnet=203.0.113.0/24 \
-			--opt "com.docker.network.driver.mtu=1500" \
+			--opt "com.docker.network.driver.mtu=9000" \
 			--opt "com.docker.network.bridge.name=mini_lab_ext" \
 			--opt "com.docker.network.bridge.enable_ip_masquerade=true" && \
 		sudo ip route add 203.0.113.128/25 via 203.0.113.2 dev mini_lab_ext; fi
diff --git a/mini-lab.cumulus.yaml b/mini-lab.cumulus.yaml
index b1bdb743..407fb2cb 100644
--- a/mini-lab.cumulus.yaml
+++ b/mini-lab.cumulus.yaml
@@ -45,6 +45,7 @@ topology:
 
   links:
     - endpoints: ["inet:ext", "mini_lab_ext:inet"]
+      mtu: 9000
     - endpoints: ["leaf01:swp1", "vms:lan0"]
     - endpoints: ["leaf02:swp1", "vms:lan1"]
     - endpoints: ["leaf01:swp2", "vms:lan2"]
diff --git a/mini-lab.sonic.yaml b/mini-lab.sonic.yaml
index 1a735070..337a13da 100644
--- a/mini-lab.sonic.yaml
+++ b/mini-lab.sonic.yaml
@@ -42,6 +42,7 @@ topology:
         - scripts:/mini-lab
   links:
     - endpoints: ["inet:ext", "mini_lab_ext:inet"]
+      mtu: 9000
     - endpoints: ["leaf01:eth1", "inet:eth1"]
     - endpoints: ["leaf02:eth1", "inet:eth2"]
     - endpoints: ["leaf01:eth2", "vms:lan0"]

From 6e3c7044ce1b7c729e6ef2ce2ca488ef2d0dfb4a Mon Sep 17 00:00:00 2001
From: Robert Volkmann <robert.volkmann@x-cellent.com>
Date: Thu, 18 Jul 2024 10:30:15 +0200
Subject: [PATCH 27/27] Fix MTU vniInternet to 9000

---
 files/inet/network.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/files/inet/network.sh b/files/inet/network.sh
index 72db1e85..418d74db 100644
--- a/files/inet/network.sh
+++ b/files/inet/network.sh
@@ -18,7 +18,7 @@ bridge vlan add vid 1000 dev bridge self
 ip link set dev vlanInternet up
 
 ip link add vniInternet type vxlan id 104009 dstport 4789 local 10.0.0.21 nolearning
-ip link set dev vlanInternet mtu 9000
+ip link set dev vniInternet mtu 9000
 ip link set dev vniInternet master bridge
 bridge vlan del vid 1 dev vniInternet
 bridge vlan del vid 1 untagged pvid dev vniInternet