From d4ea7b8c8da835a3f1dd134c3e1bd8ca33a8bdd8 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Tue, 19 Nov 2024 22:13:14 +0000 Subject: [PATCH 1/6] Allow local network to access HostOS metrics --- ic-os/components/hostos.bzl | 4 +- .../nftables.template} | 10 +++ .../nftables/hostos/setup-nftables.service | 18 +++++ .../nftables/hostos/setup-nftables.sh | 74 +++++++++++++++++++ ic-os/hostos/context/Dockerfile | 4 + 5 files changed, 109 insertions(+), 1 deletion(-) rename ic-os/components/networking/nftables/{nftables-hostos.conf => hostos/nftables.template} (97%) create mode 100644 ic-os/components/networking/nftables/hostos/setup-nftables.service create mode 100755 ic-os/components/networking/nftables/hostos/setup-nftables.sh diff --git a/ic-os/components/hostos.bzl b/ic-os/components/hostos.bzl index 88cb6c30274..9446f2aad36 100644 --- a/ic-os/components/hostos.bzl +++ b/ic-os/components/hostos.bzl @@ -79,7 +79,9 @@ component_files = { Label("networking/fallback.conf"): "/etc/systemd/resolved.conf.d/fallback.conf", Label("networking/resolv.conf"): "/etc/resolv.conf", Label("networking/network-tweaks.conf"): "/etc/sysctl.d/network-tweaks.conf", - Label("networking/nftables/nftables-hostos.conf"): "/etc/nftables.conf", + Label("networking/nftables/hostos/nftables.template"): "/opt/ic/share/nftables.template", + Label("networking/nftables/hostos/setup-nftables.service"): "/etc/systemd/system/setup-nftables.service", + Label("networking/nftables/hostos/setup-nftables.sh"): "/opt/ic/bin/setup-nftables.sh", Label("networking/hosts"): "/etc/hosts", # ssh diff --git a/ic-os/components/networking/nftables/nftables-hostos.conf b/ic-os/components/networking/nftables/hostos/nftables.template similarity index 97% rename from ic-os/components/networking/nftables/nftables-hostos.conf rename to ic-os/components/networking/nftables/hostos/nftables.template index 47d21cfc982..727ebf116f1 100644 --- a/ic-os/components/networking/nftables/nftables-hostos.conf +++ b/ic-os/components/networking/nftables/hostos/nftables.template @@ -175,6 +175,14 @@ table ip6 filter { accept } + set local_prefix { + type ipv6_addr + flags interval + elements = { + {{ ipv6_prefix }} + } + } + chain INPUT { type filter hook input priority filter; policy drop; iif "lo" accept @@ -192,6 +200,8 @@ table ip6 filter { ip6 saddr @dfinity_dcs ct state { new } tcp dport { 22, 9100, 19531, 19100 } accept ip6 saddr @telemetry_clients ct state { new } tcp dport { 9100, 19531, 19100 } accept ip6 saddr @node_providers ct state { new } tcp dport { 22, 9100, 19531 } accept + ip6 saddr @local_prefix ct state { new } tcp dport { 9100, 19531, 19100 } accept + ip6 saddr @local_prefix udp dport { 9100, 19531, 19100 } accept tcp dport { 42372 } goto metrics_proxy } diff --git a/ic-os/components/networking/nftables/hostos/setup-nftables.service b/ic-os/components/networking/nftables/hostos/setup-nftables.service new file mode 100644 index 00000000000..00237de5698 --- /dev/null +++ b/ic-os/components/networking/nftables/hostos/setup-nftables.service @@ -0,0 +1,18 @@ +[Unit] +Description=Generate Firewall Configuration +# We must wait for IC bootstrap to complete: It writes various +# state files and may also be needed to obtain network config. +After=bootstrap-ic-node.service +Wants=bootstrap-ic-node.service +# We must also wait for storage permission fixup to have finished. +After=setup-permissions.service +Wants=setup-permissions.service +Before=nftables.service +Wants=nftables.service + +[Service] +Type=oneshot +ExecStart=/opt/ic/bin/setup-nftables.sh -n /boot/config/config.ini -i /opt/ic/share/nftables.template -o /run/ic-node/nftables-ruleset/nftables.conf + +[Install] +WantedBy=multi-user.target diff --git a/ic-os/components/networking/nftables/hostos/setup-nftables.sh b/ic-os/components/networking/nftables/hostos/setup-nftables.sh new file mode 100755 index 00000000000..e7185d00e39 --- /dev/null +++ b/ic-os/components/networking/nftables/hostos/setup-nftables.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +# Substitute correct configuration parameters into nftables.conf. + +function usage() { + cat <"${OUT_FILE}" diff --git a/ic-os/hostos/context/Dockerfile b/ic-os/hostos/context/Dockerfile index bc83b3aaf61..ba79647419a 100644 --- a/ic-os/hostos/context/Dockerfile +++ b/ic-os/hostos/context/Dockerfile @@ -108,6 +108,10 @@ RUN systemctl disable \ # ------ HOSTOS WORK --------------------------------------------- +# Divert symbolic link for dynamically generated nftables +# ruleset. +RUN ln -sf /run/ic-node/nftables-ruleset/nftables.conf /etc/nftables.conf + ARG ROOT_PASSWORD= RUN \ if [ "${ROOT_PASSWORD}" != "" ]; then \ From b9f6f8395b933a148405cc8ddb72c6f6dd23eeba Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Tue, 19 Nov 2024 22:41:52 +0000 Subject: [PATCH 2/6] Allow local network to access GuestOS metrics --- ic-os/components/ic/generate-ic-config/generate-ic-config.sh | 4 ++++ ic-os/components/ic/generate-ic-config/ic.json5.template | 2 ++ 2 files changed, 6 insertions(+) diff --git a/ic-os/components/ic/generate-ic-config/generate-ic-config.sh b/ic-os/components/ic/generate-ic-config/generate-ic-config.sh index dc5aa205e65..380dd9899ca 100755 --- a/ic-os/components/ic/generate-ic-config/generate-ic-config.sh +++ b/ic-os/components/ic/generate-ic-config/generate-ic-config.sh @@ -228,12 +228,16 @@ QUERY_STATS_EPOCH_LENGTH="${query_stats_epoch_length:-600}" # TODO: If the Jaeger address is not specified the config file will contain Some(""). This needs to be fixed. JAEGER_ADDR="${jaeger_addr:-}" +# TODO: Should pass prefix directly +IPV6_PREFIX=$(echo "${IPV6_ADDRESS}" | sed -E -e 's/:/#/4' -e 's/#.*/::\/64/') + if [ "${IPV6_ADDRESS}" == "" ]; then echo "Cannot determine an IPv6 address, aborting" exit 1 fi sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \ + -e "s@{{ ipv6_prefix }}@${IPV6_PREFIX}@" \ -e "s@{{ ipv4_address }}@${IPV4_ADDRESS}@" \ -e "s@{{ ipv4_gateway }}@${IPV4_GATEWAY}@" \ -e "s@{{ domain }}@${DOMAIN}@" \ diff --git a/ic-os/components/ic/generate-ic-config/ic.json5.template b/ic-os/components/ic/generate-ic-config/ic.json5.template index e6a506b8683..445a9592464 100644 --- a/ic-os/components/ic/generate-ic-config/ic.json5.template +++ b/ic-os/components/ic/generate-ic-config/ic.json5.template @@ -256,6 +256,8 @@ table ip6 filter {\n\ icmpv6 type nd-router-advert accept\n\ icmpv6 type nd-neighbor-solicit accept\n\ icmpv6 type nd-neighbor-advert accept\n\ + ip6 saddr { {{ ipv6_prefix }} } ct state { new } tcp dport { 9100, 19531, 19100 } accept\n\ + ip6 saddr { {{ ipv6_prefix }} } udp dport { 9100, 19531, 19100 } accept\n\ ip6 saddr { hostos } ct state { new } tcp dport { 42372 } accept # Allow access from HostOS metrics-proxy so GuestOS metrics-proxy can proxy certain metrics to HostOS.\n\ <>\n\ <>\n\ From e83d925fb19d8078308fba4e98dd4cd988976525 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Wed, 20 Nov 2024 00:49:08 +0000 Subject: [PATCH 3/6] Fix golden test --- rs/orchestrator/testdata/nftables_assigned_replica.conf.golden | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rs/orchestrator/testdata/nftables_assigned_replica.conf.golden b/rs/orchestrator/testdata/nftables_assigned_replica.conf.golden index ca8ef8781e0..8539440f38e 100644 --- a/rs/orchestrator/testdata/nftables_assigned_replica.conf.golden +++ b/rs/orchestrator/testdata/nftables_assigned_replica.conf.golden @@ -83,6 +83,8 @@ table ip6 filter { icmpv6 type nd-router-advert accept icmpv6 type nd-neighbor-solicit accept icmpv6 type nd-neighbor-advert accept + ip6 saddr { {{ ipv6_prefix }} } ct state { new } tcp dport { 9100, 19531, 19100 } accept + ip6 saddr { {{ ipv6_prefix }} } udp dport { 9100, 19531, 19100 } accept ip6 saddr { hostos } ct state { new } tcp dport { 42372 } accept # Allow access from HostOS metrics-proxy so GuestOS metrics-proxy can proxy certain metrics to HostOS. ip6 saddr {::ffff:5.5.5.5} ct state { new } tcp dport {1005} accept # node_gwp4o-eaaaa-aaaaa-aaaap-2ai ip6 saddr {::ffff:3.3.3.3} ct state { new } tcp dport {1003} accept # subnet_yndj2-3ybaa-aaaaa-aaaap-yai From ef1210e5abd1e065f0a87287568fc711d1d1e526 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Wed, 20 Nov 2024 20:25:13 +0000 Subject: [PATCH 4/6] Adjust ports, remove UDP --- ic-os/components/ic/generate-ic-config/ic.json5.template | 3 +-- ic-os/components/networking/nftables/hostos/nftables.template | 1 - rs/orchestrator/testdata/nftables_assigned_replica.conf.golden | 3 +-- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/ic-os/components/ic/generate-ic-config/ic.json5.template b/ic-os/components/ic/generate-ic-config/ic.json5.template index 445a9592464..43b662488d4 100644 --- a/ic-os/components/ic/generate-ic-config/ic.json5.template +++ b/ic-os/components/ic/generate-ic-config/ic.json5.template @@ -256,8 +256,7 @@ table ip6 filter {\n\ icmpv6 type nd-router-advert accept\n\ icmpv6 type nd-neighbor-solicit accept\n\ icmpv6 type nd-neighbor-advert accept\n\ - ip6 saddr { {{ ipv6_prefix }} } ct state { new } tcp dport { 9100, 19531, 19100 } accept\n\ - ip6 saddr { {{ ipv6_prefix }} } udp dport { 9100, 19531, 19100 } accept\n\ + ip6 saddr { {{ ipv6_prefix }} } ct state { new } tcp dport { 7070, 9090, 9091, 9100, 19531, 19100 } accept\n\ ip6 saddr { hostos } ct state { new } tcp dport { 42372 } accept # Allow access from HostOS metrics-proxy so GuestOS metrics-proxy can proxy certain metrics to HostOS.\n\ <>\n\ <>\n\ diff --git a/ic-os/components/networking/nftables/hostos/nftables.template b/ic-os/components/networking/nftables/hostos/nftables.template index 727ebf116f1..04d7acb4798 100644 --- a/ic-os/components/networking/nftables/hostos/nftables.template +++ b/ic-os/components/networking/nftables/hostos/nftables.template @@ -201,7 +201,6 @@ table ip6 filter { ip6 saddr @telemetry_clients ct state { new } tcp dport { 9100, 19531, 19100 } accept ip6 saddr @node_providers ct state { new } tcp dport { 22, 9100, 19531 } accept ip6 saddr @local_prefix ct state { new } tcp dport { 9100, 19531, 19100 } accept - ip6 saddr @local_prefix udp dport { 9100, 19531, 19100 } accept tcp dport { 42372 } goto metrics_proxy } diff --git a/rs/orchestrator/testdata/nftables_assigned_replica.conf.golden b/rs/orchestrator/testdata/nftables_assigned_replica.conf.golden index 8539440f38e..f6ac2b457e2 100644 --- a/rs/orchestrator/testdata/nftables_assigned_replica.conf.golden +++ b/rs/orchestrator/testdata/nftables_assigned_replica.conf.golden @@ -83,8 +83,7 @@ table ip6 filter { icmpv6 type nd-router-advert accept icmpv6 type nd-neighbor-solicit accept icmpv6 type nd-neighbor-advert accept - ip6 saddr { {{ ipv6_prefix }} } ct state { new } tcp dport { 9100, 19531, 19100 } accept - ip6 saddr { {{ ipv6_prefix }} } udp dport { 9100, 19531, 19100 } accept + ip6 saddr { {{ ipv6_prefix }} } ct state { new } tcp dport { 7070, 9090, 9091, 9100, 19531, 19100 } accept ip6 saddr { hostos } ct state { new } tcp dport { 42372 } accept # Allow access from HostOS metrics-proxy so GuestOS metrics-proxy can proxy certain metrics to HostOS. ip6 saddr {::ffff:5.5.5.5} ct state { new } tcp dport {1005} accept # node_gwp4o-eaaaa-aaaaa-aaaap-2ai ip6 saddr {::ffff:3.3.3.3} ct state { new } tcp dport {1003} accept # subnet_yndj2-3ybaa-aaaaa-aaaap-yai From 46c1a3c2ed5b4f15239c4437b4593d02ec8c4f30 Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Wed, 20 Nov 2024 22:25:04 +0000 Subject: [PATCH 5/6] Make substitution more robust --- .../ic/generate-ic-config/generate-ic-config.sh | 5 ++++- .../networking/nftables/hostos/setup-nftables.sh | 8 ++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ic-os/components/ic/generate-ic-config/generate-ic-config.sh b/ic-os/components/ic/generate-ic-config/generate-ic-config.sh index 380dd9899ca..3798cabef20 100755 --- a/ic-os/components/ic/generate-ic-config/generate-ic-config.sh +++ b/ic-os/components/ic/generate-ic-config/generate-ic-config.sh @@ -229,7 +229,10 @@ QUERY_STATS_EPOCH_LENGTH="${query_stats_epoch_length:-600}" JAEGER_ADDR="${jaeger_addr:-}" # TODO: Should pass prefix directly -IPV6_PREFIX=$(echo "${IPV6_ADDRESS}" | sed -E -e 's/:/#/4' -e 's/#.*/::\/64/') +if ! IPV6_PREFIX=$(echo "${IPV6_ADDRESS}" | sed -E -e 's/:/#/4' -e '/#/!q1' -e 's/#.*/::\/64/'); then + # If address does not substitute correctly, fallback to loopback for easy templating + IPV6_PREFIX="::1/128" +fi if [ "${IPV6_ADDRESS}" == "" ]; then echo "Cannot determine an IPv6 address, aborting" diff --git a/ic-os/components/networking/nftables/hostos/setup-nftables.sh b/ic-os/components/networking/nftables/hostos/setup-nftables.sh index e7185d00e39..baf32d045cf 100755 --- a/ic-os/components/networking/nftables/hostos/setup-nftables.sh +++ b/ic-os/components/networking/nftables/hostos/setup-nftables.sh @@ -62,12 +62,8 @@ if [ "${NETWORK_CONFIG_FILE}" != "" -a -e "${NETWORK_CONFIG_FILE}" ]; then read_network_variables "${NETWORK_CONFIG_FILE}" fi -IPV6_PREFIX="${ipv6_prefix:-}::/64" - -if [ "${IPV6_PREFIX}" == "" ]; then - echo "Cannot determine an IPv6 prefix, aborting" - exit 1 -fi +IPV6_PREFIX="${ipv6_prefix:+${ipv6_prefix}::/64}" # Add suffix to prefix if found +IPV6_PREFIX="${IPV6_PREFIX:-::1/128}" # Default to loopback for easy templating mkdir -p /run/ic-node/nftables-ruleset/ sed -e "s@{{ ipv6_prefix }}@${IPV6_PREFIX}@" \ From 508b8c5bb1c3da6ec68ec23fd5d0830d96d8751f Mon Sep 17 00:00:00 2001 From: Eero Kelly Date: Thu, 21 Nov 2024 00:10:05 +0000 Subject: [PATCH 6/6] fmt --- ic-os/components/networking/nftables/hostos/setup-nftables.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ic-os/components/networking/nftables/hostos/setup-nftables.sh b/ic-os/components/networking/nftables/hostos/setup-nftables.sh index baf32d045cf..a35a46aa7ed 100755 --- a/ic-os/components/networking/nftables/hostos/setup-nftables.sh +++ b/ic-os/components/networking/nftables/hostos/setup-nftables.sh @@ -63,7 +63,7 @@ if [ "${NETWORK_CONFIG_FILE}" != "" -a -e "${NETWORK_CONFIG_FILE}" ]; then fi IPV6_PREFIX="${ipv6_prefix:+${ipv6_prefix}::/64}" # Add suffix to prefix if found -IPV6_PREFIX="${IPV6_PREFIX:-::1/128}" # Default to loopback for easy templating +IPV6_PREFIX="${IPV6_PREFIX:-::1/128}" # Default to loopback for easy templating mkdir -p /run/ic-node/nftables-ruleset/ sed -e "s@{{ ipv6_prefix }}@${IPV6_PREFIX}@" \