feat: [NODE-1530] Allow local network to access metrics through firew…

…all (#2703)

ic-os/components/hostos.bzl

@@ -79,7 +79,9 @@ component_files = {
     Label("networking/fallback.conf"): "/etc/systemd/resolved.conf.d/fallback.conf",
     Label("networking/resolv.conf"): "/etc/resolv.conf",
     Label("networking/network-tweaks.conf"): "/etc/sysctl.d/network-tweaks.conf",
-    Label("networking/nftables/nftables-hostos.conf"): "/etc/nftables.conf",
+    Label("networking/nftables/hostos/nftables.template"): "/opt/ic/share/nftables.template",
+    Label("networking/nftables/hostos/setup-nftables.service"): "/etc/systemd/system/setup-nftables.service",
+    Label("networking/nftables/hostos/setup-nftables.sh"): "/opt/ic/bin/setup-nftables.sh",
     Label("networking/hosts"): "/etc/hosts",
 
     # ssh

ic-os/components/ic/generate-ic-config/generate-ic-config.sh

@@ -228,12 +228,19 @@ QUERY_STATS_EPOCH_LENGTH="${query_stats_epoch_length:-600}"
 # TODO: If the Jaeger address is not specified the config file will contain Some(""). This needs to be fixed.
 JAEGER_ADDR="${jaeger_addr:-}"
 
+# TODO: Should pass prefix directly
+if ! IPV6_PREFIX=$(echo "${IPV6_ADDRESS}" | sed -E -e 's/:/#/4' -e '/#/!q1' -e 's/#.*/::\/64/'); then
+    # If address does not substitute correctly, fallback to loopback for easy templating
+    IPV6_PREFIX="::1/128"
+fi
+
 if [ "${IPV6_ADDRESS}" == "" ]; then
     echo "Cannot determine an IPv6 address, aborting"
     exit 1
 fi
 
 sed -e "s@{{ ipv6_address }}@${IPV6_ADDRESS}@" \
+    -e "s@{{ ipv6_prefix }}@${IPV6_PREFIX}@" \
     -e "s@{{ ipv4_address }}@${IPV4_ADDRESS}@" \
     -e "s@{{ ipv4_gateway }}@${IPV4_GATEWAY}@" \
     -e "s@{{ domain }}@${DOMAIN}@" \

ic-os/components/ic/generate-ic-config/ic.json5.template

@@ -256,6 +256,7 @@ table ip6 filter {\n\
     icmpv6 type nd-router-advert accept\n\
     icmpv6 type nd-neighbor-solicit accept\n\
     icmpv6 type nd-neighbor-advert accept\n\
+    ip6 saddr { {{ ipv6_prefix }} } ct state { new } tcp dport { 7070, 9090, 9091, 9100, 19531, 19100 } accept\n\
     ip6 saddr { hostos } ct state { new } tcp dport { 42372 } accept # Allow access from HostOS metrics-proxy so GuestOS metrics-proxy can proxy certain metrics to HostOS.\n\
     <<IPv6_TCP_RULES>>\n\
     <<IPv6_UDP_RULES>>\n\

ic-os/components/networking/nftables/nftables-hostos.conf → ic-os/components/networking/nftables/hostos/nftables.template

@@ -175,6 +175,14 @@ table ip6 filter {
     accept
   }
 
+  set local_prefix {
+    type ipv6_addr
+    flags interval
+    elements =  {
+      {{ ipv6_prefix }}
+    }
+  }
+
   chain INPUT {
     type filter hook input priority filter; policy drop;
     iif "lo" accept
@@ -192,6 +200,7 @@ table ip6 filter {
     ip6 saddr @dfinity_dcs ct state { new } tcp dport { 22, 9100, 19531, 19100 } accept
     ip6 saddr @telemetry_clients ct state { new } tcp dport { 9100, 19531, 19100 } accept
     ip6 saddr @node_providers ct state { new } tcp dport { 22, 9100, 19531 } accept
+    ip6 saddr @local_prefix ct state { new } tcp dport { 9100, 19531, 19100 } accept
     tcp dport { 42372 } goto metrics_proxy
   }
 

ic-os/components/networking/nftables/hostos/setup-nftables.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Generate Firewall Configuration
+# We must wait for IC bootstrap to complete: It writes various
+# state files and may also be needed to obtain network config.
+After=bootstrap-ic-node.service
+Wants=bootstrap-ic-node.service
+# We must also wait for storage permission fixup to have finished.
+After=setup-permissions.service
+Wants=setup-permissions.service
+Before=nftables.service
+Wants=nftables.service
+
+[Service]
+Type=oneshot
+ExecStart=/opt/ic/bin/setup-nftables.sh -n /boot/config/config.ini -i /opt/ic/share/nftables.template -o /run/ic-node/nftables-ruleset/nftables.conf
+
+[Install]
+WantedBy=multi-user.target

ic-os/components/networking/nftables/hostos/setup-nftables.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+# Substitute correct configuration parameters into nftables.conf.
+
+function usage() {
+    cat <<EOF
+Usage:
+  setup-nftables -n config.ini -i nftables.template -o nftables.conf
+
+  Generate nftables config from template file.
+
+  -n config.ini: network configuration description file
+  -i infile: input ic.json5.template file
+  -o outfile: output ic.json5 file
+EOF
+}
+
+# XXX: the following function is duplicate with generate-network-config.sh
+# -- consolidate
+#
+# Read the network config variables from file. The file must be of the form
+# "key=value" for each line with a specific set of keys permissible (see
+# code below).
+#
+# Arguments:
+# - $1: Name of the file to be read.
+function read_network_variables() {
+    # Read limited set of keys. Be extra-careful quoting values as it could
+    # otherwise lead to executing arbitrary shell code!
+    while IFS="=" read -r key value; do
+        case "$key" in
+            "ipv6_prefix") ipv6_prefix="${value}" ;;
+        esac
+    done <"$1"
+}
+
+while getopts "n:i:o:" OPT; do
+    case "${OPT}" in
+        n)
+            NETWORK_CONFIG_FILE="${OPTARG}"
+            ;;
+        i)
+            IN_FILE="${OPTARG}"
+            ;;
+        o)
+            OUT_FILE="${OPTARG}"
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+done
+
+if [ "${IN_FILE}" == "" -o "${OUT_FILE}" == "" ]; then
+    usage
+    exit 1
+fi
+
+# TODO: This should be pulled from the system, in case of configuration via RA.
+if [ "${NETWORK_CONFIG_FILE}" != "" -a -e "${NETWORK_CONFIG_FILE}" ]; then
+    read_network_variables "${NETWORK_CONFIG_FILE}"
+fi
+
+IPV6_PREFIX="${ipv6_prefix:+${ipv6_prefix}::/64}" # Add suffix to prefix if found
+IPV6_PREFIX="${IPV6_PREFIX:-::1/128}"             # Default to loopback for easy templating
+
+mkdir -p /run/ic-node/nftables-ruleset/
+sed -e "s@{{ ipv6_prefix }}@${IPV6_PREFIX}@" \
+    "${IN_FILE}" >"${OUT_FILE}"

ic-os/hostos/context/Dockerfile

@@ -108,6 +108,10 @@ RUN systemctl disable \
 
 # ------ HOSTOS WORK ---------------------------------------------
 
+# Divert symbolic link for dynamically generated nftables
+# ruleset.
+RUN ln -sf /run/ic-node/nftables-ruleset/nftables.conf /etc/nftables.conf
+
 ARG ROOT_PASSWORD=
 RUN \
     if [ "${ROOT_PASSWORD}" != "" ]; then \

rs/orchestrator/testdata/nftables_assigned_replica.conf.golden

@@ -83,6 +83,7 @@ table ip6 filter {
     icmpv6 type nd-router-advert accept
     icmpv6 type nd-neighbor-solicit accept
     icmpv6 type nd-neighbor-advert accept
+    ip6 saddr { {{ ipv6_prefix }} } ct state { new } tcp dport { 7070, 9090, 9091, 9100, 19531, 19100 } accept
     ip6 saddr { hostos } ct state { new } tcp dport { 42372 } accept # Allow access from HostOS metrics-proxy so GuestOS metrics-proxy can proxy certain metrics to HostOS.
     ip6 saddr {fd5b:693c:f8ea::1} ct state { new } tcp dport {2497,4100,8080} accept # Automatic node whitelisting
 ip6 saddr {::ffff:5.5.5.5} ct state { new } tcp dport {1005} accept # node_gwp4o-eaaaa-aaaaa-aaaap-2ai