Merge branch 'main' into concepts-classifier

manifests/k8s/Tiltfile

@@ -335,6 +335,12 @@ def trace(*msg):
     if TRACE:
         print("TRACE:", *msg)
 
+def merge_dicts(*dicts):
+    merged = {}
+    for d in dicts:
+        merged.update(d)
+    return merged
+
 def build_docker_img(ref, context, dockerfile="Dockerfile", **kwargs):
     # Wrapper around docker_build to not repeat image repo,
     # have context relative to git root
@@ -674,14 +680,24 @@ def resolve_objects(included_matchers, ignored_matchers):
 #################################
 
 
-def forward_port_delayed(workload, labels, resource_deps, namespace, service, local_port, remote_port):
-    cmd = "kubectl port-forward -n %s svc/%s %s:%s" % (namespace, service, local_port, remote_port)
+def forward_port_delayed(workload, labels, resource_deps, namespace, service, local_port, remote_port, extra_env={}, readiness_probe=None):
+    if not readiness_probe:
+        readiness_probe=probe(exec=exec_action(['lsof', '-i', ":%s" % local_port])) # Treat as success if some process listens on this port
     local_resource(
         name=workload,
         labels=labels,
         resource_deps=resource_deps,
-        serve_cmd='while sleep 3; do %s; done' % cmd,
-        readiness_probe=probe(exec=exec_action(['lsof', '-i', ":%s" % local_port])), # Treat as success if some process listens on this port
+        serve_cmd=['./tilt/portforward.sh'],
+        serve_env=merge_dicts(
+            {
+                "NAMESPACE": namespace,
+                "RESOURCE_NAME": service,
+                "LOCAL_PORT": str(local_port),
+                "REMOTE_PORT": str(remote_port),
+            },
+            extra_env,
+        ),
+        readiness_probe=readiness_probe,
     )
 
 
@@ -796,8 +812,19 @@ def declare_resources(resources, dep_tree, inv_dep_tree):
             k8s_kind(**kind)
 
     if "aperture-grafana" in resources:
-        forward_port_delayed(workload="aperture-grafana-forward", namespace=APERTURE_SYSTEM_NS, labels=["Aperture"],
-                          resource_deps=["aperture-grafana"], service="aperture-grafana", local_port=3000, remote_port=3000)
+        forward_port_delayed(
+            workload="aperture-grafana-forward",
+            namespace=APERTURE_SYSTEM_NS,
+            labels=["Aperture"],
+            resource_deps=["aperture-grafana"],
+            service="aperture-grafana",
+            local_port=3000,
+            remote_port=3000,
+            extra_env={
+                "PERIOD": "1s",
+                "INITIAL_DELAY": "1s",
+            },
+        )
 
     if "agent" in resources:
         forward_port_delayed(workload="aperture-agent-port-forward", namespace=APERTURE_SYSTEM_NS, labels=["Aperture"],

manifests/k8s/tilt/portforward.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Port-forward using kubectl, with simple healthchecks to work around bug in kubectl
+set -euo pipefail
+
+check_var_set() {
+  local -r var_name="${1?}"
+  if [ -z "${!var_name:-}" ]; then
+    printf 'Required variable %s is not set!\n' "${var_name}" >&2
+    return 1
+  fi
+}
+
+check_var_set RESOURCE_NAME
+check_var_set REMOTE_PORT
+
+: "${NAMESPACE:-}"
+: "${RESOURCE_TYPE:=svc}"
+: "${LOCAL_PORT:=${REMOTE_PORT}}"
+
+: "${REQUEST_PATH:=/}"
+: "${INITIAL_DELAY:=5s}"
+: "${TIMEOUT:=1}"
+: "${PERIOD:=5s}"
+: "${FAILURE_THRESHOLD:=1}"
+
+port_forward_args=()
+if [ -n "${NAMESPACE:-}" ]; then
+  port_forward_args+=( --namespace "${NAMESPACE}" )
+fi
+port_forward_args+=( "${RESOURCE_TYPE}/${RESOURCE_NAME}" "${LOCAL_PORT}:${REMOTE_PORT}" )
+curl_args=(
+  --head # Just retrieve headers - ignore body
+  --silent # Don't show stuff on stdout
+  --show-error # But still show errors
+  --max-time "${TIMEOUT}" # Make curl exit if it can't finish the operation on time
+  "localhost:${LOCAL_PORT}${REQUEST_PATH}"
+)
+
+while sleep 1; do
+  kubectl port-forward "${port_forward_args[@]}" &
+  process_id="${!}"
+  trap 'kill "${process_id}"' EXIT
+  sleep "${INITIAL_DELAY}" || break
+  failures=0
+  while :; do
+    if curl "${curl_args[@]}" | head -n1; then
+      failures=0
+    else
+      exit_code="${?}"
+      if [ "${exit_code}" -eq 124 ]; then
+        printf 'Timed out waiting for response\n' >&2
+      else
+        printf 'Curl failed with exit code %s\n' "${exit_code}" >&2
+      fi
+      if (( ++failures >= FAILURE_THRESHOLD )); then
+        printf 'Restarting port-forwarding\n' >&2
+        kill "${process_id}"
+        wait "${process_id}" || true # Ignore exit code
+        break
+      fi
+    fi
+    sleep "${PERIOD}" || break
+  done
+done