feat(IDX): don't rely on bazel cache for large test deps

bazel/BUILD.bazel

@@ -163,7 +163,7 @@ sh_test(
 )
 
 sh_binary(
-    name = "sha256sum2url_sh",
-    srcs = ["sha256sum2url.sh"],
+    name = "upload_systest_dep",
+    srcs = ["upload_systest_dep.sh"],
     visibility = ["//visibility:public"],
 )

bazel/defs.bzl

@@ -2,7 +2,6 @@
 Utilities for building IC replica and canisters.
 """
 
-load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
 load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_test", "rust_test_suite")
 load("//publish:defs.bzl", "release_nostrip_binary")
 
@@ -106,57 +105,6 @@ mcopy = rule(
     },
 )
 
-def _sha256sum2url_impl(ctx):
-    """
-    Returns cas url pointing to the artifact with checksum specified.
-
-    Waits for the artifact to be published before returning url.
-    """
-    out = ctx.actions.declare_file(ctx.label.name)
-    timeout = ctx.attr.timeout_value[BuildSettingInfo].value
-    ctx.actions.run(
-        executable = "timeout",
-        arguments = [timeout, ctx.executable._sha256sum2url_sh.path],
-        inputs = [ctx.file.src],
-        outputs = [out],
-        tools = [ctx.executable._sha256sum2url_sh],
-        env = {
-            "SHASUMFILE": ctx.file.src.path,
-            "OUT": out.path,
-        },
-    )
-    return [DefaultInfo(files = depset([out]), runfiles = ctx.runfiles(files = [out]))]
-
-_sha256sum2url = rule(
-    implementation = _sha256sum2url_impl,
-    attrs = {
-        "src": attr.label(allow_single_file = True),
-        "_sha256sum2url_sh": attr.label(executable = True, cfg = "exec", default = "//bazel:sha256sum2url_sh"),
-        "timeout_value": attr.label(default = "//bazel:timeout_value"),
-    },
-)
-
-def sha256sum2url(name, src, tags = [], **kwargs):
-    """
-    Returns cas url pointing to the artifact which checksum is returned by src.
-
-    The rule waits until the cache will return http/200 for this artifact.
-    The rule adds "requires-network" as it needs to talk to bazel cache and "manual" to only be performed
-    when its result is requested (directly or by another rule) to not wait when not required.
-
-    Args:
-        name:     the name of the rule
-        src:      the label that returns the file with sha256 checksum of requested artifact.
-        tags:     additional tags.
-        **kwargs: the rest of arguments to be passed to the underlying rule.
-    """
-    _sha256sum2url(
-        name = name,
-        src = src,
-        tags = tags + ["requires-network", "manual"],
-        **kwargs
-    )
-
 # Binaries needed for testing with canister_sandbox
 _SANDBOX_DATA = [
     "//rs/canister_sandbox",

bazel/upload_systest_dep.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+#
+# Upload a dependency to shared storage and returns the download URL.
+#
+# The path to the dependency should be specified as the first (and only) argument.
+#
+# The download URL is printed to stdout.
+
+# NOTE: This script uses bazel-remote as the CAS storage (implementation detail).
+
+# Look up a CAS key (provided as $1) through the redirect server.
+# If the key exists, then then download URL is returned (through stdout).
+# If the key does not exist, the empty string is returned.
+lookup_dep_url() {
+    REDIRECT_SERVER_URL="https://artifacts.idx.dfinity.network"
+    local redirect_url="$REDIRECT_SERVER_URL/cas/$1"
+    local result
+    result=$(curl --silent --head \
+        -w '%{http_code} %{redirect_url}' \
+        "$redirect_url" \
+        | tail -n1)
+
+    local result_code
+    result_code=$(cut -d' ' -f1 <<<"$result")
+    if [ "$result_code" == "404" ]; then
+        # The key was not found
+        return
+    fi
+
+    if [ "$result_code" != "307" ]; then
+        echo "Expected 404 or 307 when looking up dependency '$1', got '$result_code'" >&2
+        exit 1
+    fi
+
+    local result_url
+    result_url=$(cut -d' ' -f2 <<<"$result")
+    if [ -z "$result_url" ]; then
+        echo "Looking up dependency '$1' did not return a URL, got: '$result'" >&2
+        exit 1
+    fi
+
+    echo "$result_url"
+}
+
+dep="${1:?Dependency not specified}"
+dep_filename=${dep#*:}
+
+echo "Found dep to upload $dep_filename ($dep_sha256)" >&2
+result_url=$(lookup_dep_url "$dep_sha256")
+
+# First, figure out _if_ the dep should be uploaded (no point re-uploading several GBs
+# if it's been uploaded already)
+if [ -n "$result_url" ]; then
+    echo "dep '$dep_filename': already uploaded" >&2
+else
+    echo "dep '$dep_filename': not uploaded yet, uploading to $dep_upload_url" >&2
+
+    # Here we figure out the best URL where to upload the dependency. We use the hash of the
+    # empty blob, which will always be found by bazel-remote (baked in) and expect the redirect
+    # server to return the best URL (in practice, the fastest to complete the request)
+    EMPTY_SHA=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+    empty_sha_url=$(lookup_dep_url "$EMPTY_SHA")
+    UPLOAD_URL="${empty_sha_url%"/$EMPTY_SHA"}" # strip the empty blob key from the url
+    echo "Using upload URL: '$UPLOAD_URL'" >&2
+
+    # Upload the dep
+    dep_sha256=$(sha256sum "$dep_filename" | cut -d' ' -f1)
+    dep_upload_url="$UPLOAD_URL/$dep_sha256"
+    curl --silent --fail "$dep_upload_url" --upload-file "$dep_filename"
+
+    # Check that it was actually uploaded and can be served (this sometimes takes a minute)
+    attempt=1
+    result_url=
+    while true; do
+        result_url=$(lookup_dep_url "$dep_sha256")
+
+        if [ -n "$result_url" ]; then
+            break
+        fi
+
+        echo "attempt $attempt failed" >&2
+        if [ "$attempt" -ge 10 ]; then
+            echo "  giving up" >&2
+            exit 1
+        fi
+
+        echo "  will retry in 1s" >&2
+        sleep 1
+
+        attempt=$((attempt + 1))
+    done
+fi
+
+# extract cluster
+# NOTE: this assumes the result URL is https://artifacts.<CLUSTER>.dfinity.network/...
+cluster=$(sed <<<"$result_url" -n -E 's$^https://artifacts.([^.]+).*$\1$p')
+if [ -z "$cluster" ]; then
+    echo "could not read cluster from '$result_url'" >&2
+    exit 1
+fi
+
+echo "dep '$dep_filename': cluster is '$cluster'" >&2
+
+# Use the direct URL, without going through the redirect server
+dep_download_url="http://$cluster.artifacts.proxy-global.dfinity.network:8080/cas/$dep_sha256"
+echo "dep '$dep_filename': download_url: '$dep_download_url'" >&2
+echo "$dep_download_url"

ic-os/defs.bzl

@@ -3,7 +3,7 @@ A macro to build multiple versions of the ICOS image (i.e., dev vs prod)
 """
 
 load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
-load("//bazel:defs.bzl", "gzip_compress", "sha256sum2url", "zstd_compress")
+load("//bazel:defs.bzl", "gzip_compress", "zstd_compress")
 load("//bazel:output_files.bzl", "output_files")
 load("//ci/src/artifacts:upload.bzl", "upload_artifacts")
 load("//ic-os/bootloader:defs.bzl", "build_grub_partition")
@@ -356,20 +356,6 @@ def icos_build(
         tags = ["manual"],
     )
 
-    sha256sum(
-        name = "disk-img.tar.zst.sha256",
-        srcs = [":disk-img.tar.zst"],
-        visibility = visibility,
-        tags = ["manual"],
-    )
-
-    sha256sum2url(
-        name = "disk-img.tar.zst.cas-url",
-        src = ":disk-img.tar.zst.sha256",
-        visibility = visibility,
-        tags = ["manual"],
-    )
-
     # -------------------- Assemble upgrade image --------------------
 
     if upgrades:
@@ -391,20 +377,6 @@ def icos_build(
             tags = ["manual"],
         )
 
-        sha256sum(
-            name = "update-img.tar.zst.sha256",
-            srcs = [":update-img.tar.zst"],
-            visibility = visibility,
-            tags = ["manual"],
-        )
-
-        sha256sum2url(
-            name = "update-img.tar.zst.cas-url",
-            src = ":update-img.tar.zst.sha256",
-            visibility = visibility,
-            tags = ["manual"],
-        )
-
         upgrade_image(
             name = "update-img-test.tar",
             boot_partition = ":partition-boot-test.tzst",
@@ -423,20 +395,6 @@ def icos_build(
             tags = ["manual"],
         )
 
-        sha256sum(
-            name = "update-img-test.tar.zst.sha256",
-            srcs = [":update-img-test.tar.zst"],
-            visibility = visibility,
-            tags = ["manual"],
-        )
-
-        sha256sum2url(
-            name = "update-img-test.tar.zst.cas-url",
-            src = ":update-img-test.tar.zst.sha256",
-            visibility = visibility,
-            tags = ["manual"],
-        )
-
     # -------------------- Upload artifacts --------------------
 
     upload_suffix = ""
@@ -538,16 +496,16 @@ EOF
         data = [
             "//rs/ic_os/dev_test_tools/launch-single-vm:launch-single-vm",
             "//ic-os/components:hostos-scripts/build-bootstrap-config-image.sh",
-            ":disk-img.tar.zst.cas-url",
-            ":disk-img.tar.zst.sha256",
+            ":disk-img.tar.zst",
             ":version.txt",
+            "//bazel:upload_systest_dep",
         ],
         env = {
             "BIN": "$(location //rs/ic_os/dev_test_tools/launch-single-vm:launch-single-vm)",
+            "UPLOAD_SYSTEST_DEP": "$(location //bazel:upload_systest_dep)",
             "SCRIPT": "$(location //ic-os/components:hostos-scripts/build-bootstrap-config-image.sh)",
             "VERSION_FILE": "$(location :version.txt)",
-            "URL_FILE": "$(location :disk-img.tar.zst.cas-url)",
-            "SHA_FILE": "$(location :disk-img.tar.zst.sha256)",
+            "DISK_IMG": "$(location :disk-img.tar.zst)",
         },
         testonly = True,
         tags = ["manual"],
@@ -854,20 +812,6 @@ EOF
         tags = ["manual"],
     )
 
-    sha256sum(
-        name = "disk-img.tar.zst.sha256",
-        srcs = [":disk-img.tar.zst"],
-        visibility = visibility,
-        tags = ["manual"],
-    )
-
-    sha256sum2url(
-        name = "disk-img.tar.zst.cas-url",
-        src = ":disk-img.tar.zst.sha256",
-        visibility = visibility,
-        tags = ["manual"],
-    )
-
     gzip_compress(
         name = "disk-img.tar.gz",
         srcs = ["disk-img.tar"],

ic-os/dev-tools/launch-remote-vm.sh

@@ -2,7 +2,11 @@
 
 set -euo pipefail
 
-cmd="$BIN --version $(cat "$VERSION_FILE") --url $(cat "$URL_FILE") --sha256 $(cat "$SHA_FILE") --build-bootstrap-script $(realpath "$SCRIPT")"
+# First upload the image
+sha256=$(sha256sum "$DISK_IMG" | cut -d' ' -f1)
+image_download_url=$("$UPLOAD_SYSTEST_DEP" "$DISK_IMG")
+
+cmd="$BIN --version $(cat "$VERSION_FILE") --url "$image_download_url" --sha256 "$sha256" --build-bootstrap-script $(realpath "$SCRIPT")"
 
 # Hack to switch nested for SetupOS
 if [[ "$0" =~ "setupos" ]]; then

rs/tests/boundary_nodes/BUILD.bazel

@@ -105,6 +105,7 @@ system_test_nns(
         "k8s",
     ],
     target_compatible_with = ["@platforms//os:linux"],  # requires libssh that does not build on Mac OS
+    uses_boundary_guestos = True,
     runtime_deps = BOUNDARY_NODE_GUESTOS_RUNTIME_DEPS +
                    GUESTOS_RUNTIME_DEPS +
                    UNIVERSAL_VM_RUNTIME_DEPS +

rs/tests/common.bzl

@@ -315,8 +315,6 @@ GRAFANA_RUNTIME_DEPS = UNIVERSAL_VM_RUNTIME_DEPS + [
 ]
 
 BOUNDARY_NODE_GUESTOS_RUNTIME_DEPS = [
-    "//ic-os/boundary-guestos/envs/dev:disk-img.tar.zst.cas-url",
-    "//ic-os/boundary-guestos/envs/dev:disk-img.tar.zst.sha256",
     "//ic-os/boundary-guestos:scripts/build-bootstrap-config-image.sh",
 ]