Add support for remote_files to bzlmod

This is a continuation of bazelbuild#22155 that adds the newly added 'remote_files' attribute for http_archive to the bzlmod functionality. The end goal is to then update BCR to this new functionality to overlay files rather than use patch files when providing MODULE/WORKSPACE/BUILD files. bazelbuild/bazel-central-registry#1566 has a good discussion of the rationale.
fzakaria · May 13, 2024 · 7fa7d37 · 7fa7d37
1 parent b9a0578
commit 7fa7d37
Show file tree

Hide file tree

Showing 5 changed files with 84 additions and 0 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java
@@ -19,6 +19,11 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.Collectors;
 import net.starlark.java.eval.StarlarkInt;
 
 /**
@@ -77,6 +82,19 @@ public ArchiveRepoSpecBuilder setRemotePatches(ImmutableMap<String, String> remo
     return this;
   }
 
+  @CanIgnoreReturnValue
+  public ArchiveRepoSpecBuilder setOverlay(ImmutableMap<String, RemoteFile> overlay) {
+    final Map<String, List<String>> remoteFiles = overlay.entrySet().stream().collect(Collectors.toMap(
+        Entry::getKey,
+        e -> e.getValue().urls.stream().map(URL::toString).collect(Collectors.toList())
+    ));
+    final Map<String, String> remoteFilesIntegrity = overlay.entrySet().stream()
+        .collect(Collectors.toMap(Entry::getKey, e -> e.getValue().integrity));
+    attrBuilder.put("remote_file_urls", remoteFiles);
+    attrBuilder.put("remote_file_integrity", remoteFilesIntegrity);
+    return this;
+  }
+
   @CanIgnoreReturnValue
   public ArchiveRepoSpecBuilder setRemotePatchStrip(int remotePatchStrip) {
     attrBuilder.put("remote_patch_strip", StarlarkInt.of(remotePatchStrip));

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD
@@ -21,6 +21,7 @@ java_library(
         "GitRepoSpecBuilder.java",
         "ModuleFile.java",
         "ModuleKey.java",
+        "RemoteFile.java",
         "RepoSpec.java",
         "Version.java",
     ],

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java
@@ -196,6 +196,7 @@ private static class ArchiveSourceJson {
     String integrity;
     String stripPrefix;
     Map<String, String> patches;
+    Map<String, RemoteFile> overlay;
     int patchStrip;
     String archiveType;
   }
@@ -386,11 +387,17 @@ private RepoSpec createArchiveRepoSpec(
       }
     }
 
+    ImmutableMap.Builder<String, RemoteFile> overlay = new ImmutableMap.Builder<>();
+    if (sourceJson .overlay != null) {
+      overlay.putAll(sourceJson.overlay);
+    }
+
     return new ArchiveRepoSpecBuilder()
         .setUrls(urls.build())
         .setIntegrity(sourceJson.integrity)
         .setStripPrefix(Strings.nullToEmpty(sourceJson.stripPrefix))
         .setRemotePatches(remotePatches.buildOrThrow())
+        .setOverlay(overlay.buildOrThrow())
         .setRemotePatchStrip(sourceJson.patchStrip)
         .setArchiveType(sourceJson.archiveType)
         .build();

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RemoteFile.java
@@ -0,0 +1,20 @@
+package com.google.devtools.build.lib.bazel.bzlmod;
+import java.net.URL;
+import java.util.List;
+
+/**
+ * For use with IndexRegistry and associated files. A simple pojo to track remote files that are
+ * offered at multiple urls (mirrors) with a single integrity.
+ * We split up the file here to simplify the dependency.
+ */
+class RemoteFile {
+
+  RemoteFile(String integrity, List<URL> urls) {
+    this.integrity = integrity;
+    this.urls = urls;
+  }
+
+  List<URL> urls;
+  String integrity;
+}
+
diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java
@@ -41,6 +41,7 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.Writer;
+import java.net.URL;
 import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.List;
@@ -182,6 +183,18 @@ public void testGetArchiveRepoSpec() throws Exception {
         "  },",
         "  \"patch_strip\": 3",
         "}");
+    server.serve(
+        "/modules/baz/3.0/source.json",
+        "{",
+        "  \"url\": \"https://example.com/archive.jar?with=query\",",
+        "  \"integrity\": \"sha256-bleh\",",
+        "  \"overlay\": {",
+        "    \"BUILD.bazel\": {",
+        "       \"urls\": [\"http://mirror1\", \"http://mirror2\"],",
+        "       \"integrity\": \"sha256-bleh-overlay\"",
+        "   }",
+        "  }",
+        "}");
     server.start();
 
     Registry registry =
@@ -198,6 +211,7 @@ public void testGetArchiveRepoSpec() throws Exception {
                 .setIntegrity("sha256-blah")
                 .setStripPrefix("pref")
                 .setRemotePatches(ImmutableMap.of())
+                .setOverlay(ImmutableMap.of())
                 .setRemotePatchStrip(0)
                 .build());
     assertThat(registry.getRepoSpec(createModuleKey("bar", "2.0"), reporter))
@@ -216,6 +230,28 @@ public void testGetArchiveRepoSpec() throws Exception {
                         server.getUrl() + "/modules/bar/2.0/patches/2.fix-that.patch",
                             "sha256-kek"))
                 .setRemotePatchStrip(3)
+                .setOverlay(ImmutableMap.of())
+                .build());
+    assertThat(registry.getRepoSpec(createModuleKey("baz", "3.0"), reporter))
+        .isEqualTo(
+            new ArchiveRepoSpecBuilder()
+                .setUrls(
+                    ImmutableList.of(
+                        "https://mirror.bazel.build/example.com/archive.jar?with=query",
+                        "file:///home/bazel/mymirror/example.com/archive.jar?with=query",
+                        "https://example.com/archive.jar?with=query"))
+                .setIntegrity("sha256-bleh")
+                .setStripPrefix("")
+                .setOverlay(
+                    ImmutableMap.of(
+                        "BUILD.bazel", new RemoteFile(
+                            "sha256-bleh-overlay",
+                            List.of(new URL("http://mirror1"), new URL("http://mirror2"))
+                        )
+                    )
+                )
+                .setRemotePatches(ImmutableMap.of())
+                .setRemotePatchStrip(0)
                 .build());
   }
 
@@ -264,6 +300,7 @@ public void testGetRepoInvalidRegistryJsonSpec() throws Exception {
                 .setIntegrity("sha256-blah")
                 .setStripPrefix("pref")
                 .setRemotePatches(ImmutableMap.of())
+                .setOverlay(ImmutableMap.of())
                 .setRemotePatchStrip(0)
                 .build());
   }
@@ -351,6 +388,7 @@ public void testArchiveWithExplicitType() throws Exception {
                 .setArchiveType("zip")
                 .setRemotePatches(ImmutableMap.of())
                 .setRemotePatchStrip(0)
+                .setOverlay(ImmutableMap.of())
                 .build());
   }