diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 96262f88df2b0d..78e9995cea73e8 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -428,6 +428,7 @@ tasks: - "-//src/test/py/bazel:bazel_yanked_versions_test" - "-//src/test/py/bazel:bzlmod_query_test" - "-//src/test/py/bazel:mod_command_test" + - "-//src/test/shell/bazel:starlark_repository_test" - "-//src/test/shell/bazel:verify_workspace" # Flaky on rbe_ubuntu2004 # https://github.com/bazelbuild/continuous-integration/issues/1631 diff --git a/MODULE.bazel b/MODULE.bazel index 6f08b7adc58537..c420597a6c4f2e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -33,6 +33,9 @@ bazel_dep(name = "rules_testing", version = "0.6.0") bazel_dep(name = "googletest", version = "1.14.0", repo_name = "com_google_googletest") bazel_dep(name = "with_cfg.bzl", version = "0.2.4") +# TODO(fmeum): Remove the dependency on buildozer after Bazel is built with 7.2.0. +bazel_dep(name = "buildozer", version = "7.1.1.1") + # TODO(pcloudy): Add remoteapis and googleapis as Bazel modules in the BCR. bazel_dep(name = "remoteapis", version = "") bazel_dep(name = "googleapis", version = "") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 78f44e4fb78dbf..98d5e8e9cbc352 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,6 +1,6 @@ { "lockFileVersion": 6, - "moduleFileHash": "dea29024bdcd8f84856d5187b9d45ade34a8a0349387397a95abec1c998a3e30", + "moduleFileHash": "690d0e76bfb200c981d03e6c0f20853ff9533788627a779890330e089aa7c41c", "flags": { "cmdRegistries": [ "https://bcr.bazel.build/" @@ -39,7 +39,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 69, + "line": 72, "column": 22 }, "imports": { @@ -170,7 +170,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 70, + "line": 73, "column": 14 } }, @@ -185,7 +185,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 196, + "line": 199, "column": 19 } }, @@ -200,7 +200,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 196, + "line": 199, "column": 19 } }, @@ -215,7 +215,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 196, + "line": 199, "column": 19 } }, @@ -230,7 +230,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 196, + "line": 199, "column": 19 } }, @@ -245,7 +245,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 196, + "line": 199, "column": 19 } }, @@ -260,7 +260,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 196, + "line": 199, "column": 19 } }, @@ -275,7 +275,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 196, + "line": 199, "column": 19 } }, @@ -290,7 +290,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 196, + "line": 199, "column": 19 } }, @@ -305,7 +305,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 196, + "line": 199, "column": 19 } }, @@ -333,7 +333,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 336, + "line": 339, "column": 22 } } @@ -347,7 +347,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 217, + "line": 220, "column": 32 }, "imports": { @@ -387,7 +387,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 251, + "line": 254, "column": 23 }, "imports": {}, @@ -401,7 +401,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 252, + "line": 255, "column": 17 } } @@ -415,7 +415,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 254, + "line": 257, "column": 20 }, "imports": { @@ -433,7 +433,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 255, + "line": 258, "column": 10 } } @@ -447,7 +447,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 266, + "line": 269, "column": 33 }, "imports": { @@ -478,7 +478,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 287, + "line": 290, "column": 29 }, "imports": { @@ -495,7 +495,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 290, + "line": 293, "column": 20 }, "imports": { @@ -514,7 +514,7 @@ "devDependency": false, "location": { "file": "@@//:MODULE.bazel", - "line": 291, + "line": 294, "column": 12 } } @@ -528,7 +528,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 303, + "line": 306, "column": 32 }, "imports": { @@ -547,7 +547,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 311, + "line": 314, "column": 31 }, "imports": { @@ -564,7 +564,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 314, + "line": 317, "column": 48 }, "imports": { @@ -581,7 +581,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 358, + "line": 361, "column": 35 }, "imports": { @@ -598,7 +598,7 @@ "usingModule": "", "location": { "file": "@@//:MODULE.bazel", - "line": 361, + "line": 364, "column": 42 }, "imports": { @@ -632,6 +632,7 @@ "rules_testing": "rules_testing@0.6.0", "com_google_googletest": "googletest@1.14.0", "with_cfg.bzl": "with_cfg.bzl@0.2.4", + "buildozer": "buildozer@7.1.1.1", "remoteapis": "remoteapis@_", "googleapis": "googleapis@_", "apple_support": "apple_support@1.8.1", @@ -1830,6 +1831,72 @@ } } }, + "buildozer@7.1.1.1": { + "name": "buildozer", + "version": "7.1.1.1", + "key": "buildozer@7.1.1.1", + "repoName": "buildozer", + "executionPlatformsToRegister": [], + "toolchainsToRegister": [], + "extensionUsages": [ + { + "extensionBzlFile": "@buildozer//:buildozer_binary.bzl", + "extensionName": "buildozer_binary", + "usingModule": "buildozer@7.1.1.1", + "location": { + "file": "https://bcr.bazel.build/modules/buildozer/7.1.1.1/MODULE.bazel", + "line": 7, + "column": 33 + }, + "imports": { + "buildozer_binary": "buildozer_binary" + }, + "devImports": [], + "tags": [ + { + "tagName": "buildozer", + "attributeValues": { + "sha256": { + "darwin-amd64": "6286bbbcb5314d675fcb4ede30edf397a68c611005412c937b2cc0c4bf4b714b", + "darwin-arm64": "0a2f70aa7b864de9bf71d1aac39659017b7a84169506e350d9ec77c609265212", + "linux-amd64": "9a7424aca7ca911c85cfedeadf065f0d95c492e80f0e29bd07ea98f6eb087259", + "linux-arm64": "07c8ed5ca3efea7e20756e9060660b7e658c781953c60650a1b99cd8bb857fcf", + "windows-amd64": "edad2e85cc691b064de9675db8e732b2c5a4ada0b32282ecdf4b18692cc8d3fe" + }, + "version": "7.1.1" + }, + "devDependency": false, + "location": { + "file": "https://bcr.bazel.build/modules/buildozer/7.1.1.1/MODULE.bazel", + "line": 8, + "column": 27 + } + } + ], + "hasDevUseExtension": false, + "hasNonDevUseExtension": true + } + ], + "deps": { + "bazel_tools": "bazel_tools@_", + "local_config_platform": "local_config_platform@_" + }, + "repoSpec": { + "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", + "ruleClassName": "http_archive", + "attributes": { + "urls": [ + "https://github.com/fmeum/buildozer/releases/download/v7.1.1.1/buildozer-v7.1.1.1.tar.gz" + ], + "integrity": "sha256-wjmMsVBR96yT+WOlwN8g1gZRkc/VSPPnTjckHmIs9m8=", + "strip_prefix": "buildozer-7.1.1.1", + "remote_patches": { + "https://bcr.bazel.build/modules/buildozer/7.1.1.1/patches/module_dot_bazel_version.patch": "sha256-dOIwFLCmsm5pLgHK9aPnVXTjEsvn3xC4aHTXpbyrR2o=" + }, + "remote_patch_strip": 1 + } + } + }, "remoteapis@_": { "name": "remoteapis", "version": "", @@ -2426,7 +2493,7 @@ "rules_license": "rules_license@0.0.7", "rules_proto": "rules_proto@5.3.0-21.7", "rules_python": "rules_python@0.26.0", - "buildozer": "buildozer@6.4.0.2", + "buildozer": "buildozer@7.1.1.1", "platforms": "platforms@0.0.9", "com_google_protobuf": "protobuf@21.7", "zlib": "zlib@1.3", @@ -2673,78 +2740,12 @@ "remote_patch_strip": 0 } } - }, - "buildozer@6.4.0.2": { - "name": "buildozer", - "version": "6.4.0.2", - "key": "buildozer@6.4.0.2", - "repoName": "buildozer", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [ - { - "extensionBzlFile": "@buildozer//:buildozer_binary.bzl", - "extensionName": "buildozer_binary", - "usingModule": "buildozer@6.4.0.2", - "location": { - "file": "https://bcr.bazel.build/modules/buildozer/6.4.0.2/MODULE.bazel", - "line": 7, - "column": 33 - }, - "imports": { - "buildozer_binary": "buildozer_binary" - }, - "devImports": [], - "tags": [ - { - "tagName": "buildozer", - "attributeValues": { - "sha256": { - "darwin-amd64": "d29e347ecd6b5673d72cb1a8de05bf1b06178dd229ff5eb67fad5100c840cc8e", - "darwin-arm64": "9b9e71bdbec5e7223871e913b65d12f6d8fa026684daf991f00e52ed36a6978d", - "linux-amd64": "8dfd6345da4e9042daa738d7fdf34f699c5dfce4632f7207956fceedd8494119", - "linux-arm64": "6559558fded658c8fa7432a9d011f7c4dcbac6b738feae73d2d5c352e5f605fa", - "windows-amd64": "e7f05bf847f7c3689dd28926460ce6e1097ae97380ac8e6ae7147b7b706ba19b" - }, - "version": "6.4.0" - }, - "devDependency": false, - "location": { - "file": "https://bcr.bazel.build/modules/buildozer/6.4.0.2/MODULE.bazel", - "line": 8, - "column": 27 - } - } - ], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/fmeum/buildozer/releases/download/v6.4.0.2/buildozer-v6.4.0.2.tar.gz" - ], - "integrity": "sha256-k7tFKQMR2AygxpmZfH0yEPnQmF3efFgD9rBPkj+Yz/8=", - "strip_prefix": "buildozer-6.4.0.2", - "remote_patches": { - "https://bcr.bazel.build/modules/buildozer/6.4.0.2/patches/module_dot_bazel_version.patch": "sha256-gKANF2HMilj7bWmuXs4lbBIAAansuWC4IhWGB/CerjU=" - }, - "remote_patch_strip": 1 - } - } } }, "moduleExtensions": { "//:extensions.bzl%bazel_android_deps": { "general": { - "bzlTransitiveDigest": "yndmb1zHgPO9iXw7vFZ+9ZrV/a94ZFKtLw4M9SZIarg=", + "bzlTransitiveDigest": "ZQlKKnSCJVqDHuXPmDAvyFc/HP1UqQ4Hgsl6trOvHvA=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2785,6 +2786,11 @@ "blake3", "blake3~" ], + [ + "", + "buildozer", + "buildozer~" + ], [ "", "c-ares", @@ -2884,16 +2890,21 @@ "bazel_tools", "bazel_tools", "bazel_tools" + ], + [ + "buildozer~", + "buildozer_binary", + "buildozer~~buildozer_binary~buildozer_binary" ] ] } }, "//:extensions.bzl%bazel_build_deps": { "general": { - "bzlTransitiveDigest": "yndmb1zHgPO9iXw7vFZ+9ZrV/a94ZFKtLw4M9SZIarg=", + "bzlTransitiveDigest": "ZQlKKnSCJVqDHuXPmDAvyFc/HP1UqQ4Hgsl6trOvHvA=", "recordedFileInputs": { - "@@//MODULE.bazel": "dea29024bdcd8f84856d5187b9d45ade34a8a0349387397a95abec1c998a3e30", - "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "025c1c4a389201142d146e850e121f4c80c142ef44bca8bf1cc912140fc8a7e1" + "@@//MODULE.bazel": "690d0e76bfb200c981d03e6c0f20853ff9533788627a779890330e089aa7c41c", + "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "ffc0f61d2fb31b4ce6dad9a773279ad434f2aabadfaebcb1df5c94bb7ddb4d7d" }, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2912,23 +2923,23 @@ "ruleClassName": "repo_cache_tar", "attributes": { "repos": [ + "abseil-cpp~", + "apple_support~", + "bazel_skylib~", + "buildozer~", + "googletest~", + "platforms", + "protobuf~", "rules_cc~", "rules_java~", + "rules_jvm_external~", "rules_license~", + "rules_pkg~", "rules_proto~", "rules_python~", - "buildozer~", - "platforms", - "protobuf~", - "zlib~", - "apple_support~", - "bazel_skylib~", - "rules_pkg~", - "abseil-cpp~", + "stardoc~", "upb~", - "rules_jvm_external~", - "googletest~", - "stardoc~" + "zlib~" ], "lockfile": "@@//src/test/tools/bzlmod:MODULE.bazel.lock" } @@ -2978,6 +2989,7 @@ "apple_support~", "bazel_skylib~", "blake3~", + "buildozer~", "c-ares~", "grpc~", "protobuf~", @@ -2985,10 +2997,10 @@ "platforms", "rules_cc~", "rules_go~", + "rules_graalvm~", "rules_java~", "rules_jvm_external~", "rules_kotlin~", - "rules_graalvm~", "rules_license~", "rules_pkg~", "rules_proto~", @@ -3006,7 +3018,13 @@ "grpc~~grpc_repo_deps_ext~rules_cc", "rules_kotlin~~rules_kotlin_extensions~com_github_jetbrains_kotlin" ], - "dirname": "derived/repository_cache" + "dirname": "derived/repository_cache", + "module_files": [ + "@@//:MODULE.bazel", + "@@//third_party/googleapis:MODULE.bazel", + "@@//third_party/remoteapis:MODULE.bazel", + "@@//src:MODULE.tools" + ] } }, "debian_bin_deps": { @@ -3160,6 +3178,11 @@ "blake3", "blake3~" ], + [ + "", + "buildozer", + "buildozer~" + ], [ "", "c-ares", @@ -3259,13 +3282,18 @@ "bazel_tools", "bazel_tools", "bazel_tools" + ], + [ + "buildozer~", + "buildozer_binary", + "buildozer~~buildozer_binary~buildozer_binary" ] ] } }, "//:extensions.bzl%bazel_test_deps": { "general": { - "bzlTransitiveDigest": "yndmb1zHgPO9iXw7vFZ+9ZrV/a94ZFKtLw4M9SZIarg=", + "bzlTransitiveDigest": "ZQlKKnSCJVqDHuXPmDAvyFc/HP1UqQ4Hgsl6trOvHvA=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -3316,6 +3344,11 @@ "blake3", "blake3~" ], + [ + "", + "buildozer", + "buildozer~" + ], [ "", "c-ares", @@ -3415,6 +3448,11 @@ "bazel_tools", "bazel_tools", "bazel_tools" + ], + [ + "buildozer~", + "buildozer_binary", + "buildozer~~buildozer_binary~buildozer_binary" ] ] } @@ -3667,13 +3705,13 @@ "ruleClassName": "_buildozer_binary_repo", "attributes": { "sha256": { - "darwin-amd64": "d29e347ecd6b5673d72cb1a8de05bf1b06178dd229ff5eb67fad5100c840cc8e", - "darwin-arm64": "9b9e71bdbec5e7223871e913b65d12f6d8fa026684daf991f00e52ed36a6978d", - "linux-amd64": "8dfd6345da4e9042daa738d7fdf34f699c5dfce4632f7207956fceedd8494119", - "linux-arm64": "6559558fded658c8fa7432a9d011f7c4dcbac6b738feae73d2d5c352e5f605fa", - "windows-amd64": "e7f05bf847f7c3689dd28926460ce6e1097ae97380ac8e6ae7147b7b706ba19b" + "darwin-amd64": "6286bbbcb5314d675fcb4ede30edf397a68c611005412c937b2cc0c4bf4b714b", + "darwin-arm64": "0a2f70aa7b864de9bf71d1aac39659017b7a84169506e350d9ec77c609265212", + "linux-amd64": "9a7424aca7ca911c85cfedeadf065f0d95c492e80f0e29bd07ea98f6eb087259", + "linux-arm64": "07c8ed5ca3efea7e20756e9060660b7e658c781953c60650a1b99cd8bb857fcf", + "windows-amd64": "edad2e85cc691b064de9675db8e732b2c5a4ada0b32282ecdf4b18692cc8d3fe" }, - "version": "6.4.0" + "version": "7.1.1" } } }, diff --git a/distdir.bzl b/distdir.bzl index e872dc4d253b1e..cbc02a7b9f3a72 100644 --- a/distdir.bzl +++ b/distdir.bzl @@ -13,7 +13,7 @@ # limitations under the License. """Defines a repository rule that generates an archive consisting of the specified files to fetch""" -load("//src/tools/bzlmod:utils.bzl", "parse_http_artifacts") +load("//src/tools/bzlmod:utils.bzl", "parse_http_artifacts", "parse_registry_files") _BUILD = """ load("@rules_pkg//pkg:tar.bzl", "pkg_tar") @@ -87,10 +87,11 @@ def _repo_cache_tar_impl(ctx): """ lockfile_path = ctx.path(ctx.attr.lockfile) http_artifacts = parse_http_artifacts(ctx, lockfile_path, ctx.attr.repos) + registry_files = parse_registry_files(ctx, lockfile_path, ctx.attr.module_files) archive_files = [] readme_content = "This directory contains repository cache artifacts for the following URLs:\n\n" - for artifact in http_artifacts: + for artifact in http_artifacts + registry_files: url = artifact["url"] if "integrity" in artifact: # ./tempfile could be a hard link if --experimental_repository_cache_hardlinks is used, @@ -122,6 +123,7 @@ _repo_cache_tar_attrs = { "lockfile": attr.label(default = Label("//:MODULE.bazel.lock")), "dirname": attr.string(default = "repository_cache"), "repos": attr.string_list(), + "module_files": attr.label_list(), } repo_cache_tar = repository_rule( diff --git a/extensions.bzl b/extensions.bzl index c5cfdfbb4cc7dc..d531b97f61c2a2 100644 --- a/extensions.bzl +++ b/extensions.bzl @@ -29,7 +29,17 @@ def _bazel_build_deps(_ctx): _ctx.path(Label("//:MODULE.bazel")) # Make sure the `bootstrap_repo_cache` repo is updated when MODULE.bazel changes. embedded_jdk_repositories() debian_deps() - repo_cache_tar(name = "bootstrap_repo_cache", repos = DIST_ARCHIVE_REPOS, dirname = "derived/repository_cache") + repo_cache_tar( + name = "bootstrap_repo_cache", + repos = DIST_ARCHIVE_REPOS, + dirname = "derived/repository_cache", + module_files = [ + "//:MODULE.bazel", + "//third_party/googleapis:MODULE.bazel", + "//third_party/remoteapis:MODULE.bazel", + "//src:MODULE.tools", + ], + ) BAZEL_TOOLS_DEPS_REPOS = parse_bazel_module_repos(_ctx, _ctx.path(Label("//src/test/tools/bzlmod:MODULE.bazel.lock"))) repo_cache_tar(name = "bazel_tools_repo_cache", repos = BAZEL_TOOLS_DEPS_REPOS, lockfile = "//src/test/tools/bzlmod:MODULE.bazel.lock") distdir_tar(name = "workspace_repo_cache", dist_deps = WORKSPACE_REPOS) diff --git a/repositories.bzl b/repositories.bzl index 911ab621d8cb50..71be37187708b4 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -24,10 +24,12 @@ load("//src/tools/bzlmod:utils.bzl", "get_canonical_repo_name") # ################################################################################## DIST_ARCHIVE_REPOS = [get_canonical_repo_name(repo) for repo in [ + # keep sorted "abseil-cpp", "apple_support", "bazel_skylib", "blake3", + "buildozer", "c-ares", "com_github_grpc_grpc", "com_google_protobuf", @@ -35,10 +37,10 @@ DIST_ARCHIVE_REPOS = [get_canonical_repo_name(repo) for repo in [ "platforms", "rules_cc", "rules_go", + "rules_graalvm", "rules_java", "rules_jvm_external", "rules_kotlin", - "rules_graalvm", "rules_license", "rules_pkg", "rules_proto", diff --git a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java index 8378eedc6e6368..58ce1c66eef484 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java @@ -270,8 +270,8 @@ SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace .addSkyFunction( SkyFunctions.REGISTRY, new RegistryFunction( - new RegistryFactoryImpl( - directories.getWorkspace(), downloadManager, clientEnvironmentSupplier))) + new RegistryFactoryImpl(downloadManager, clientEnvironmentSupplier), + directories.getWorkspace())) .addSkyFunction(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) .addSkyFunction(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction()) .addSkyFunction( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index be364dfffe00ba..f35ea95e08397d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -79,12 +79,13 @@ java_library( ], deps = [ ":common", + ":yanked_versions_value", + "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache", "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/events", "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/util:os", - "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//third_party:gson", @@ -101,10 +102,11 @@ java_library( "//src/main/java/com/google/devtools/build/lib:runtime", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension", "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", - "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/cmdline", + "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value", "//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster", "//src/main/java/com/google/devtools/build/lib/vfs", + "//src/main/java/com/google/devtools/build/skyframe", "//third_party:flogger", "//third_party:guava", "//third_party:jsr305", @@ -119,9 +121,7 @@ java_library( "BazelDepGraphValue.java", "BazelFetchAllValue.java", "BazelLockFileValue.java", - "BazelModuleResolutionEvent.java", "BazelModuleResolutionValue.java", - "BzlmodFlagsAndEnvVars.java", "CompiledModuleFile.java", "GitOverride.java", "InterimModule.java", @@ -188,7 +188,6 @@ java_library( "BazelFetchAllFunction.java", "BazelLockFileFunction.java", "BazelModuleResolutionFunction.java", - "BzlmodFlagsAndEnvVars.java", "DelegateTypeAdapterFactory.java", "Discovery.java", "GsonTypeAdapterUtil.java", @@ -298,12 +297,10 @@ java_library( deps = [ ":resolution", ":root_module_file_fixup", - "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", "//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:serialization-constant", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", - "//src/main/java/net/starlark/java/eval", "//third_party:auto_value", "//third_party:guava", ], @@ -316,20 +313,16 @@ java_library( ], deps = [ ":common", - ":exception", ":resolution", - ":resolution_impl", ":root_module_file_fixup", ":tidy", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function", - "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value", "//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value", "//src/main/java/com/google/devtools/build/lib/vfs", "//src/main/java/com/google/devtools/build/skyframe", "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", "//src/main/java/net/starlark/java/eval", - "//src/main/protobuf:failure_details_java_proto", "//third_party:guava", "//third_party:jsr305", ], @@ -401,3 +394,16 @@ java_library( "//third_party:guava", ], ) + +java_library( + name = "yanked_versions_value", + srcs = ["YankedVersionsValue.java"], + deps = [ + ":common", + "//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions", + "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec", + "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects", + "//third_party:auto_value", + "//third_party:guava", + ], +) diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java index 7b11bd8507c109..5770b09e1d3d74 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java @@ -15,40 +15,31 @@ package com.google.devtools.build.lib.bazel.bzlmod; -import static com.google.common.base.Strings.nullToEmpty; import static com.google.common.collect.ImmutableBiMap.toImmutableBiMap; import static com.google.common.collect.ImmutableList.toImmutableList; -import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static java.util.stream.Collectors.counting; import static java.util.stream.Collectors.groupingBy; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableBiMap; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableTable; -import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; -import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.PackageIdentifier; import com.google.devtools.build.lib.cmdline.RepositoryMapping; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.packages.LabelConverter; import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code; -import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; -import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue; import com.google.devtools.build.lib.vfs.PathFragment; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import java.util.Map; import java.util.Map.Entry; import javax.annotation.Nullable; @@ -65,59 +56,12 @@ public BazelDepGraphFunction() {} @Nullable public SkyValue compute(SkyKey skyKey, Environment env) throws BazelDepGraphFunctionException, InterruptedException { - RootModuleFileValue root = - (RootModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE); - if (root == null) { + BazelModuleResolutionValue selectionResult = + (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); + if (env.valuesMissing()) { return null; } - LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env); - - ImmutableMap localOverrideHashes = null; - ImmutableMap depGraph = null; - BzlmodFlagsAndEnvVars flags = null; - BazelLockFileValue lockfile = null; - - // If the module has not changed (has the same contents and flags as the lockfile), - // read the dependency graph from the lock file, else run resolution and update lockfile - if (!lockfileMode.equals(LockfileMode.OFF)) { - lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); - if (lockfile == null) { - return null; - } - flags = getFlagsAndEnvVars(env); - if (flags == null) { // unable to read environment variables - return null; - } - localOverrideHashes = getLocalOverridesHashes(root.getOverrides(), env); - if (localOverrideHashes == null) { // still reading an override "module" - return null; - } - - if (root.getModuleFileHash().equals(lockfile.getModuleFileHash()) - && flags.equals(lockfile.getFlags()) - && localOverrideHashes.equals(lockfile.getLocalOverrideHashes())) { - depGraph = lockfile.getModuleDepGraph(); - } else if (lockfileMode.equals(LockfileMode.ERROR)) { - ImmutableList diffLockfile = - lockfile.getModuleAndFlagsDiff(root.getModuleFileHash(), localOverrideHashes, flags); - throw new BazelDepGraphFunctionException( - ExternalDepsException.withMessage( - Code.BAD_MODULE, - "MODULE.bazel.lock is no longer up-to-date because: %s. " - + "Please run `bazel mod deps --lockfile_mode=update` to update your lockfile.", - String.join(", ", diffLockfile)), - Transience.PERSISTENT); - } - } - - if (depGraph == null) { - BazelModuleResolutionValue selectionResult = - (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY); - if (env.valuesMissing()) { - return null; - } - depGraph = selectionResult.getResolvedDepGraph(); - } + var depGraph = selectionResult.getResolvedDepGraph(); ImmutableBiMap canonicalRepoNameLookup = computeCanonicalRepoNameLookup(depGraph); @@ -131,20 +75,6 @@ public SkyValue compute(SkyKey skyKey, Environment env) ImmutableBiMap extensionUniqueNames = calculateUniqueNameForUsedExtensionId(extensionUsagesById); - if (lockfileMode.equals(LockfileMode.UPDATE)) { - BazelLockFileValue resolutionOnlyLockfile = - BazelLockFileValue.builder() - .setModuleFileHash(root.getModuleFileHash()) - .setFlags(flags) - .setLocalOverrideHashes(localOverrideHashes) - .setModuleDepGraph(depGraph) - .build(); - env.getListener() - .post( - BazelModuleResolutionEvent.create( - lockfile, resolutionOnlyLockfile, extensionUsagesById)); - } - return BazelDepGraphValue.create( depGraph, canonicalRepoNameLookup, @@ -153,63 +83,6 @@ public SkyValue compute(SkyKey skyKey, Environment env) extensionUniqueNames.inverse()); } - @Nullable - @VisibleForTesting - static ImmutableMap getLocalOverridesHashes( - Map overrides, Environment env) throws InterruptedException { - ImmutableMap.Builder localOverrideHashes = new ImmutableMap.Builder<>(); - for (Entry entry : overrides.entrySet()) { - if (entry.getValue() instanceof LocalPathOverride) { - ModuleFileValue moduleValue = - (ModuleFileValue) - env.getValue( - ModuleFileValue.key( - ModuleKey.create(entry.getKey(), Version.EMPTY), entry.getValue())); - if (moduleValue == null) { - return null; - } - localOverrideHashes.put(entry.getKey(), moduleValue.getModuleFileHash()); - } - } - return localOverrideHashes.buildOrThrow(); - } - - @VisibleForTesting - @Nullable - static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws InterruptedException { - ClientEnvironmentValue allowedYankedVersionsFromEnv = - (ClientEnvironmentValue) - env.getValue( - ClientEnvironmentFunction.key( - YankedVersionsUtil.BZLMOD_ALLOWED_YANKED_VERSIONS_ENV)); - if (allowedYankedVersionsFromEnv == null) { - return null; - } - - ImmutableMap moduleOverrides = - ModuleFileFunction.MODULE_OVERRIDES.get(env).entrySet().stream() - .collect( - toImmutableMap(Entry::getKey, e -> ((LocalPathOverride) e.getValue()).getPath())); - - ImmutableList yankedVersions = - ImmutableList.copyOf(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.get(env)); - Boolean ignoreDevDeps = ModuleFileFunction.IGNORE_DEV_DEPS.get(env); - String compatabilityMode = - BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.get(env).name(); - String directDepsMode = BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.get(env).name(); - - String envYanked = allowedYankedVersionsFromEnv.getValue(); - - return BzlmodFlagsAndEnvVars.create( - ModuleFileFunction.REGISTRIES.get(env), - moduleOverrides, - yankedVersions, - nullToEmpty(envYanked), - ignoreDevDeps, - directDepsMode, - compatabilityMode); - } - private static ImmutableTable getExtensionUsagesById( ImmutableMap depGraph, @@ -237,13 +110,13 @@ static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws Interrup Code.BAD_MODULE, e, "invalid label for module extension found at %s", - usage.getLocation()); + usage.getProxies().getFirst().getLocation()); } if (!moduleExtensionId.getBzlFileLabel().getRepository().isVisible()) { throw ExternalDepsException.withMessage( Code.BAD_MODULE, "invalid label for module extension found at %s: no repo visible as '@%s' here", - usage.getLocation(), + usage.getProxies().getFirst().getLocation(), moduleExtensionId.getBzlFileLabel().getRepository().getName()); } extensionUsagesTableBuilder.put(moduleExtensionId, module.getKey(), usage); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphValue.java index 1a5a9e1b0ead4b..6fb4c98bf8432c 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphValue.java @@ -118,14 +118,16 @@ public static BazelDepGraphValue createEmptyDepGraph() { */ public final RepositoryMapping getFullRepoMapping(ModuleKey key) { ImmutableMap.Builder mapping = ImmutableMap.builder(); - for (Map.Entry e : + for (Map.Entry extIdAndUsage : getExtensionUsagesTable().column(key).entrySet()) { - ModuleExtensionId extensionId = e.getKey(); - ModuleExtensionUsage usage = e.getValue(); - for (Map.Entry entry : usage.getImports().entrySet()) { - String canonicalRepoName = - getExtensionUniqueNames().get(extensionId) + "~" + entry.getValue(); - mapping.put(entry.getKey(), RepositoryName.createUnvalidated(canonicalRepoName)); + ModuleExtensionId extensionId = extIdAndUsage.getKey(); + ModuleExtensionUsage usage = extIdAndUsage.getValue(); + String repoNamePrefix = getExtensionUniqueNames().get(extensionId) + "~"; + for (ModuleExtensionUsage.Proxy proxy : usage.getProxies()) { + for (Map.Entry entry : proxy.getImports().entrySet()) { + String canonicalRepoName = repoNamePrefix + entry.getValue(); + mapping.put(entry.getKey(), RepositoryName.createUnvalidated(canonicalRepoName)); + } } } return getDepGraph() diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java index 20206b8d30ab69..f34b03ba70c062 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunction.java @@ -17,9 +17,6 @@ import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.actions.FileValue; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.LabelConstants; @@ -54,20 +51,7 @@ public class BazelLockFileFunction implements SkyFunction { private final Path rootDirectory; - private static final BzlmodFlagsAndEnvVars EMPTY_FLAGS = - BzlmodFlagsAndEnvVars.create( - ImmutableSet.of(), ImmutableMap.of(), ImmutableList.of(), "", false, "", ""); - - private static final BazelLockFileValue EMPTY_LOCKFILE = - BazelLockFileValue.builder() - .setLockFileVersion(BazelLockFileValue.LOCK_FILE_VERSION) - .setModuleFileHash("") - .setFlags(EMPTY_FLAGS) - .setLocalOverrideHashes(ImmutableMap.of()) - .setModuleDepGraph(ImmutableMap.of()) - .setModuleExtensions(ImmutableMap.of()) - .setRegistryFileHashes(ImmutableMap.of()) - .build(); + private static final BazelLockFileValue EMPTY_LOCKFILE = BazelLockFileValue.builder().build(); public BazelLockFileFunction(Path rootDirectory) { this.rootDirectory = rootDirectory; @@ -86,11 +70,11 @@ public SkyValue compute(SkyKey skyKey, Environment env) } try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, "parse lockfile")) { - return getLockfileValue(lockfilePath, rootDirectory); + return getLockfileValue(lockfilePath, LOCKFILE_MODE.get(env)); } catch (IOException | JsonSyntaxException | NullPointerException e) { throw new BazelLockfileFunctionException( ExternalDepsException.withMessage( - Code.BAD_MODULE, + Code.BAD_LOCKFILE, "Failed to read and parse the MODULE.bazel.lock file with error: %s." + " Try deleting it and rerun the build.", e.getMessage()), @@ -98,31 +82,31 @@ public SkyValue compute(SkyKey skyKey, Environment env) } } - public static BazelLockFileValue getLockfileValue(RootedPath lockfilePath, Path rootDirectory) - throws IOException { - BazelLockFileValue bazelLockFileValue; + public static BazelLockFileValue getLockfileValue( + RootedPath lockfilePath, LockfileMode lockfileMode) + throws IOException, BazelLockfileFunctionException { try { String json = FileSystemUtils.readContent(lockfilePath.asPath(), UTF_8); Matcher matcher = LOCKFILE_VERSION_PATTERN.matcher(json); int version = matcher.find() ? Integer.parseInt(matcher.group(1)) : -1; if (version == BazelLockFileValue.LOCK_FILE_VERSION) { - bazelLockFileValue = - GsonTypeAdapterUtil.createLockFileGson( - lockfilePath - .asPath() - .getParentDirectory() - .getRelative(LabelConstants.MODULE_DOT_BAZEL_FILE_NAME), - rootDirectory) - .fromJson(json, BazelLockFileValue.class); + return GsonTypeAdapterUtil.LOCKFILE_GSON.fromJson(json, BazelLockFileValue.class); } else { - // This is an old version, needs to be updated - // Keep old version to recognize the problem in error mode - bazelLockFileValue = EMPTY_LOCKFILE.toBuilder().setLockFileVersion(version).build(); + // This is an old version, its information can't be used. + if (lockfileMode == LockfileMode.ERROR) { + throw new BazelLockfileFunctionException( + ExternalDepsException.withMessage( + Code.BAD_LOCKFILE, + "The version of MODULE.bazel.lock is not supported by this version of Bazel." + + " Please run `bazel mod deps --lockfile_mode=update` to update your" + + " lockfile."), + Transience.PERSISTENT); + } + return EMPTY_LOCKFILE; } } catch (FileNotFoundException e) { - bazelLockFileValue = EMPTY_LOCKFILE; + return EMPTY_LOCKFILE; } - return bazelLockFileValue; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java index c4bf267a990237..39573a5383f05a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileModule.java @@ -14,27 +14,26 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import static com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction.LOCKFILE_MODE; import static java.nio.charset.StandardCharsets.UTF_8; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedMap; -import com.google.common.eventbus.Subscribe; import com.google.common.flogger.GoogleLogger; -import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; -import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.runtime.BlazeModule; import com.google.devtools.build.lib.runtime.CommandEnvironment; +import com.google.devtools.build.lib.skyframe.PrecomputedValue; import com.google.devtools.build.lib.skyframe.SkyframeExecutor; import com.google.devtools.build.lib.vfs.FileSystemUtils; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.Root; import com.google.devtools.build.lib.vfs.RootedPath; +import com.google.devtools.build.skyframe.MemoizingEvaluator; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import javax.annotation.Nullable; @@ -46,8 +45,6 @@ public class BazelLockFileModule extends BlazeModule { private SkyframeExecutor executor; private Path workspaceRoot; - private boolean enabled; - @Nullable private BazelModuleResolutionEvent moduleResolutionEvent; private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); @@ -55,42 +52,40 @@ public class BazelLockFileModule extends BlazeModule { public void beforeCommand(CommandEnvironment env) { executor = env.getSkyframeExecutor(); workspaceRoot = env.getWorkspace(); - - enabled = - env.getOptions().getOptions(RepositoryOptions.class).lockfileMode == LockfileMode.UPDATE; - moduleResolutionEvent = null; - env.getEventBus().register(this); } @Override public void afterCommand() { - if (!enabled || moduleResolutionEvent == null) { - // Command does not use Bazel modules or the lockfile mode is not update. - // Since Skyframe caches events, they are replayed even when nothing has changed. - return; - } - - BazelDepGraphValue depGraphValue; + MemoizingEvaluator evaluator = executor.getEvaluator(); BazelModuleResolutionValue moduleResolutionValue; + BazelDepGraphValue depGraphValue; + BazelLockFileValue oldLockfile; try { - depGraphValue = - (BazelDepGraphValue) executor.getEvaluator().getExistingValue(BazelDepGraphValue.KEY); + PrecomputedValue lockfileModeValue = + (PrecomputedValue) evaluator.getExistingValue(LOCKFILE_MODE.getKeyForTesting()); + if (lockfileModeValue == null) { + // No command run on this server has triggered module resolution yet. + return; + } + LockfileMode lockfileMode = (LockfileMode) lockfileModeValue.get(); + // Check the Skyframe value instead of the option since some commands (e.g. shutdown) don't + // propagate the options to Skyframe, but we can only operate on Skyframe values that were + // generated in UPDATE mode. + if (lockfileMode != LockfileMode.UPDATE) { + return; + } moduleResolutionValue = - (BazelModuleResolutionValue) - executor.getEvaluator().getExistingValue(BazelModuleResolutionValue.KEY); + (BazelModuleResolutionValue) evaluator.getExistingValue(BazelModuleResolutionValue.KEY); + depGraphValue = (BazelDepGraphValue) evaluator.getExistingValue(BazelDepGraphValue.KEY); + oldLockfile = (BazelLockFileValue) evaluator.getExistingValue(BazelLockFileValue.KEY); } catch (InterruptedException e) { // Not thrown in Bazel. throw new IllegalStateException(e); } - - BazelLockFileValue oldLockfile = moduleResolutionEvent.getOnDiskLockfileValue(); - ImmutableMap> fileHashes; - if (moduleResolutionValue == null) { - // BazelDepGraphFunction took the dep graph from the lockfile and didn't cause evaluation of - // BazelModuleResolutionFunction. The file hashes in the lockfile are still up-to-date. - fileHashes = oldLockfile.getRegistryFileHashes(); - } else { - fileHashes = ImmutableSortedMap.copyOf(moduleResolutionValue.getRegistryFileHashes()); + if (moduleResolutionValue == null || depGraphValue == null || oldLockfile == null) { + // An error during the actual build prevented the evaluation of these values and has already + // been reported at this point. + return; } // All nodes corresponding to module extensions that have been evaluated in the current build @@ -121,8 +116,10 @@ public void afterCommand() { // Create an updated version of the lockfile, keeping only the extension results from the old // lockfile that are still up-to-date and adding the newly resolved extension results. BazelLockFileValue newLockfile = - moduleResolutionEvent.getResolutionOnlyLockfileValue().toBuilder() - .setRegistryFileHashes(fileHashes) + BazelLockFileValue.builder() + .setRegistryFileHashes( + ImmutableSortedMap.copyOf(moduleResolutionValue.getRegistryFileHashes())) + .setSelectedYankedVersions(moduleResolutionValue.getSelectedYankedVersions()) .setModuleExtensions(combinedExtensionInfos) .build(); @@ -248,31 +245,17 @@ private boolean shouldKeepExtension( * @param workspaceRoot Root of the workspace where the lockfile is located * @param updatedLockfile The updated lockfile data to save */ - public static void updateLockfile(Path workspaceRoot, BazelLockFileValue updatedLockfile) { + private static void updateLockfile(Path workspaceRoot, BazelLockFileValue updatedLockfile) { RootedPath lockfilePath = RootedPath.toRootedPath(Root.fromPath(workspaceRoot), LabelConstants.MODULE_LOCKFILE_NAME); try { FileSystemUtils.writeContent( lockfilePath.asPath(), UTF_8, - GsonTypeAdapterUtil.createLockFileGson( - lockfilePath - .asPath() - .getParentDirectory() - .getRelative(LabelConstants.MODULE_DOT_BAZEL_FILE_NAME), - workspaceRoot) - .toJson(updatedLockfile) - + "\n"); + GsonTypeAdapterUtil.LOCKFILE_GSON.toJson(updatedLockfile) + "\n"); } catch (IOException e) { logger.atSevere().withCause(e).log( "Error while updating MODULE.bazel.lock file: %s", e.getMessage()); } } - - @Subscribe - public void bazelModuleResolved(BazelModuleResolutionEvent moduleResolutionEvent) { - // Latest event wins, which is relevant in the case of `bazel mod tidy`, where a new event is - // sent after the command has modified the module file. - this.moduleResolutionEvent = moduleResolutionEvent; - } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java index cd3eff788a8962..441dc285eb21a2 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileValue.java @@ -15,10 +15,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; -import static com.google.devtools.build.lib.bazel.bzlmod.InterimModule.toModule; - import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; @@ -27,7 +24,6 @@ import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; -import java.util.Map; import java.util.Optional; /** @@ -39,35 +35,30 @@ @GenerateTypeAdapter public abstract class BazelLockFileValue implements SkyValue, Postable { - public static final int LOCK_FILE_VERSION = 8; + public static final int LOCK_FILE_VERSION = 10; @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_LOCK_FILE; static Builder builder() { return new AutoValue_BazelLockFileValue.Builder() .setLockFileVersion(LOCK_FILE_VERSION) - .setModuleExtensions(ImmutableMap.of()) - .setRegistryFileHashes(ImmutableMap.of()); + .setRegistryFileHashes(ImmutableMap.of()) + .setSelectedYankedVersions(ImmutableMap.of()) + .setModuleExtensions(ImmutableMap.of()); } /** Current version of the lock file */ public abstract int getLockFileVersion(); - /** Hash of the Module file */ - public abstract String getModuleFileHash(); - - /** Command line flags and environment variables that can affect the resolution */ - public abstract BzlmodFlagsAndEnvVars getFlags(); - - /** Module hash of each local path override in the root module file */ - public abstract ImmutableMap getLocalOverrideHashes(); - - /** The post-selection dep graph retrieved from the lock file. */ - public abstract ImmutableMap getModuleDepGraph(); - /** Hashes of files retrieved from registries. */ public abstract ImmutableMap> getRegistryFileHashes(); + /** + * Selected module versions that are known to be yanked (and hence must have been explicitly + * allowed by the user). + */ + public abstract ImmutableMap getSelectedYankedVersions(); + /** Mapping the extension id to the module extension data */ public abstract ImmutableMap< ModuleExtensionId, ImmutableMap> @@ -80,16 +71,10 @@ static Builder builder() { public abstract static class Builder { public abstract Builder setLockFileVersion(int value); - public abstract Builder setModuleFileHash(String value); - - public abstract Builder setFlags(BzlmodFlagsAndEnvVars value); - - public abstract Builder setLocalOverrideHashes(ImmutableMap value); - - public abstract Builder setModuleDepGraph(ImmutableMap value); - public abstract Builder setRegistryFileHashes(ImmutableMap> value); + public abstract Builder setSelectedYankedVersions(ImmutableMap value); + public abstract Builder setModuleExtensions( ImmutableMap< ModuleExtensionId, @@ -98,59 +83,4 @@ public abstract Builder setModuleExtensions( public abstract BazelLockFileValue build(); } - - /** Returns the difference between the lockfile and the current module & flags */ - public ImmutableList getModuleAndFlagsDiff( - String moduleFileHash, - ImmutableMap localOverrideHashes, - BzlmodFlagsAndEnvVars flags) { - ImmutableList.Builder moduleDiff = new ImmutableList.Builder<>(); - if (getLockFileVersion() != BazelLockFileValue.LOCK_FILE_VERSION) { - return moduleDiff - .add("the version of the lockfile is not compatible with the current Bazel") - .build(); - } - if (!moduleFileHash.equals(getModuleFileHash())) { - moduleDiff.add("the root MODULE.bazel has been modified"); - } - moduleDiff.addAll(getFlags().getDiffFlags(flags)); - - for (Map.Entry entry : localOverrideHashes.entrySet()) { - String currentValue = entry.getValue(); - String lockfileValue = getLocalOverrideHashes().get(entry.getKey()); - // If the lockfile value is null, the module hash would be different anyway - if (lockfileValue != null && !currentValue.equals(lockfileValue)) { - moduleDiff.add( - "The MODULE.bazel file has changed for the overriden module: " + entry.getKey()); - } - } - return moduleDiff.build(); - } - - /** - * Returns a new BazelLockFileValue in which all information about the root module has been - * replaced by the given value. - * - *

This operation is shallow: If the new root module has different dependencies, the dep graph - * will not be updated. - */ - public BazelLockFileValue withShallowlyReplacedRootModule( - ModuleFileValue.RootModuleFileValue value) { - ImmutableMap.Builder newDepGraph = ImmutableMap.builder(); - newDepGraph.putAll(getModuleDepGraph()); - newDepGraph.put( - ModuleKey.ROOT, - toModule( - value - .getModule() - .withDepSpecsTransformed( - InterimModule.applyOverrides( - value.getOverrides(), value.getModule().getName())), - /* override= */ null, - /* remoteRepoSpec= */ null)); - return toBuilder() - .setModuleFileHash(value.getModuleFileHash()) - .setModuleDepGraph(newDepGraph.buildKeepingLast()) - .build(); - } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyFunction.java index 89cd8b4d708a1d..6545dce9fb2586 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyFunction.java @@ -16,10 +16,6 @@ package com.google.devtools.build.lib.bazel.bzlmod; import static com.google.common.collect.ImmutableSet.toImmutableSet; -import static com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction.LOCKFILE_MODE; -import static com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction.IGNORE_DEV_DEPS; -import static com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction.MODULE_OVERRIDES; -import static com.google.devtools.build.lib.skyframe.PrecomputedValue.STARLARK_SEMANTICS; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -114,10 +110,6 @@ public SkyValue compute(SkyKey skyKey, Environment env) return BazelModTidyValue.create( fixups.build(), buildozer.asPath(), - rootModuleFileValue.getIncludeLabelToCompiledModuleFile(), - MODULE_OVERRIDES.get(env), - IGNORE_DEV_DEPS.get(env), - LOCKFILE_MODE.get(env), - STARLARK_SEMANTICS.get(env)); + rootModuleFileValue.getIncludeLabelToCompiledModuleFile()); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyValue.java index 492d47c0eefa81..d476c9481c6f00 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModTidyValue.java @@ -18,15 +18,12 @@ import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.skyframe.SkyFunctions; import com.google.devtools.build.lib.skyframe.serialization.autocodec.SerializationConstant; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import java.util.List; -import java.util.Map; -import net.starlark.java.eval.StarlarkSemantics; /** All Skyframe information required for the {@code bazel mod tidy} command. */ @AutoValue @@ -42,36 +39,11 @@ public abstract class BazelModTidyValue implements SkyValue { public abstract ImmutableMap includeLabelToCompiledModuleFile(); - /** The value of {@link ModuleFileFunction#MODULE_OVERRIDES}. */ - public abstract ImmutableMap moduleOverrides(); - - /** The value of {@link ModuleFileFunction#IGNORE_DEV_DEPS}. */ - public abstract boolean ignoreDevDeps(); - - /** The value of {@link BazelLockFileFunction#LOCKFILE_MODE}. */ - public abstract LockfileMode lockfileMode(); - - /** - * The value of {@link - * com.google.devtools.build.lib.skyframe.PrecomputedValue#STARLARK_SEMANTICS}. - */ - public abstract StarlarkSemantics starlarkSemantics(); - static BazelModTidyValue create( List fixups, Path buildozer, - ImmutableMap includeLabelToCompiledModuleFile, - Map moduleOverrides, - boolean ignoreDevDeps, - LockfileMode lockfileMode, - StarlarkSemantics starlarkSemantics) { + ImmutableMap includeLabelToCompiledModuleFile) { return new AutoValue_BazelModTidyValue( - ImmutableList.copyOf(fixups), - buildozer, - includeLabelToCompiledModuleFile, - ImmutableMap.copyOf(moduleOverrides), - ignoreDevDeps, - lockfileMode, - starlarkSemantics); + ImmutableList.copyOf(fixups), buildozer, includeLabelToCompiledModuleFile); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java deleted file mode 100644 index 867eec90cac0b2..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionEvent.java +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright 2023 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.bazel.bzlmod; - -import com.google.common.collect.ImmutableTable; -import com.google.devtools.build.lib.events.ExtendedEventHandler.Postable; - -/** - * After resolving bazel module this event is sent from {@link BazelDepGraphFunction} holding the - * lockfile value with module updates, and the module extension usages. It will be received in - * {@link BazelLockFileModule} to be used to update the lockfile content - * - *

Instances of this class are retained in Skyframe nodes and subject to frequent {@link - * java.util.Set}-based deduplication. As such, it must have a cheap implementation of {@link - * #hashCode()} and {@link #equals(Object)}. It currently uses reference equality since the logic - * that creates instances of this class already ensures that there is only one instance per build. - */ -public final class BazelModuleResolutionEvent implements Postable { - - private final BazelLockFileValue onDiskLockfileValue; - private final BazelLockFileValue resolutionOnlyLockfileValue; - private final ImmutableTable - extensionUsagesById; - - private BazelModuleResolutionEvent( - BazelLockFileValue onDiskLockfileValue, - BazelLockFileValue resolutionOnlyLockfileValue, - ImmutableTable extensionUsagesById) { - this.onDiskLockfileValue = onDiskLockfileValue; - this.resolutionOnlyLockfileValue = resolutionOnlyLockfileValue; - this.extensionUsagesById = extensionUsagesById; - } - - public static BazelModuleResolutionEvent create( - BazelLockFileValue onDiskLockfileValue, - BazelLockFileValue resolutionOnlyLockfileValue, - ImmutableTable extensionUsagesById) { - return new BazelModuleResolutionEvent( - onDiskLockfileValue, resolutionOnlyLockfileValue, extensionUsagesById); - } - - /** Returns the contents of the lockfile as it existed on disk before the current build. */ - public BazelLockFileValue getOnDiskLockfileValue() { - return onDiskLockfileValue; - } - - /** - * Returns the result of Bazel module resolution in the form of a lockfile without any information - * about module extension results. - */ - public BazelLockFileValue getResolutionOnlyLockfileValue() { - return resolutionOnlyLockfileValue; - } - - public ImmutableTable - getExtensionUsagesById() { - return extensionUsagesById; - } - - @Override - public boolean storeForReplay() { - return true; - } -} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java index 4e3fb5d60f8bfb..c320a2d241f5de 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunction.java @@ -15,7 +15,6 @@ package com.google.devtools.build.lib.bazel.bzlmod; -import static com.google.common.collect.ImmutableMap.toImmutableMap; import static com.google.common.collect.ImmutableSet.toImmutableSet; import static com.google.devtools.build.lib.bazel.bzlmod.YankedVersionsUtil.BZLMOD_ALLOWED_YANKED_VERSIONS_ENV; @@ -47,6 +46,7 @@ import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.build.skyframe.SkyframeLookupResult; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; @@ -68,7 +68,8 @@ public class BazelModuleResolutionFunction implements SkyFunction { private record Result( Selection.Result selectionResult, - ImmutableMap> registryFileHashes) {} + ImmutableMap> registryFileHashes, + ImmutableMap selectedYankedVersions) {} private static class ModuleResolutionComputeState implements Environment.SkyKeyComputeState { Result discoverAndSelectResult; @@ -142,7 +143,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) return BazelModuleResolutionValue.create( finalDepGraph, state.discoverAndSelectResult.selectionResult.getUnprunedDepGraph(), - ImmutableMap.copyOf(registryFileHashes)); + ImmutableMap.copyOf(registryFileHashes), + state.discoverAndSelectResult.selectedYankedVersions); } @Nullable @@ -171,21 +173,12 @@ private static Result discoverAndSelect( } ImmutableMap resolvedDepGraph = selectionResult.getResolvedDepGraph(); - var yankedVersionsKeys = - resolvedDepGraph.values().stream() - .filter(m -> m.getRegistry() != null) - .map(m -> YankedVersionsValue.Key.create(m.getName(), m.getRegistry().getUrl())) - .collect(toImmutableSet()); - SkyframeLookupResult yankedVersionsResult = env.getValuesAndExceptions(yankedVersionsKeys); - if (env.valuesMissing()) { - return null; + ImmutableMap yankedVersionsValues; + try (SilentCloseable c = + Profiler.instance().profile(ProfilerTask.BZLMOD, "collect yanked versions")) { + yankedVersionsValues = collectYankedVersionsValues(env, resolvedDepGraph.values()); } - var yankedVersionValues = - yankedVersionsKeys.stream() - .collect( - toImmutableMap( - key -> key, key -> (YankedVersionsValue) yankedVersionsResult.get(key))); - if (yankedVersionValues.values().stream().anyMatch(Objects::isNull)) { + if (yankedVersionsValues == null) { return null; } @@ -206,12 +199,46 @@ private static Result discoverAndSelect( env.getListener()); } - try (SilentCloseable c = - Profiler.instance().profile(ProfilerTask.BZLMOD, "check no yanked versions")) { - checkNoYankedVersions(resolvedDepGraph, yankedVersionValues, allowedYankedVersions); - } + var selectedYankedVersions = checkNoYankedVersions(yankedVersionsValues, allowedYankedVersions); + return new Result( + selectionResult, discoveryResult.registryFileHashes(), selectedYankedVersions); + } - return new Result(selectionResult, discoveryResult.registryFileHashes()); + @Nullable + private static ImmutableMap collectYankedVersionsValues( + Environment env, ImmutableCollection modules) throws InterruptedException { + ImmutableMap.Builder yankedVersionsValues = + ImmutableMap.builder(); + Map yankedVersionsKeys = new HashMap<>(); + for (InterimModule m : modules) { + if (m.getRegistry() == null) { + // Modules with a non-registry override are never yanked. + yankedVersionsValues.put(m.getKey(), YankedVersionsValue.NONE_YANKED); + continue; + } + var lockfileYankedVersionsValue = + m.getRegistry().tryGetYankedVersionsFromLockfile(m.getKey()); + if (lockfileYankedVersionsValue.isPresent()) { + yankedVersionsValues.put(m.getKey(), lockfileYankedVersionsValue.get()); + } else { + // We need to download the list of yanked versions from the registry. + yankedVersionsKeys.put( + m.getKey(), YankedVersionsValue.Key.create(m.getName(), m.getRegistry().getUrl())); + } + } + SkyframeLookupResult yankedVersionsResult = + env.getValuesAndExceptions(yankedVersionsKeys.values()); + if (env.valuesMissing()) { + return null; + } + for (var entry : yankedVersionsKeys.entrySet()) { + var yankedVersionsValue = (YankedVersionsValue) yankedVersionsResult.get(entry.getValue()); + if (yankedVersionsValue == null) { + return null; + } + yankedVersionsValues.put(entry.getKey(), yankedVersionsValue); + } + return yankedVersionsValues.buildOrThrow(); } private static void verifyAllOverridesAreOnExistentModules( @@ -304,36 +331,46 @@ public static void checkBazelCompatibility( } } - private static void checkNoYankedVersions( - ImmutableMap depGraph, - ImmutableMap yankedVersionValues, + /** + * Fail if any selected module is yanked and not explicitly allowed. + * + * @return the yanked info for each yanked but explicitly allowed module + */ + private static ImmutableMap checkNoYankedVersions( + ImmutableMap yankedVersionValues, Optional> allowedYankedVersions) throws BazelModuleResolutionFunctionException { - for (InterimModule m : depGraph.values()) { - if (m.getRegistry() == null) { - // Non-registry modules do not have yanked versions. + ImmutableMap.Builder selectedYankedVersions = ImmutableMap.builder(); + for (var entry : yankedVersionValues.entrySet()) { + ModuleKey key = entry.getKey(); + YankedVersionsValue yankedVersionsValue = entry.getValue(); + if (yankedVersionsValue.yankedVersions().isEmpty()) { + // No yanked version information available for this module. continue; } - Optional yankedInfo = - YankedVersionsUtil.getYankedInfo( - m.getKey(), - yankedVersionValues.get( - YankedVersionsValue.Key.create(m.getName(), m.getRegistry().getUrl())), - allowedYankedVersions); - if (yankedInfo.isPresent()) { - throw new BazelModuleResolutionFunctionException( - ExternalDepsException.withMessage( - Code.VERSION_RESOLUTION_ERROR, - "Yanked version detected in your resolved dependency graph: %s, for the reason: " - + "%s.\nYanked versions may contain serious vulnerabilities and should not be " - + "used. To fix this, use a bazel_dep on a newer version of this module. To " - + "continue using this version, allow it using the --allow_yanked_versions " - + "flag or the BZLMOD_ALLOW_YANKED_VERSIONS env variable.", - m.getKey(), - yankedInfo.get()), - Transience.PERSISTENT); + String yankedInfo = yankedVersionsValue.yankedVersions().get().get(key.getVersion()); + if (yankedInfo == null) { + // The selected version is not yanked. + continue; + } + if (allowedYankedVersions.isEmpty() || allowedYankedVersions.get().contains(key)) { + // The selected version is yanked but explicitly allowed. + selectedYankedVersions.put(key, yankedInfo); + continue; } + throw new BazelModuleResolutionFunctionException( + ExternalDepsException.withMessage( + Code.VERSION_RESOLUTION_ERROR, + "Yanked version detected in your resolved dependency graph: %s, for the reason: " + + "%s.\nYanked versions may contain serious vulnerabilities and should not be " + + "used. To fix this, use a bazel_dep on a newer version of this module. To " + + "continue using this version, allow it using the --allow_yanked_versions " + + "flag or the BZLMOD_ALLOW_YANKED_VERSIONS env variable.", + key, + yankedInfo), + Transience.PERSISTENT); } + return selectedYankedVersions.buildOrThrow(); } private static ImmutableMap computeFinalDepGraph( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java index 60971030d549b0..5b1d45e5abd450 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionValue.java @@ -30,9 +30,6 @@ */ @AutoValue abstract class BazelModuleResolutionValue implements SkyValue { - /* TODO(andreisolo): Also load the modules overridden by {@code single_version_override} or - NonRegistryOverride if we need to detect changes in the dependency graph caused by them. - */ @SerializationConstant public static final SkyKey KEY = () -> SkyFunctions.BAZEL_MODULE_RESOLUTION; @@ -53,11 +50,18 @@ abstract class BazelModuleResolutionValue implements SkyValue { */ abstract ImmutableMap> getRegistryFileHashes(); + /** + * Selected module versions that are known to be yanked (and hence must have been explicitly + * allowed by the user). + */ + abstract ImmutableMap getSelectedYankedVersions(); + static BazelModuleResolutionValue create( ImmutableMap resolvedDepGraph, ImmutableMap unprunedDepGraph, - ImmutableMap> registryFileHashes) { + ImmutableMap> registryFileHashes, + ImmutableMap selectedYankedVersions) { return new AutoValue_BazelModuleResolutionValue( - resolvedDepGraph, unprunedDepGraph, registryFileHashes); + resolvedDepGraph, unprunedDepGraph, registryFileHashes, selectedYankedVersions); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java deleted file mode 100644 index d425bc4518042d..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2023 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.google.devtools.build.lib.bazel.bzlmod; - -import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.ryanharter.auto.value.gson.GenerateTypeAdapter; - -/** Stores the values of flags and environment variables that affect the resolution */ -@AutoValue -@GenerateTypeAdapter -abstract class BzlmodFlagsAndEnvVars { - - public static BzlmodFlagsAndEnvVars create( - ImmutableSet registries, - ImmutableMap moduleOverrides, - ImmutableList yankedVersions, - String envVarYankedVersions, - boolean ignoreDevDeps, - String directDepsMode, - String compatabilityMode) { - return new AutoValue_BzlmodFlagsAndEnvVars( - registries, - moduleOverrides, - yankedVersions, - envVarYankedVersions, - ignoreDevDeps, - directDepsMode, - compatabilityMode); - } - - /** Registries provided via command line */ - public abstract ImmutableSet cmdRegistries(); - - /** ModulesOverride provided via command line */ - public abstract ImmutableMap cmdModuleOverrides(); - - /** Allowed yanked version in the dependency graph */ - public abstract ImmutableList allowedYankedVersions(); - - /** Allowed yanked version in the dependency graph from environment variables */ - public abstract String envVarAllowedYankedVersions(); - - /** Whether to ignore things declared as dev dependencies or not */ - public abstract boolean ignoreDevDependency(); - - /** Error level of direct dependencies check */ - public abstract String directDependenciesMode(); - - /** Error level of bazel compatability check */ - public abstract String compatibilityMode(); - - public ImmutableList getDiffFlags(BzlmodFlagsAndEnvVars flags) { - ImmutableList.Builder diffFlags = new ImmutableList.Builder<>(); - if (!flags.cmdRegistries().equals(cmdRegistries())) { - diffFlags.add("the value of --registry flag has been modified"); - } - if (!flags.cmdModuleOverrides().equals(cmdModuleOverrides())) { - diffFlags.add("the value of --override_module flag has been modified"); - } - if (!flags.allowedYankedVersions().equals(allowedYankedVersions())) { - diffFlags.add("the value of --allow_yanked_versions flag has been modified"); - } - if (!flags.envVarAllowedYankedVersions().equals(envVarAllowedYankedVersions())) { - diffFlags.add( - "the value of BZLMOD_ALLOW_YANKED_VERSIONS environment variable has been modified"); - } - if (flags.ignoreDevDependency() != ignoreDevDependency()) { - diffFlags.add("the value of --ignore_dev_dependency flag has been modified"); - } - if (!flags.directDependenciesMode().equals(directDependenciesMode())) { - diffFlags.add("the value of --check_direct_dependencies flag has been modified"); - } - if (!flags.compatibilityMode().equals(compatibilityMode())) { - diffFlags.add("the value of --check_bazel_compatibility flag has been modified"); - } - return diffFlags.build(); - } -} diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java index 6cc5eaebeda813..f42c2eec30ae56 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Discovery.java @@ -91,6 +91,14 @@ public static Result run(Environment env, RootModuleFileValue root) moduleFileValue = (ModuleFileValue) result.getOrThrow(skyKey, ExternalDepsException.class); } catch (ExternalDepsException e) { + if (e.getDetailedExitCode().getFailureDetail() == null + || e.getDetailedExitCode().getFailureDetail().getExternalDeps().getCode() + != FailureDetails.ExternalDeps.Code.BAD_MODULE) { + // This is not due to a bad module, so don't print a dependency chain. This covers cases + // such as a parse error in the lockfile or an I/O exception during registry access, + // which aren't related to any particular module dep. + throw e; + } // Trace back a dependency chain to the root module. There can be multiple paths to the // failing module, but any of those is useful for debugging. List depChain = new ArrayList<>(); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ExternalDepsException.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ExternalDepsException.java index a6ae28b7132155..0d6a862d491700 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ExternalDepsException.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ExternalDepsException.java @@ -51,6 +51,10 @@ public static ExternalDepsException withCauseAndMessage( String.format(format, args) + ": " + cause.getMessage(), cause, code); } + public static ExternalDepsException withCause(ExternalDeps.Code code, Throwable cause) { + return new ExternalDepsException(cause.getMessage(), cause, code); + } + @Override public DetailedExitCode getDetailedExitCode() { return detailedExitCode; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java index 912b455287352e..440e39a8edd502 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GsonTypeAdapterUtil.java @@ -20,7 +20,6 @@ import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_MAP; import static com.google.devtools.build.lib.bazel.bzlmod.DelegateTypeAdapterFactory.IMMUTABLE_SET; -import com.google.auto.value.AutoValue; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableTable; @@ -32,7 +31,6 @@ import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.RepositoryName; import com.google.devtools.build.lib.rules.repository.RepoRecordedInput; -import com.google.devtools.build.lib.vfs.Path; import com.google.gson.Gson; import com.google.gson.GsonBuilder; import com.google.gson.JsonParseException; @@ -49,7 +47,6 @@ import java.util.Base64; import java.util.Optional; import javax.annotation.Nullable; -import net.starlark.java.syntax.Location; /** * Utility class to hold type adapters and helper methods to get gson registered with type adapters @@ -366,87 +363,6 @@ public ImmutableTable read(JsonReader jsonReader) } }; - /** - * A variant of {@link Location} that converts the absolute path to the root module file to a - * constant and back. - */ - // protected only for @AutoValue - @GenerateTypeAdapter - @AutoValue - protected abstract static class RootModuleFileEscapingLocation { - // This marker string is neither a valid absolute path nor a valid URL and thus cannot conflict - // with any real module file location. - private static final String ROOT_MODULE_FILE_LABEL = "@@//:MODULE.bazel"; - - public abstract String file(); - - public abstract int line(); - - public abstract int column(); - - public Location toLocation(String moduleFilePath, String workspaceRoot) { - String file; - if (file().equals(ROOT_MODULE_FILE_LABEL)) { - file = moduleFilePath; - } else { - file = file().replace("%workspace%", workspaceRoot); - } - return Location.fromFileLineColumn(file, line(), column()); - } - - public static RootModuleFileEscapingLocation fromLocation( - Location location, String moduleFilePath, String workspaceRoot) { - String file; - if (location.file().equals(moduleFilePath)) { - file = ROOT_MODULE_FILE_LABEL; - } else { - file = location.file().replace(workspaceRoot, "%workspace%"); - } - return new AutoValue_GsonTypeAdapterUtil_RootModuleFileEscapingLocation( - file, location.line(), location.column()); - } - } - - private static final class LocationTypeAdapterFactory implements TypeAdapterFactory { - - private final String moduleFilePath; - private final String workspaceRoot; - - public LocationTypeAdapterFactory(Path moduleFilePath, Path workspaceRoot) { - this.moduleFilePath = moduleFilePath.getPathString(); - this.workspaceRoot = workspaceRoot.getPathString(); - } - - @Nullable - @Override - @SuppressWarnings("unchecked") - public TypeAdapter create(Gson gson, TypeToken typeToken) { - if (typeToken.getRawType() != Location.class) { - return null; - } - TypeAdapter relativizedLocationTypeAdapter = - gson.getAdapter(RootModuleFileEscapingLocation.class); - return (TypeAdapter) - new TypeAdapter() { - - @Override - public void write(JsonWriter jsonWriter, Location location) throws IOException { - relativizedLocationTypeAdapter.write( - jsonWriter, - RootModuleFileEscapingLocation.fromLocation( - location, moduleFilePath, workspaceRoot)); - } - - @Override - public Location read(JsonReader jsonReader) throws IOException { - return relativizedLocationTypeAdapter - .read(jsonReader) - .toLocation(moduleFilePath, workspaceRoot); - } - }; - } - } - private static final TypeAdapter REPO_RECORDED_INPUT_FILE_TYPE_ADAPTER = new TypeAdapter<>() { @Override @@ -528,17 +444,13 @@ public Optional read(JsonReader jsonReader) throws IOException { } } - public static Gson createLockFileGson(Path moduleFilePath, Path workspaceRoot) { - return newGsonBuilder() - .setPrettyPrinting() - .registerTypeAdapterFactory(new LocationTypeAdapterFactory(moduleFilePath, workspaceRoot)) - .registerTypeAdapterFactory(new OptionalChecksumTypeAdapterFactory()) - .create(); - } + public static final Gson LOCKFILE_GSON = + newGsonBuilder() + .setPrettyPrinting() + .registerTypeAdapterFactory(new OptionalChecksumTypeAdapterFactory()) + .create(); - public static Gson createSingleExtensionUsagesValueHashGson() { - return newGsonBuilder().create(); - } + public static final Gson SINGLE_EXTENSION_USAGES_VALUE_GSON = newGsonBuilder().create(); private static GsonBuilder newGsonBuilder() { return new GsonBuilder() diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java index 578e78a69d9d1d..9de56c594550a1 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java @@ -17,11 +17,13 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum.MissingChecksumException; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.devtools.build.lib.events.StoredEventHandler; @@ -57,17 +59,16 @@ public class IndexRegistry implements Registry { */ public enum KnownFileHashesMode { IGNORE, - USE_AND_UPDATE; + USE_AND_UPDATE, + ENFORCE } - /** The unresolved version of the url. Ex: has %workspace% placeholder */ - private final String unresolvedUri; - private final URI uri; private final DownloadManager downloadManager; private final Map clientEnv; private final Gson gson; private final ImmutableMap> knownFileHashes; + private final ImmutableMap previouslySelectedYankedVersions; private final KnownFileHashesMode knownFileHashesMode; private volatile Optional bazelRegistryJson; private volatile StoredEventHandler bazelRegistryJsonEvents; @@ -76,13 +77,12 @@ public enum KnownFileHashesMode { public IndexRegistry( URI uri, - String unresolvedUri, DownloadManager downloadManager, Map clientEnv, ImmutableMap> knownFileHashes, - KnownFileHashesMode knownFileHashesMode) { + KnownFileHashesMode knownFileHashesMode, + ImmutableMap previouslySelectedYankedVersions) { this.uri = uri; - this.unresolvedUri = unresolvedUri; this.downloadManager = downloadManager; this.clientEnv = clientEnv; this.gson = @@ -91,11 +91,12 @@ public IndexRegistry( .create(); this.knownFileHashes = knownFileHashes; this.knownFileHashesMode = knownFileHashesMode; + this.previouslySelectedYankedVersions = previouslySelectedYankedVersions; } @Override public String getUrl() { - return unresolvedUri; + return uri.toString(); } private String constructUrl(String base, String... segments) { @@ -127,6 +128,14 @@ private Optional doGrabFile( if (knownFileHashesMode != KnownFileHashesMode.IGNORE && useChecksum) { Optional knownChecksum = knownFileHashes.get(url); if (knownChecksum == null) { + if (knownFileHashesMode == KnownFileHashesMode.ENFORCE) { + throw new MissingChecksumException( + String.format( + "Missing checksum for registry file %s not permitted with --lockfile_mode=error." + + " Please run `bazel mod deps --lockfile_mode=update` to update your" + + " lockfile.", + url)); + } // This is a new file, download without providing a checksum. checksum = Optional.empty(); } else if (knownChecksum.isEmpty()) { @@ -140,10 +149,17 @@ private Optional doGrabFile( } else { checksum = Optional.empty(); } + if (knownFileHashesMode == KnownFileHashesMode.ENFORCE) { + Preconditions.checkState( + checksum.isPresent(), + "Cannot fetch a file without a checksum in ENFORCE mode. This is a bug in Bazel, please " + + "report at https://github.com/bazelbuild/bazel/issues/new/choose."); + } try (SilentCloseable c = Profiler.instance().profile(ProfilerTask.BZLMOD, () -> "download file: " + url)) { return Optional.of( - downloadManager.downloadAndReadOneUrl(new URL(url), eventHandler, clientEnv, checksum)); + downloadManager.downloadAndReadOneUrlForBzlmod( + new URL(url), eventHandler, clientEnv, checksum)); } catch (FileNotFoundException e) { return Optional.empty(); } catch (IOException e) { @@ -158,7 +174,7 @@ public Optional getModuleFile(ModuleKey key, ExtendedEventHandler ev throws IOException, InterruptedException { String url = constructUrl( - uri.toString(), "modules", key.getName(), key.getVersion().toString(), "MODULE.bazel"); + getUrl(), "modules", key.getName(), key.getVersion().toString(), "MODULE.bazel"); Optional maybeContent = grabFile(url, eventHandler, /* useChecksum= */ true); return maybeContent.map(content -> ModuleFile.create(content, url)); } @@ -237,17 +253,12 @@ private T parseJson(String jsonString, String url, Class klass) throws IO @Override public RepoSpec getRepoSpec(ModuleKey key, ExtendedEventHandler eventHandler) throws IOException, InterruptedException { - String jsonUrl = - constructUrl( - uri.toString(), - "modules", - key.getName(), - key.getVersion().toString(), - SOURCE_JSON_FILENAME); + String jsonUrl = getSourceJsonUrl(key); Optional jsonString = grabJsonFile(jsonUrl, eventHandler, /* useChecksum= */ true); if (jsonString.isEmpty()) { throw new FileNotFoundException( - String.format("Module %s's %s not found in registry %s", key, SOURCE_JSON_FILENAME, uri)); + String.format( + "Module %s's %s not found in registry %s", key, SOURCE_JSON_FILENAME, getUrl())); } SourceJson sourceJson = parseJson(jsonString.get(), jsonUrl, SourceJson.class); switch (sourceJson.type) { @@ -275,7 +286,11 @@ public RepoSpec getRepoSpec(ModuleKey key, ExtendedEventHandler eventHandler) } } - @SuppressWarnings("OptionalAssignedToNull") + private String getSourceJsonUrl(ModuleKey key) { + return constructUrl( + getUrl(), "modules", key.getName(), key.getVersion().toString(), SOURCE_JSON_FILENAME); + } + private Optional getBazelRegistryJson(ExtendedEventHandler eventHandler) throws IOException, InterruptedException { if (bazelRegistryJson == null) { @@ -284,7 +299,7 @@ private Optional getBazelRegistryJson(ExtendedEventHandler ev var storedEventHandler = new StoredEventHandler(); bazelRegistryJson = grabJson( - constructUrl(uri.toString(), "bazel_registry.json"), + constructUrl(getUrl(), "bazel_registry.json"), BazelRegistryJson.class, storedEventHandler, /* useChecksum= */ true); @@ -361,7 +376,7 @@ private RepoSpec createArchiveRepoSpec( for (Map.Entry entry : sourceJson.patches.entrySet()) { remotePatches.put( constructUrl( - unresolvedUri, + getUrl(), "modules", key.getName(), key.getVersion().toString(), @@ -399,7 +414,7 @@ public Optional> getYankedVersions( throws IOException, InterruptedException { Optional metadataJson = grabJson( - constructUrl(uri.toString(), "modules", moduleName, "metadata.json"), + constructUrl(getUrl(), "modules", moduleName, "metadata.json"), MetadataJson.class, eventHandler, // metadata.json is not immutable @@ -423,6 +438,40 @@ public Optional> getYankedVersions( } } + @Override + public Optional tryGetYankedVersionsFromLockfile( + ModuleKey selectedModuleKey) { + String yankedInfo = previouslySelectedYankedVersions.get(selectedModuleKey); + if (yankedInfo != null) { + // The module version was selected when the lockfile was created, but known to be yanked + // (hence, it was explicitly allowed by the user). We reuse the yanked info from the lockfile. + // Rationale: A module that was yanked in the past should remain yanked in the future. The + // yanked info may have been updated since then, but by not fetching it, we avoid network + // access if the set of yanked versions has not changed, but the set allowed versions has. + return Optional.of( + YankedVersionsValue.create( + Optional.of(ImmutableMap.of(selectedModuleKey.getVersion(), yankedInfo)))); + } + if (knownFileHashes.containsKey(getSourceJsonUrl(selectedModuleKey))) { + // If the source.json hash is recorded in the lockfile, we know that the module was selected + // when the lockfile was created. Since it does not appear in the list of selected yanked + // versions recorded in the lockfile, it must not have been yanked at that time. We do not + // refresh yanked versions information. + // Rationale: This ensures that builds with --lockfile_mode=update or error are reproducible + // and do not fail due to changes in the set of yanked versions. Furthermore, it avoids + // refetching yanked versions for all modules every time the user modifies or adds a + // dependency. If the selected version for a module changes, yanked version information is + // always refreshed. + return Optional.of(YankedVersionsValue.NONE_YANKED); + } + // The lockfile does not contain sufficient information to determine the "yanked" status of the + // module - network access to the registry is required. + // Note that this point can't (and must not) be reached with --lockfile_mode=error: The lockfile + // records the source.json hashes of all selected modules and the result of selection is fully + // determined by the lockfile. + return Optional.empty(); + } + /** Represents fields available in {@code metadata.json} for each module. */ static class MetadataJson { // There are other attributes in the metadata.json file, but for now, we only care about diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java index ebc1c05324672b..454c1ec8a81e38 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionMetadata.java @@ -41,7 +41,6 @@ import net.starlark.java.eval.Starlark; import net.starlark.java.eval.StarlarkList; import net.starlark.java.eval.StarlarkValue; -import net.starlark.java.syntax.Location; /** The Starlark object passed to the implementation function of module extension metadata. */ @StarlarkBuiltin( @@ -219,15 +218,19 @@ private static Optional generateFixup( Set expectedImports, Set expectedDevImports, EventHandler eventHandler) { - var actualDevImports = ImmutableSet.copyOf(rootUsage.getDevImports()); + var actualDevImports = + rootUsage.getProxies().stream() + .filter(p -> p.isDevDependency()) + .flatMap(p -> p.getImports().values().stream()) + .collect(toImmutableSet()); var actualImports = - rootUsage.getImports().values().stream() - .filter(repo -> !actualDevImports.contains(repo)) + rootUsage.getProxies().stream() + .filter(p -> !p.isDevDependency()) + .flatMap(p -> p.getImports().values().stream()) .collect(toImmutableSet()); String extensionBzlFile = rootUsage.getExtensionBzlFile(); String extensionName = rootUsage.getExtensionName(); - Location location = rootUsage.getLocation(); var importsToAdd = ImmutableSortedSet.copyOf(Sets.difference(expectedImports, actualImports)); var importsToRemove = @@ -341,7 +344,7 @@ private static Optional generateFixup( .flatMap(Optional::stream) .collect(toImmutableList()); - eventHandler.handle(Event.warn(location, message)); + eventHandler.handle(Event.warn(rootUsage.getProxies().getFirst().getLocation(), message)); return Optional.of(new RootModuleFileFixup(buildozerCommands, rootUsage)); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java index 31657e5dee765d..2dbfd1620e353b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionUsage.java @@ -17,17 +17,17 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.ryanharter.auto.value.gson.GenerateTypeAdapter; import java.util.Optional; import net.starlark.java.syntax.Location; /** - * Represents one usage of a module extension in one MODULE.bazel file. This class records all the - * information pertinent to the proxy object returned from the {@code use_extension} call. + * Represents the usage of a module extension in one module. This class records all the information + * pertinent to all the proxy objects returned from any {@code use_extension} calls in this module + * that refer to the same extension (or isolate, when applicable). * *

When adding new fields, make sure to update {@link #trimForEvaluation()} as well. */ @@ -49,40 +49,80 @@ public abstract class ModuleExtensionUsage { /** The module that contains this particular extension usage. */ public abstract ModuleKey getUsingModule(); - /** - * The location where this proxy object was created (by the {@code use_extension} call). Note that - * if there were multiple {@code use_extension} calls on same extension, then this only stores the - * location of the first one. - */ - public abstract Location getLocation(); + /** Represents one "proxy object" returned from one {@code use_extension} call. */ + @AutoValue + @GenerateTypeAdapter + public abstract static class Proxy { + /** The location of the {@code use_extension} call. */ + public abstract Location getLocation(); + + /** + * The name of the proxy object; as in, the name that the return value of {@code use_extension} + * is bound to. Is the empty string if the return value is not bound to any name (e.g. {@code + * use_repo(use_extension(...))}). + */ + public abstract String getProxyName(); + + /** Whether {@code dev_dependency} is set to true. */ + public abstract boolean isDevDependency(); + + /** + * All the repos imported, through this proxy, from this module extension into the scope of the + * current module. The key is the local repo name (in the scope of the current module), and the + * value is the name exported by the module extension. + */ + public abstract ImmutableMap getImports(); + + public static Builder builder() { + return new AutoValue_ModuleExtensionUsage_Proxy.Builder().setProxyName(""); + } - /** - * All the repos imported from this module extension into the scope of the current module. The key - * is the local repo name (in the scope of the current module), and the value is the name exported - * by the module extension. - */ - public abstract ImmutableBiMap getImports(); + /** Builder for {@link ModuleExtensionUsage.Proxy}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setLocation(Location value); - /** - * The repo names as exported by the module extension that were imported using a proxy marked as a - * dev dependency. - */ - public abstract ImmutableSet getDevImports(); + public abstract String getProxyName(); + + public abstract Builder setProxyName(String value); + + public abstract boolean isDevDependency(); + + public abstract Builder setDevDependency(boolean value); + + abstract ImmutableMap.Builder importsBuilder(); + + @CanIgnoreReturnValue + public final Builder addImport(String key, String value) { + importsBuilder().put(key, value); + return this; + } + + public abstract Builder setImports(ImmutableMap value); + + public abstract Proxy build(); + } + } + + /** The list of proxy objects that constitute */ + public abstract ImmutableList getProxies(); /** All the tags specified by this module for this extension. */ public abstract ImmutableList getTags(); /** - * Whether any use_extension calls for this usage had dev_dependency = True - * set.* + * Whether any {@code use_extension} calls for this usage had {@code dev_dependency = True} set. */ - public abstract boolean getHasDevUseExtension(); + public final boolean getHasDevUseExtension() { + return getProxies().stream().anyMatch(p -> p.isDevDependency()); + } /** - * Whether any use_extension calls for this usage had dev_dependency = False - * set.* + * Whether any {@code use_extension} calls for this usage had {@code dev_dependency = False} set. */ - public abstract boolean getHasNonDevUseExtension(); + public final boolean getHasNonDevUseExtension() { + return getProxies().stream().anyMatch(p -> !p.isDevDependency()); + } public abstract Builder toBuilder(); @@ -100,13 +140,12 @@ ModuleExtensionUsage trimForEvaluation() { // preserves correctness in case new fields are added without updating this code. return toBuilder() .setTags(getTags().stream().map(Tag::trimForEvaluation).collect(toImmutableList())) + // Clear out all proxies as information contained therein isn't useful for evaluation. // Locations are only used for error reporting and thus don't influence whether the // evaluation of the extension is successful and what its result is in case of success. - .setLocation(Location.BUILTIN) // Extension implementation functions do not see the imports, they are only validated // against the set of generated repos in a validation step that comes afterward. - .setImports(ImmutableBiMap.of()) - .setDevImports(ImmutableSet.of()) + .setProxies(ImmutableList.of()) .build(); } @@ -122,26 +161,26 @@ public abstract static class Builder { public abstract Builder setUsingModule(ModuleKey value); - public abstract Builder setLocation(Location value); + public abstract Builder setProxies(ImmutableList value); - public abstract Builder setImports(ImmutableBiMap value); + abstract ImmutableList.Builder proxiesBuilder(); - public abstract Builder setDevImports(ImmutableSet value); + @CanIgnoreReturnValue + public Builder addProxy(Proxy value) { + proxiesBuilder().add(value); + return this; + } public abstract Builder setTags(ImmutableList value); abstract ImmutableList.Builder tagsBuilder(); @CanIgnoreReturnValue - public ModuleExtensionUsage.Builder addTag(Tag value) { + public Builder addTag(Tag value) { tagsBuilder().add(value); return this; } - public abstract Builder setHasDevUseExtension(boolean hasDevUseExtension); - - public abstract Builder setHasNonDevUseExtension(boolean hasNonDevUseExtension); - public abstract ModuleExtensionUsage build(); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java index 90cf0e9ce1d614..60f09174bd7d2f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunction.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.bazel.bzlmod.CompiledModuleFile.IncludeStatement; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.NonRootModuleFileValue; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; +import com.google.devtools.build.lib.bazel.repository.downloader.Checksum.MissingChecksumException; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.cmdline.LabelConstants; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; @@ -439,13 +440,14 @@ public static RootModuleFileValue evaluateRootModuleFile( throw errorf(Code.BAD_MODULE, "error executing MODULE.bazel file for the root module"); } for (ModuleExtensionUsage usage : module.getExtensionUsages()) { - if (usage.getIsolationKey().isPresent() && usage.getImports().isEmpty()) { + ModuleExtensionUsage.Proxy firstProxy = usage.getProxies().getFirst(); + if (usage.getIsolationKey().isPresent() && firstProxy.getImports().isEmpty()) { throw errorf( Code.BAD_MODULE, "the isolated usage at %s of extension %s defined in %s has no effect as no " + "repositories are imported from it. Either import one or more repositories " + "generated by the extension with use_repo or remove the usage.", - usage.getLocation(), + firstProxy.getLocation(), usage.getExtensionName(), usage.getExtensionBzlFile()); } @@ -616,6 +618,9 @@ private GetModuleFileResult getModuleFile( continue; } return new GetModuleFileResult(moduleFile.get(), registry, downloadEventHandler); + } catch (MissingChecksumException e) { + throw new ModuleFileFunctionException( + ExternalDepsException.withCause(Code.BAD_LOCKFILE, e)); } catch (IOException e) { throw errorf( Code.ERROR_ACCESSING_REGISTRY, e, "Error accessing registry %s", registry.getUrl()); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java index ae0eae0b0a1995..fcf2103bd17dd6 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileGlobals.java @@ -428,14 +428,18 @@ public ModuleExtensionProxy useExtension( "innate extensions cannot be directly used; try `use_repo_rule` instead"); } + var proxyBuilder = + ModuleExtensionUsage.Proxy.builder() + .setLocation(thread.getCallerLocation()) + .setDevDependency(devDependency); + String extensionBzlFile = normalizeLabelString(context.getModuleBuilder(), rawExtensionBzlFile); - ModuleExtensionUsageBuilder newUsageBuilder = - new ModuleExtensionUsageBuilder( - context, extensionBzlFile, extensionName, isolate, thread.getCallerLocation()); + var newUsageBuilder = + new ModuleExtensionUsageBuilder(context, extensionBzlFile, extensionName, isolate); if (context.shouldIgnoreDevDeps() && devDependency) { // This is a no-op proxy. - return ModuleExtensionProxy.createFromUsagesBuilder(newUsageBuilder, devDependency); + return new ModuleExtensionProxy(newUsageBuilder, proxyBuilder); } // Find an existing usage builder corresponding to this extension. Isolated usages need to get @@ -443,14 +447,14 @@ public ModuleExtensionProxy useExtension( if (!isolate) { for (ModuleExtensionUsageBuilder usageBuilder : context.getExtensionUsageBuilders()) { if (usageBuilder.isForExtension(extensionBzlFile, extensionName)) { - return ModuleExtensionProxy.createFromUsagesBuilder(usageBuilder, devDependency); + return new ModuleExtensionProxy(usageBuilder, proxyBuilder); } } } // If no such proxy exists, we can just use a new one. context.getExtensionUsageBuilders().add(newUsageBuilder); - return ModuleExtensionProxy.createFromUsagesBuilder(newUsageBuilder, devDependency); + return new ModuleExtensionProxy(newUsageBuilder, proxyBuilder); } private String normalizeLabelString(InterimModule.Builder module, String rawExtensionBzlFile) { @@ -507,30 +511,19 @@ private Object parseOverrideLabelAttribute(InterimModule.Builder module, String @StarlarkBuiltin(name = "module_extension_proxy", documented = false) static class ModuleExtensionProxy implements Structure, StarlarkExportable { private final ModuleExtensionUsageBuilder usageBuilder; - private final boolean devDependency; - - /** - * Creates a proxy with the specified dev_dependency bit that shares accumulated imports and - * tags with all other such proxies, thus preserving their order across dev/non-dev deps. - */ - static ModuleExtensionProxy createFromUsagesBuilder( - ModuleExtensionUsageBuilder usageBuilder, boolean devDependency) { - if (devDependency) { - usageBuilder.setHasDevUseExtension(); - } else { - usageBuilder.setHasNonDevUseExtension(); - } - return new ModuleExtensionProxy(usageBuilder, devDependency); - } + private final ModuleExtensionUsage.Proxy.Builder proxyBuilder; - private ModuleExtensionProxy(ModuleExtensionUsageBuilder usageBuilder, boolean devDependency) { + ModuleExtensionProxy( + ModuleExtensionUsageBuilder usageBuilder, ModuleExtensionUsage.Proxy.Builder proxyBuilder) { this.usageBuilder = usageBuilder; - this.devDependency = devDependency; + this.proxyBuilder = proxyBuilder; + usageBuilder.addProxyBuilder(proxyBuilder); } void addImport(String localRepoName, String exportedName, String byWhat, Location location) throws EvalException { - usageBuilder.addImport(localRepoName, exportedName, devDependency, byWhat, location); + usageBuilder.addImport(localRepoName, exportedName, byWhat, location); + proxyBuilder.addImport(localRepoName, exportedName); } class TagCallable implements StarlarkValue { @@ -551,7 +544,7 @@ public void call(Dict kwargs, StarlarkThread thread) { Tag.builder() .setTagName(tagName) .setAttributeValues(AttributeValues.create(kwargs)) - .setDevDependency(devDependency) + .setDevDependency(proxyBuilder.isDevDependency()) .setLocation(thread.getCallerLocation()) .build()); } @@ -575,12 +568,12 @@ public String getErrorMessageForUnknownField(String field) { @Override public boolean isExported() { - return usageBuilder.isExported(); + return !proxyBuilder.getProxyName().isEmpty(); } @Override public void export(EventHandler handler, Label bzlFileLabel, String name) { - usageBuilder.setExportedName(name); + proxyBuilder.setProxyName(name); } } @@ -646,15 +639,13 @@ public RepoRuleProxy useRepoRule(String bzlFile, String ruleName, StarlarkThread throws EvalException { ModuleThreadContext context = ModuleThreadContext.fromOrFail(thread, "use_repo_rule()"); context.setNonModuleCalled(); - // The builder for the singular "innate" extension of this module. Note that there's only one - // such usage (and it's fabricated), so the usage location just points to this file. + // The builder for the singular "innate" extension of this module. ModuleExtensionUsageBuilder newUsageBuilder = new ModuleExtensionUsageBuilder( context, "//:MODULE.bazel", ModuleExtensionId.INNATE_EXTENSION_NAME, - /* isolate= */ false, - Location.fromFile(thread.getCallerLocation().file())); + /* isolate= */ false); for (ModuleExtensionUsageBuilder usageBuilder : context.getExtensionUsageBuilders()) { if (usageBuilder.isForExtension("//:MODULE.bazel", ModuleExtensionId.INNATE_EXTENSION_NAME)) { return new RepoRuleProxy(usageBuilder, bzlFile + '%' + ruleName); @@ -693,7 +684,11 @@ public void call( } kwargs.putEntry("name", name); ModuleExtensionProxy extensionProxy = - ModuleExtensionProxy.createFromUsagesBuilder(usageBuilder, devDependency); + new ModuleExtensionProxy( + usageBuilder, + ModuleExtensionUsage.Proxy.builder() + .setDevDependency(devDependency) + .setLocation(thread.getCallerLocation())); extensionProxy.getValue(tagName).call(kwargs, thread); extensionProxy.addImport(name, name, "by a repo rule", thread.getCallerLocation()); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleThreadContext.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleThreadContext.java index 8126ffc3ea00ef..ed70e57f7fdb81 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleThreadContext.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleThreadContext.java @@ -15,11 +15,12 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import static com.google.common.collect.ImmutableList.toImmutableList; + import com.google.common.collect.HashBiMap; -import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.devtools.build.lib.bazel.bzlmod.InterimModule.DepSpec; import com.google.devtools.build.lib.cmdline.RepositoryName; import java.util.ArrayList; @@ -121,28 +122,21 @@ static class ModuleExtensionUsageBuilder { private final String extensionBzlFile; private final String extensionName; private final boolean isolate; - private final Location location; + private final ArrayList proxyBuilders; private final HashBiMap imports; - private final ImmutableSet.Builder devImports; private final ImmutableList.Builder tags; - private boolean hasNonDevUseExtension; - private boolean hasDevUseExtension; - private String exportedName; - ModuleExtensionUsageBuilder( ModuleThreadContext context, String extensionBzlFile, String extensionName, - boolean isolate, - Location location) { + boolean isolate) { this.context = context; this.extensionBzlFile = extensionBzlFile; this.extensionName = extensionName; this.isolate = isolate; - this.location = location; + this.proxyBuilders = new ArrayList<>(); this.imports = HashBiMap.create(); - this.devImports = ImmutableSet.builder(); this.tags = ImmutableList.builder(); } @@ -150,20 +144,8 @@ ModuleThreadContext getContext() { return context; } - void setHasNonDevUseExtension() { - hasNonDevUseExtension = true; - } - - void setHasDevUseExtension() { - hasDevUseExtension = true; - } - - void setExportedName(String exportedName) { - this.exportedName = exportedName; - } - - boolean isExported() { - return exportedName != null; + void addProxyBuilder(ModuleExtensionUsage.Proxy.Builder builder) { + proxyBuilders.add(builder); } boolean isForExtension(String extensionBzlFile, String extensionName) { @@ -175,7 +157,6 @@ boolean isForExtension(String extensionBzlFile, String extensionName) { void addImport( String localRepoName, String exportedName, - boolean devDependency, String byWhat, Location location) throws EvalException { @@ -189,9 +170,6 @@ void addImport( exportedName, extensionName, context.repoNameUsages.get(collisionRepoName).where()); } imports.put(localRepoName, exportedName); - if (devDependency) { - devImports.add(exportedName); - } } void addTag(Tag tag) { @@ -199,26 +177,25 @@ void addTag(Tag tag) { } ModuleExtensionUsage buildUsage() throws EvalException { + var proxies = proxyBuilders.stream().map(p -> p.build()).collect(toImmutableList()); var builder = ModuleExtensionUsage.builder() .setExtensionBzlFile(extensionBzlFile) .setExtensionName(extensionName) .setUsingModule(context.getModuleBuilder().getKey()) - .setLocation(location) - .setImports(ImmutableBiMap.copyOf(imports)) - .setDevImports(devImports.build()) - .setHasDevUseExtension(hasDevUseExtension) - .setHasNonDevUseExtension(hasNonDevUseExtension) + .setProxies(proxies) .setTags(tags.build()); if (isolate) { - if (exportedName == null) { + ModuleExtensionUsage.Proxy onlyProxy = Iterables.getOnlyElement(proxies); + if (onlyProxy.getProxyName().isEmpty()) { throw Starlark.errorf( - "Isolated extension usage at %s must be assigned to a top-level variable", location); + "Isolated extension usage at %s must be assigned to a top-level variable", + onlyProxy.getLocation()); } builder.setIsolationKey( Optional.of( ModuleExtensionId.IsolationKey.create( - context.getModuleBuilder().getKey(), exportedName))); + context.getModuleBuilder().getKey(), onlyProxy.getProxyName()))); } else { builder.setIsolationKey(Optional.empty()); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Registry.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Registry.java index 619149d7621a36..77420ee02c74c9 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Registry.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Registry.java @@ -48,4 +48,10 @@ RepoSpec getRepoSpec(ModuleKey key, ExtendedEventHandler eventHandler) Optional> getYankedVersions( String moduleName, ExtendedEventHandler eventHandler) throws IOException, InterruptedException; + + /** + * Returns the yanked versions information, limited to the given selected module version, purely + * based on the lockfile (if possible). + */ + Optional tryGetYankedVersionsFromLockfile(ModuleKey selectedModuleKey); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java index e24b3452fe8e07..d910c3d2d75c33 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactory.java @@ -16,6 +16,7 @@ package com.google.devtools.build.lib.bazel.bzlmod; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import java.net.URISyntaxException; import java.util.Optional; @@ -28,6 +29,10 @@ public interface RegistryFactory { * *

Outside of tests, only {@link RegistryFunction} should call this method. */ - Registry createRegistry(String url, ImmutableMap> fileHashes) + Registry createRegistry( + String url, + RepositoryOptions.LockfileMode lockfileMode, + ImmutableMap> fileHashes, + ImmutableMap previouslySelectedYankedVersions) throws URISyntaxException; } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java index 06efd4050cbd0d..f202b650db17ec 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryImpl.java @@ -17,9 +17,9 @@ import com.google.common.collect.ImmutableMap; import com.google.devtools.build.lib.bazel.bzlmod.IndexRegistry.KnownFileHashesMode; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; -import com.google.devtools.build.lib.vfs.Path; import java.net.URI; import java.net.URISyntaxException; import java.util.Map; @@ -28,24 +28,23 @@ /** Prod implementation of {@link RegistryFactory}. */ public class RegistryFactoryImpl implements RegistryFactory { - private final Path workspacePath; private final DownloadManager downloadManager; private final Supplier> clientEnvironmentSupplier; public RegistryFactoryImpl( - Path workspacePath, - DownloadManager downloadManager, - Supplier> clientEnvironmentSupplier) { - this.workspacePath = workspacePath; + DownloadManager downloadManager, Supplier> clientEnvironmentSupplier) { this.downloadManager = downloadManager; this.clientEnvironmentSupplier = clientEnvironmentSupplier; } @Override public Registry createRegistry( - String unresolvedUrl, ImmutableMap> knownFileHashes) + String url, + LockfileMode lockfileMode, + ImmutableMap> knownFileHashes, + ImmutableMap previouslySelectedYankedVersions) throws URISyntaxException { - URI uri = new URI(unresolvedUrl.replace("%workspace%", workspacePath.getPathString())); + URI uri = new URI(url); if (uri.getScheme() == null) { throw new URISyntaxException( uri.toString(), @@ -60,17 +59,20 @@ public Registry createRegistry( } var knownFileHashesMode = switch (uri.getScheme()) { - case "http", "https" -> KnownFileHashesMode.USE_AND_UPDATE; + case "http", "https" -> + lockfileMode == LockfileMode.ERROR + ? KnownFileHashesMode.ENFORCE + : KnownFileHashesMode.USE_AND_UPDATE; case "file" -> KnownFileHashesMode.IGNORE; default -> throw new URISyntaxException(uri.toString(), "Unrecognized registry URL protocol"); }; return new IndexRegistry( uri, - unresolvedUrl, downloadManager, clientEnvironmentSupplier.get(), knownFileHashes, - knownFileHashesMode); + knownFileHashesMode, + previouslySelectedYankedVersions); } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java index 0c6c12a7ac84bc..121027c658e2aa 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFunction.java @@ -15,7 +15,9 @@ package com.google.devtools.build.lib.bazel.bzlmod; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.server.FailureDetails; +import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.skyframe.SkyFunction; import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyKey; @@ -26,15 +28,19 @@ /** A simple SkyFunction that creates a {@link Registry} with a given URL. */ public class RegistryFunction implements SkyFunction { private final RegistryFactory registryFactory; + private final Path workspaceRoot; - public RegistryFunction(RegistryFactory registryFactory) { + public RegistryFunction(RegistryFactory registryFactory, Path workspaceRoot) { this.registryFactory = registryFactory; + this.workspaceRoot = workspaceRoot; } @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException, RegistryException { + LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env); + BazelLockFileValue lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY); if (lockfile == null) { return null; @@ -42,7 +48,11 @@ public SkyValue compute(SkyKey skyKey, Environment env) RegistryKey key = (RegistryKey) skyKey.argument(); try { - return registryFactory.createRegistry(key.getUrl(), lockfile.getRegistryFileHashes()); + return registryFactory.createRegistry( + key.getUrl().replace("%workspace%", workspaceRoot.getPathString()), + lockfileMode, + lockfile.getRegistryFileHashes(), + lockfile.getSelectedYankedVersions()); } catch (URISyntaxException e) { throw new RegistryException( ExternalDepsException.withCauseAndMessage( diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java index 9bf5373f040255..ccd9ec0a1b3b9a 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionEvalFunction.java @@ -199,7 +199,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) if (lockedExtension == null && extensionShouldHaveBeenLocked) { throw new SingleExtensionEvalFunctionException( ExternalDepsException.withMessage( - Code.BAD_MODULE, + Code.BAD_LOCKFILE, "The module extension '%s'%s does not exist in the lockfile", extensionId, extension.getEvalFactors().isEmpty() @@ -224,8 +224,7 @@ public SkyValue compute(SkyKey skyKey, Environment env) .setBzlTransitiveDigest(extension.getBzlTransitiveDigest()) .setUsagesDigest( SingleExtensionUsagesValue.hashForEvaluation( - GsonTypeAdapterUtil.createSingleExtensionUsagesValueHashGson(), - usagesValue)) + GsonTypeAdapterUtil.SINGLE_EXTENSION_USAGES_VALUE_GSON, usagesValue)) .setRecordedFileInputs(moduleExtensionResult.getRecordedFileInputs()) .setRecordedDirentsInputs(moduleExtensionResult.getRecordedDirentsInputs()) .setEnvVariables(extension.getEnvVars()) @@ -281,7 +280,7 @@ private SingleExtensionValue tryGettingValueFromLockFile( // relevant for the evaluation of the extension. if (!Arrays.equals( SingleExtensionUsagesValue.hashForEvaluation( - GsonTypeAdapterUtil.createSingleExtensionUsagesValueHashGson(), usagesValue), + GsonTypeAdapterUtil.SINGLE_EXTENSION_USAGES_VALUE_GSON, usagesValue), lockedExtension.getUsagesDigest())) { diffRecorder.record("The usages of the extension '" + extensionId + "' have changed"); } @@ -316,7 +315,7 @@ private SingleExtensionValue tryGettingValueFromLockFile( if (lockfileMode.equals(LockfileMode.ERROR)) { throw new SingleExtensionEvalFunctionException( ExternalDepsException.withMessage( - Code.BAD_MODULE, + Code.BAD_LOCKFILE, "MODULE.bazel.lock is no longer up-to-date because: %s. " + "Please run `bazel mod deps --lockfile_mode=update` to update your lockfile.", diffRecorder.getRecordedDiffMessages()), @@ -769,8 +768,8 @@ private RegularRunnableExtension loadRegularRunnableExtension( StarlarkSemantics starlarkSemantics, Environment env) throws InterruptedException, SingleExtensionEvalFunctionException { - Location sampleUsageLocation = - usagesValue.getExtensionUsages().values().iterator().next().getLocation(); + ModuleExtensionUsage sampleUsage = usagesValue.getExtensionUsages().values().iterator().next(); + Location sampleUsageLocation = sampleUsage.getProxies().getFirst().getLocation(); BzlLoadValue bzlLoadValue = loadBzlFile(extensionId.getBzlFileLabel(), sampleUsageLocation, starlarkSemantics, env); if (bzlLoadValue == null) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionFunction.java index a1f406063abf42..d0a8e98c8dc51d 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/SingleExtensionFunction.java @@ -50,21 +50,23 @@ public SkyValue compute(SkyKey skyKey, Environment env) // Check that all imported repos have actually been generated. for (ModuleExtensionUsage usage : usagesValue.getExtensionUsages().values()) { - for (Entry repoImport : usage.getImports().entrySet()) { - if (!evalOnlyValue.getGeneratedRepoSpecs().containsKey(repoImport.getValue())) { - throw new SingleExtensionFunctionException( - ExternalDepsException.withMessage( - Code.INVALID_EXTENSION_IMPORT, - "module extension \"%s\" from \"%s\" does not generate repository \"%s\", yet it" - + " is imported as \"%s\" in the usage at %s%s", - extensionId.getExtensionName(), - extensionId.getBzlFileLabel(), - repoImport.getValue(), - repoImport.getKey(), - usage.getLocation(), - SpellChecker.didYouMean( - repoImport.getValue(), evalOnlyValue.getGeneratedRepoSpecs().keySet())), - Transience.PERSISTENT); + for (ModuleExtensionUsage.Proxy proxy : usage.getProxies()) { + for (Entry repoImport : proxy.getImports().entrySet()) { + if (!evalOnlyValue.getGeneratedRepoSpecs().containsKey(repoImport.getValue())) { + throw new SingleExtensionFunctionException( + ExternalDepsException.withMessage( + Code.INVALID_EXTENSION_IMPORT, + "module extension \"%s\" from \"%s\" does not generate repository \"%s\", yet" + + " it is imported as \"%s\" in the usage at %s%s", + extensionId.getExtensionName(), + extensionId.getBzlFileLabel(), + repoImport.getValue(), + repoImport.getKey(), + proxy.getLocation(), + SpellChecker.didYouMean( + repoImport.getValue(), evalOnlyValue.getGeneratedRepoSpecs().keySet())), + Transience.PERSISTENT); + } } } } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsFunction.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsFunction.java index 89cd60a5da3bdd..57a2344f6657fd 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsFunction.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsFunction.java @@ -27,8 +27,8 @@ import javax.annotation.Nullable; /** - * A simple SkyFunction that computes a {@link RepoSpec} for the given {@link InterimModule} by - * fetching required information from its {@link Registry}. + * A simple SkyFunction that fetches the yanked versions for a given module from its {@link + * Registry}. */ public class YankedVersionsFunction implements SkyFunction { @@ -53,7 +53,8 @@ public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedExcept .handle( Event.warn( String.format( - "Could not read metadata file for module %s: %s", key, e.getMessage()))); + "Could not read metadata file for module %s from registry %s: %s", + key.getModuleName(), key.getRegistryUrl(), e.getMessage()))); // This is failing open: If we can't read the metadata file, we allow yanked modules to be // fetched. return YankedVersionsValue.create(Optional.empty()); diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsUtil.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsUtil.java index 8cfcd32c37a0b0..ac98fb8fe0dcd0 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsUtil.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsUtil.java @@ -59,25 +59,6 @@ static Optional> parseAllowedYankedVersions( return Optional.of(allowedYankedVersionBuilder.build()); } - static Optional getYankedInfo( - ModuleKey key, - YankedVersionsValue yankedVersionsValue, - Optional> allowedYankedVersions) { - return yankedVersionsValue - .yankedVersions() - .flatMap( - yankedVersions -> { - String yankedInfo = yankedVersions.get(key.getVersion()); - if (yankedInfo != null - && allowedYankedVersions.isPresent() - && !allowedYankedVersions.get().contains(key)) { - return Optional.of(yankedInfo); - } else { - return Optional.empty(); - } - }); - } - /** * Parse of a comma-separated list of module version(s) of the form '@' or * 'all' from the string. Returns true if 'all' is present, otherwise returns false. diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsValue.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsValue.java index d3dd5f4ab52532..93362ed1acc4af 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsValue.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/YankedVersionsValue.java @@ -27,6 +27,9 @@ @AutoValue public abstract class YankedVersionsValue implements SkyValue { + /** A value representing a module without yanked versions. */ + public static final YankedVersionsValue NONE_YANKED = create(Optional.of(ImmutableMap.of())); + public abstract Optional> yankedVersions(); public static YankedVersionsValue create(Optional> yankedVersions) { diff --git a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModExecutor.java b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModExecutor.java index 861d5d7bd600df..cd9b6921c674a4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModExecutor.java @@ -444,7 +444,9 @@ private MaybeCompleteSet colorReversePathsToRoot(MaybeCompleteSet repo : usage.getImports().entrySet()) { - printer.printf( - " %s,\n", - repo.getKey().equals(repo.getValue()) - ? String.format("\"%s\"", repo.getKey()) - : String.format("%s=\"%s\"", repo.getKey(), repo.getValue())); + for (ModuleExtensionUsage.Proxy proxy : usage.getProxies()) { + for (Entry repo : proxy.getImports().entrySet()) { + printer.printf( + " %s,\n", + repo.getKey().equals(repo.getValue()) + ? String.format("\"%s\"", repo.getKey()) + : String.format("%s=\"%s\"", repo.getKey(), repo.getValue())); + } } printer.printf(")\n\n"); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java index 36320c5ecefaf5..f82f882854e43b 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/commands/ModCommand.java @@ -28,21 +28,14 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; -import com.google.common.eventbus.Subscribe; import com.google.devtools.build.lib.analysis.NoBuildEvent; import com.google.devtools.build.lib.analysis.NoBuildRequestFinishedEvent; import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphValue; import com.google.devtools.build.lib.bazel.bzlmod.BazelModTidyValue; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue; import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule; -import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionEvent; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; -import com.google.devtools.build.lib.bazel.bzlmod.CompiledModuleFile; -import com.google.devtools.build.lib.bazel.bzlmod.ExternalDepsException; import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId; -import com.google.devtools.build.lib.bazel.bzlmod.ModuleFile; -import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileFunction; -import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue; import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey; import com.google.devtools.build.lib.bazel.bzlmod.RootModuleFileFixup; import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ExtensionArg; @@ -54,7 +47,6 @@ import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModOptions.ModSubcommandConverter; import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg; import com.google.devtools.build.lib.bazel.bzlmod.modcommand.ModuleArg.ModuleArgConverter; -import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.cmdline.LabelSyntaxException; import com.google.devtools.build.lib.cmdline.RepositoryMapping; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -79,11 +71,8 @@ import com.google.devtools.build.lib.util.DetailedExitCode; import com.google.devtools.build.lib.util.InterruptedFailureDetails; import com.google.devtools.build.lib.util.MaybeCompleteSet; -import com.google.devtools.build.lib.vfs.FileSystemUtils; -import com.google.devtools.build.lib.vfs.RootedPath; import com.google.devtools.build.skyframe.EvaluationContext; import com.google.devtools.build.skyframe.EvaluationResult; -import com.google.devtools.build.skyframe.SkyFunctionException; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; import com.google.devtools.common.options.OptionsParsingException; @@ -193,7 +182,6 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe @Nullable BazelModuleInspectorValue moduleInspector; @Nullable BazelModTidyValue modTidyValue; ImmutableList repoMappingValues; - TidyEventRecorder tidyEventRecorder = new TidyEventRecorder(); SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor(); LoadingPhaseThreadsOption threadsOption = options.getOptions(LoadingPhaseThreadsOption.class); @@ -212,7 +200,6 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe keys.addAll(repoMappingKeys); } else if (subcommand.equals(ModSubcommand.TIDY)) { keys.add(BazelModTidyValue.KEY); - env.getEventBus().register(tidyEventRecorder); } else { keys.add(BazelDepGraphValue.KEY, BazelModuleInspectorValue.KEY); } @@ -279,7 +266,7 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe return reportAndCreateFailureResult( env, "the 'tidy' command doesn't take extra arguments", Code.TOO_MANY_ARGUMENTS); } - return runTidy(env, modTidyValue, tidyEventRecorder); + return runTidy(env, modTidyValue); } // Extract and check the --base_module argument first to use it when parsing the other args. @@ -566,17 +553,7 @@ private BlazeCommandResult execInternal(CommandEnvironment env, OptionsParsingRe return BlazeCommandResult.success(); } - private static class TidyEventRecorder { - @Nullable BazelModuleResolutionEvent bazelModuleResolutionEvent; - - @Subscribe - public void bazelModuleResolved(BazelModuleResolutionEvent event) { - bazelModuleResolutionEvent = event; - } - } - - private BlazeCommandResult runTidy( - CommandEnvironment env, BazelModTidyValue modTidyValue, TidyEventRecorder eventRecorder) { + private BlazeCommandResult runTidy(CommandEnvironment env, BazelModTidyValue modTidyValue) { CommandBuilder buildozerCommand = new CommandBuilder() .setWorkingDir(env.getWorkspace()) @@ -612,68 +589,6 @@ private BlazeCommandResult runTidy( env.getReporter().handle(Event.info(fixupEvent.getSuccessMessage())); } - if (modTidyValue.lockfileMode().equals(LockfileMode.UPDATE)) { - // We cannot safely rerun Skyframe evaluation here to pick up the updated module file. - // Instead, we construct a new BazelModuleResolutionEvent with the updated module file - // contents to be picked up by BazelLockFileModule. Since changing use_repos doesn't affect - // module resolution or module extension evaluation, we can reuse the existing lockfile - // information except for the root module file value. - RootedPath moduleFilePath = ModuleFileFunction.getModuleFilePath(env.getWorkspace()); - byte[] moduleFileContents; - try { - moduleFileContents = FileSystemUtils.readContent(moduleFilePath.asPath()); - } catch (IOException e) { - return reportAndCreateFailureResult( - env, - "Unexpected error while reading module file after running buildozer: " + e.getMessage(), - Code.BUILDOZER_FAILED); - } - CompiledModuleFile compiledModuleFile; - try { - compiledModuleFile = - CompiledModuleFile.parseAndCompile( - ModuleFile.create(moduleFileContents, moduleFilePath.asPath().getPathString()), - ModuleKey.ROOT, - modTidyValue.starlarkSemantics(), - env.getRuntime().getRuleClassProvider().getBazelStarlarkEnvironment(), - env.getReporter()); - } catch (ExternalDepsException e) { - return reportAndCreateFailureResult( - env, - "Unexpected error while compiling module file after running buildozer: " - + e.getMessage(), - Code.BUILDOZER_FAILED); - } - ModuleFileValue.RootModuleFileValue newRootModuleFileValue; - try { - newRootModuleFileValue = - ModuleFileFunction.evaluateRootModuleFile( - compiledModuleFile, - modTidyValue.includeLabelToCompiledModuleFile(), - ModuleFileFunction.getBuiltinModules( - env.getDirectories().getEmbeddedBinariesRoot()), - modTidyValue.moduleOverrides(), - modTidyValue.ignoreDevDeps(), - modTidyValue.starlarkSemantics(), - env.getReporter()); - } catch (SkyFunctionException | InterruptedException e) { - return reportAndCreateFailureResult( - env, - "Unexpected error parsing module file after running buildozer: " + e.getMessage(), - Code.BUILDOZER_FAILED); - } - // BazelModuleResolutionEvent is cached by Skyframe and thus always emitted. - BazelModuleResolutionEvent updatedModuleResolutionEvent = - BazelModuleResolutionEvent.create( - eventRecorder.bazelModuleResolutionEvent.getOnDiskLockfileValue(), - eventRecorder - .bazelModuleResolutionEvent - .getResolutionOnlyLockfileValue() - .withShallowlyReplacedRootModule(newRootModuleFileValue), - eventRecorder.bazelModuleResolutionEvent.getExtensionUsagesById()); - env.getReporter().post(updatedModuleResolutionEvent); - } - return BlazeCommandResult.success(); } diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Checksum.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Checksum.java index 8a5ea43b114bb5..8a75b5f91c483f 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Checksum.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/Checksum.java @@ -17,6 +17,7 @@ import com.google.common.base.Ascii; import com.google.common.hash.HashCode; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType; +import java.io.IOException; import java.util.Base64; /** The content checksum for an HTTP download, which knows its own type. */ @@ -32,6 +33,13 @@ private InvalidChecksumException(String msg) { } } + /** Exception thrown to indicate that a checksum is missing. */ + public static final class MissingChecksumException extends IOException { + public MissingChecksumException(String message) { + super(message); + } + } + private final KeyType keyType; private final HashCode hashCode; private final boolean useSubresourceIntegrity; diff --git a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java index 9737437cec0cf9..e67aaf8adeade4 100644 --- a/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java +++ b/src/main/java/com/google/devtools/build/lib/bazel/repository/downloader/DownloadManager.java @@ -375,6 +375,9 @@ private Path downloadInExecutor( /** * Downloads the contents of one URL and reads it into a byte array. * + *

This is only meant to be used for Bzlmod registry downloads as it ignores the value of + * --repository_disable_download. + * *

If the checksum and path to the repository cache is specified, attempt to load the file from * the {@link RepositoryCache}. If it doesn't exist, proceed to download the file and load it into * the cache prior to returning the value. @@ -388,7 +391,7 @@ private Path downloadInExecutor( * @throws IOException if download was attempted and ended up failing * @throws InterruptedException if this thread is being cast into oblivion */ - public byte[] downloadAndReadOneUrl( + public byte[] downloadAndReadOneUrlForBzlmod( URL originalUrl, ExtendedEventHandler eventHandler, Map clientEnv, @@ -439,11 +442,6 @@ public byte[] downloadAndReadOneUrl( authHeaders = rewriter.updateAuthHeaders(rewrittenUrlMappings, authHeaders, netrcCreds); } - if (disableDownload) { - throw new IOException( - String.format("Failed to download %s: download is disabled.", originalUrl)); - } - if (rewrittenUrls.isEmpty()) { throw new IOException(getRewriterBlockedAllUrlsMessage(ImmutableList.of(originalUrl))); } diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java b/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java index 850bf379914f9b..97bcf7770a0a48 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/BzlmodRepoRuleFunction.java @@ -14,7 +14,6 @@ package com.google.devtools.build.lib.skyframe; -import static com.google.common.collect.ImmutableMap.toImmutableMap; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -22,7 +21,6 @@ import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.analysis.BlazeDirectories; import com.google.devtools.build.lib.bazel.bzlmod.ArchiveRepoSpecBuilder; -import com.google.devtools.build.lib.bazel.bzlmod.AttributeValues; import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphValue; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleCreator; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodRepoRuleValue; @@ -51,7 +49,6 @@ import com.google.devtools.build.skyframe.SkyFunctionException.Transience; import com.google.devtools.build.skyframe.SkyKey; import com.google.devtools.build.skyframe.SkyValue; -import java.util.Map; import java.util.Map.Entry; import java.util.Optional; import java.util.Set; @@ -193,7 +190,7 @@ private BzlmodRepoRuleValue createRuleFromSpec( var attributes = ImmutableMap.builder() - .putAll(resolveRemotePatchesUrl(repoSpec).attributes()) + .putAll(repoSpec.attributes().attributes()) .put("name", repositoryName.getName()) .buildOrThrow(); try { @@ -217,35 +214,6 @@ private BzlmodRepoRuleValue createRuleFromSpec( } } - /* Resolves repo specs containing remote patches that are stored with %workspace% place holder*/ - @SuppressWarnings("unchecked") - private AttributeValues resolveRemotePatchesUrl(RepoSpec repoSpec) { - if (repoSpec - .getRuleClass() - .equals(ArchiveRepoSpecBuilder.HTTP_ARCHIVE_PATH + "%http_archive")) { - return AttributeValues.create( - repoSpec.attributes().attributes().entrySet().stream() - .collect( - toImmutableMap( - Map.Entry::getKey, - e -> { - if (e.getKey().equals("remote_patches")) { - Map remotePatches = (Map) e.getValue(); - return remotePatches.keySet().stream() - .collect( - toImmutableMap( - key -> - key.replace( - "%workspace%", - directories.getWorkspace().getPathString()), - remotePatches::get)); - } - return e.getValue(); - }))); - } - return repoSpec.attributes(); - } - // Starlark rules loaded from bazel_tools that may define Bazel module repositories and thus must // be loaded without relying on any other modules. private static final Set BOOTSTRAP_RULE_CLASSES = diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java index a27685dc05ee31..0f12a465d215b4 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/SkyframeExecutor.java @@ -271,6 +271,7 @@ import java.util.concurrent.locks.Lock; import java.util.function.Consumer; import java.util.function.Supplier; +import java.util.stream.Stream; import javax.annotation.Nullable; import javax.annotation.concurrent.GuardedBy; import net.starlark.java.eval.StarlarkSemantics; diff --git a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java index 344048abe5413c..36e72039fdcb95 100644 --- a/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java +++ b/src/main/java/com/google/devtools/build/lib/skyframe/packages/BazelPackageLoader.java @@ -139,8 +139,7 @@ public BazelPackageLoader buildImpl() { HttpDownloader httpDownloader = new HttpDownloader(); DownloadManager downloadManager = new DownloadManager(repositoryCache, httpDownloader); RegistryFactory registryFactory = - new RegistryFactoryImpl( - directories.getWorkspace(), downloadManager, Suppliers.ofInstance(ImmutableMap.of())); + new RegistryFactoryImpl(downloadManager, Suppliers.ofInstance(ImmutableMap.of())); // Allow tests to override the following functions to use fake registry or custom built-in // modules @@ -160,7 +159,9 @@ public BazelPackageLoader buildImpl() { } if (!this.extraSkyFunctions.containsKey(SkyFunctions.REGISTRY)) { addExtraSkyFunctions( - ImmutableMap.of(SkyFunctions.REGISTRY, new RegistryFunction(registryFactory))); + ImmutableMap.of( + SkyFunctions.REGISTRY, + new RegistryFunction(registryFactory, directories.getWorkspace()))); } addExtraSkyFunctions( diff --git a/src/main/protobuf/failure_details.proto b/src/main/protobuf/failure_details.proto index 98419ca1b9497f..5a0f7a9664793f 100644 --- a/src/main/protobuf/failure_details.proto +++ b/src/main/protobuf/failure_details.proto @@ -1300,6 +1300,7 @@ message ExternalDeps { INVALID_REGISTRY_URL = 4 [(metadata) = { exit_code: 48 }]; ERROR_ACCESSING_REGISTRY = 5 [(metadata) = { exit_code: 32 }]; INVALID_EXTENSION_IMPORT = 6 [(metadata) = { exit_code: 48 }]; + BAD_LOCKFILE = 7 [(metadata) = { exit_code: 48 }]; } Code code = 1; diff --git a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java index 9121c7f5a062fa..1d142e19e2cdb7 100644 --- a/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java +++ b/src/test/java/com/google/devtools/build/lib/analysis/util/AnalysisMock.java @@ -181,7 +181,9 @@ public ImmutableMap getSkyFunctions(BlazeDirectori SkyFunctions.SINGLE_EXTENSION_EVAL, new SingleExtensionEvalFunction(directories, ImmutableMap::of, downloadManager)) .put(SkyFunctions.SINGLE_EXTENSION_USAGES, new SingleExtensionUsagesFunction()) - .put(SkyFunctions.REGISTRY, new RegistryFunction(FakeRegistry.DEFAULT_FACTORY)) + .put( + SkyFunctions.REGISTRY, + new RegistryFunction(FakeRegistry.DEFAULT_FACTORY, directories.getWorkspace())) .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) .put(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction()) .put( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD index 09fe37dc762c5f..378fff3ced078c 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD @@ -33,7 +33,6 @@ java_library( "//src/main/java/com/google/devtools/build/lib/analysis:blaze_version_info", "//src/main/java/com/google/devtools/build/lib/analysis:server_directories", "//src/main/java/com/google/devtools/build/lib/authandtls", - "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:bazel_lockfile_module", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:exception", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:inspection", @@ -117,6 +116,7 @@ java_library( "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:registry", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution", "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl", + "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options", "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader", "//src/main/java/com/google/devtools/build/lib/cmdline", "//src/main/java/com/google/devtools/build/lib/events", diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java index a8d61f8fa61ed5..f8852f22cfc8d6 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunctionTest.java @@ -129,7 +129,9 @@ public void setup() throws Exception { .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, resolutionFunctionMock) - .put(SkyFunctions.REGISTRY, new RegistryFunction(new FakeRegistry.Factory())) + .put( + SkyFunctions.REGISTRY, + new RegistryFunction(new FakeRegistry.Factory(), directories.getWorkspace())) .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) .put(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction()) .put( @@ -217,12 +219,13 @@ private static ModuleExtensionUsage createModuleExtensionUsage( .setExtensionBzlFile(bzlFile) .setExtensionName(name) .setIsolationKey(Optional.empty()) - .setImports(importsBuilder.buildOrThrow()) - .setDevImports(ImmutableSet.of()) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setDevDependency(false) + .setLocation(Location.BUILTIN) + .setImports(importsBuilder.buildOrThrow()) + .build()) .setUsingModule(ModuleKey.ROOT) - .setLocation(Location.BUILTIN) - .setHasDevUseExtension(false) - .setHasNonDevUseExtension(true) .build(); } @@ -364,7 +367,8 @@ public void setDepGraph(ImmutableMap depGraph) { @Override @Nullable public SkyValue compute(SkyKey skyKey, Environment env) { - return BazelModuleResolutionValue.create(depGraph, ImmutableMap.of(), ImmutableMap.of()); + return BazelModuleResolutionValue.create( + depGraph, ImmutableMap.of(), ImmutableMap.of(), ImmutableMap.of()); } } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java deleted file mode 100644 index f4dc8815ef01cd..00000000000000 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java +++ /dev/null @@ -1,548 +0,0 @@ -// Copyright 2023 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -package com.google.devtools.build.lib.bazel.bzlmod; - -import static com.google.common.truth.Truth.assertThat; -import static org.junit.Assert.fail; - -import com.google.auto.value.AutoValue; -import com.google.common.base.Suppliers; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.devtools.build.lib.actions.FileValue; -import com.google.devtools.build.lib.analysis.BlazeDirectories; -import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider; -import com.google.devtools.build.lib.analysis.ServerDirectories; -import com.google.devtools.build.lib.analysis.util.AnalysisMock; -import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction.BazelLockfileFunctionException; -import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; -import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode; -import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode; -import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; -import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule; -import com.google.devtools.build.lib.clock.BlazeClock; -import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions; -import com.google.devtools.build.lib.pkgcache.PathPackageLocator; -import com.google.devtools.build.lib.rules.repository.LocalRepositoryFunction; -import com.google.devtools.build.lib.rules.repository.LocalRepositoryRule; -import com.google.devtools.build.lib.rules.repository.RepositoryDelegatorFunction; -import com.google.devtools.build.lib.rules.repository.RepositoryFunction; -import com.google.devtools.build.lib.skyframe.BazelSkyframeExecutorConstants; -import com.google.devtools.build.lib.skyframe.BzlmodRepoRuleFunction; -import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction; -import com.google.devtools.build.lib.skyframe.ExternalFilesHelper; -import com.google.devtools.build.lib.skyframe.ExternalFilesHelper.ExternalFileAction; -import com.google.devtools.build.lib.skyframe.FileFunction; -import com.google.devtools.build.lib.skyframe.FileStateFunction; -import com.google.devtools.build.lib.skyframe.PrecomputedFunction; -import com.google.devtools.build.lib.skyframe.PrecomputedValue; -import com.google.devtools.build.lib.skyframe.SkyFunctions; -import com.google.devtools.build.lib.starlarkbuildapi.repository.RepositoryBootstrap; -import com.google.devtools.build.lib.testutil.FoundationTestCase; -import com.google.devtools.build.lib.testutil.TestRuleClassProvider; -import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor; -import com.google.devtools.build.lib.vfs.FileStateKey; -import com.google.devtools.build.lib.vfs.Root; -import com.google.devtools.build.lib.vfs.SyscallCache; -import com.google.devtools.build.skyframe.EvaluationContext; -import com.google.devtools.build.skyframe.EvaluationResult; -import com.google.devtools.build.skyframe.InMemoryMemoizingEvaluator; -import com.google.devtools.build.skyframe.MemoizingEvaluator; -import com.google.devtools.build.skyframe.RecordingDifferencer; -import com.google.devtools.build.skyframe.SequencedRecordingDifferencer; -import com.google.devtools.build.skyframe.SkyFunction; -import com.google.devtools.build.skyframe.SkyFunctionName; -import com.google.devtools.build.skyframe.SkyKey; -import com.google.devtools.build.skyframe.SkyValue; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; -import java.util.regex.Pattern; -import javax.annotation.Nullable; -import net.starlark.java.eval.StarlarkSemantics; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; - -/** Tests for {@link BazelLockFileFunction}. */ -@RunWith(JUnit4.class) -public class BazelLockFileFunctionTest extends FoundationTestCase { - - private MemoizingEvaluator evaluator; - private RecordingDifferencer differencer; - private EvaluationContext evaluationContext; - private static SkyFunctionName updateLockfileFunction; - - @Before - public void setup() throws Exception { - differencer = new SequencedRecordingDifferencer(); - evaluationContext = - EvaluationContext.newBuilder().setParallelism(8).setEventHandler(reporter).build(); - - AtomicReference packageLocator = - new AtomicReference<>( - new PathPackageLocator( - outputBase, - ImmutableList.of(Root.fromPath(rootDirectory)), - BazelSkyframeExecutorConstants.BUILD_FILES_BY_PRIORITY)); - BlazeDirectories directories = - new BlazeDirectories( - new ServerDirectories(rootDirectory, outputBase, rootDirectory), - rootDirectory, - /* defaultSystemJavabase= */ null, - AnalysisMock.get().getProductName()); - ExternalFilesHelper externalFilesHelper = - ExternalFilesHelper.createForTesting( - packageLocator, - ExternalFileAction.DEPEND_ON_EXTERNAL_PKG_FOR_EXTERNAL_REPO_PATHS, - directories); - RepositoryFunction localRepositoryFunction = new LocalRepositoryFunction(); - ImmutableMap repositoryHandlers = - ImmutableMap.of(LocalRepositoryRule.NAME, localRepositoryFunction); - - ConfiguredRuleClassProvider.Builder builder = new ConfiguredRuleClassProvider.Builder(); - TestRuleClassProvider.addStandardRules(builder); - builder - .clearWorkspaceFileSuffixForTesting() - .addStarlarkBootstrap(new RepositoryBootstrap(new StarlarkRepositoryModule())); - ConfiguredRuleClassProvider ruleClassProvider = builder.build(); - - updateLockfileFunction = SkyFunctionName.createHermetic("LockfileWrite"); - - evaluator = - new InMemoryMemoizingEvaluator( - ImmutableMap.builder() - .put(FileValue.FILE, new FileFunction(packageLocator, directories)) - .put( - FileStateKey.FILE_STATE, - new FileStateFunction( - Suppliers.ofInstance( - new TimestampGranularityMonitor(BlazeClock.instance())), - SyscallCache.NO_CACHE, - externalFilesHelper)) - .put( - SkyFunctions.REPOSITORY_DIRECTORY, - new RepositoryDelegatorFunction( - repositoryHandlers, - null, - new AtomicBoolean(true), - ImmutableMap::of, - directories, - BazelSkyframeExecutorConstants.EXTERNAL_PACKAGE_HELPER)) - .put( - BzlmodRepoRuleValue.BZLMOD_REPO_RULE, - new BzlmodRepoRuleFunction(ruleClassProvider, directories)) - .put(SkyFunctions.PRECOMPUTED, new PrecomputedFunction()) - .put( - SkyFunctions.MODULE_FILE, - new ModuleFileFunction( - ruleClassProvider.getBazelStarlarkEnvironment(), - rootDirectory, - ImmutableMap.of())) - .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) - .put(SkyFunctions.REGISTRY, new RegistryFunction(new FakeRegistry.Factory())) - .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) - .put(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction()) - .put( - SkyFunctions.MODULE_EXTENSION_REPO_MAPPING_ENTRIES, - new ModuleExtensionRepoMappingEntriesFunction()) - .put( - SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, - new ClientEnvironmentFunction( - new AtomicReference<>(ImmutableMap.of("BZLMOD_ALLOW_YANKED_VERSIONS", "")))) - .put( - updateLockfileFunction, - new SkyFunction() { - @Nullable - @Override - public SkyValue compute(SkyKey skyKey, Environment env) - throws BazelLockfileFunctionException, InterruptedException { - - UpdateLockFileKey key = (UpdateLockFileKey) skyKey; - BzlmodFlagsAndEnvVars flags = BazelDepGraphFunction.getFlagsAndEnvVars(env); - if (flags == null) { - return null; - } - - ImmutableMap localOverrideHashes = - BazelDepGraphFunction.getLocalOverridesHashes(key.overrides(), env); - if (localOverrideHashes == null) { - return null; - } - BazelLockFileModule.updateLockfile( - rootDirectory, - BazelLockFileValue.builder() - .setModuleFileHash(key.moduleHash()) - .setFlags(flags) - .setLocalOverrideHashes(localOverrideHashes) - .setModuleDepGraph(key.depGraph()) - .setRegistryFileHashes(ImmutableMap.of()) - .build()); - - return new SkyValue() {}; - } - }) - .buildOrThrow(), - differencer); - - PrecomputedValue.STARLARK_SEMANTICS.set( - differencer, - StarlarkSemantics.builder().setBool(BuildLanguageOptions.ENABLE_BZLMOD, true).build()); - ModuleFileFunction.REGISTRIES.set(differencer, ImmutableSet.of()); - ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); - ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); - YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); - BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( - differencer, BazelCompatibilityMode.ERROR); - BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( - differencer, CheckDirectDepsMode.ERROR); - BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); - RepositoryDelegatorFunction.REPOSITORY_OVERRIDES.set(differencer, ImmutableMap.of()); - RepositoryDelegatorFunction.FORCE_FETCH.set( - differencer, RepositoryDelegatorFunction.FORCE_FETCH_DISABLED); - RepositoryDelegatorFunction.VENDOR_DIRECTORY.set(differencer, Optional.empty()); - } - - @Test - public void simpleModule() throws Exception { - scratch.overwriteFile( - rootDirectory.getRelative("MODULE.bazel").getPathString(), - "module(name='my_root', version='1.0')", - "bazel_dep(name = 'dep_1', version = '1.0')", - "bazel_dep(name = 'dep_2', version = '2.0')"); - - EvaluationResult rootResult = - evaluator.evaluate( - ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); - if (rootResult.hasError()) { - fail(rootResult.getError().toString()); - } - RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); - - ImmutableMap depGraph = - ImmutableMap.of(ModuleKey.ROOT, InterimModule.toModule(rootValue.getModule(), null, null)); - - UpdateLockFileKey key = - UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); - EvaluationResult result = - evaluator.evaluate(ImmutableList.of(key), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - - result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - - BazelLockFileValue value = result.get(BazelLockFileValue.KEY); - assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); - } - - @Test - public void moduleWithFlags() throws Exception { - // Test having --override_module, --ignore_dev_dependency, --check_bazel_compatibility - // --check_direct_dependencies & --registry - scratch.overwriteFile( - rootDirectory.getRelative("MODULE.bazel").getPathString(), - "module(name='my_root', version='1.0')"); - - EvaluationResult rootResult = - evaluator.evaluate( - ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); - if (rootResult.hasError()) { - fail(rootResult.getError().toString()); - } - RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); - - ImmutableMap depGraph = - ImmutableMap.of(ModuleKey.ROOT, InterimModule.toModule(rootValue.getModule(), null, null)); - - ImmutableList yankedVersions = ImmutableList.of("2.4", "2.3"); - LocalPathOverride override = LocalPathOverride.create("override_path"); - ImmutableSet registries = ImmutableSet.of("registry1", "registry2"); - ImmutableMap moduleOverride = ImmutableMap.of("my_dep_1", override.getPath()); - - ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, true); - ModuleFileFunction.REGISTRIES.set(differencer, registries); - ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of("my_dep_1", override)); - YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, yankedVersions); - BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.set( - differencer, CheckDirectDepsMode.WARNING); - BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.set( - differencer, BazelCompatibilityMode.WARNING); - - UpdateLockFileKey key = - UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); - EvaluationResult result = - evaluator.evaluate(ImmutableList.of(key), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - - result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - - BazelLockFileValue value = result.get(BazelLockFileValue.KEY); - assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); - assertThat(value.getFlags().ignoreDevDependency()).isTrue(); - assertThat(value.getFlags().cmdRegistries()).isEqualTo(registries); - assertThat(value.getFlags().cmdModuleOverrides()).isEqualTo(moduleOverride); - assertThat(value.getFlags().allowedYankedVersions()).isEqualTo(yankedVersions); - assertThat(value.getFlags().directDependenciesMode()) - .isEqualTo(CheckDirectDepsMode.WARNING.toString()); - assertThat(value.getFlags().compatibilityMode()) - .isEqualTo(BazelCompatibilityMode.WARNING.toString()); - } - - @Test - public void moduleWithLocalOverrides() throws Exception { - scratch.overwriteFile( - rootDirectory.getRelative("MODULE.bazel").getPathString(), - "module(name='root',version='0.1')", - "local_path_override(module_name='ss',path='code_for_ss')"); - scratch.file( - rootDirectory.getRelative("code_for_ss/MODULE.bazel").getPathString(), - "module(name='ss',version='1.0')"); - scratch.file(rootDirectory.getRelative("code_for_ss/WORKSPACE").getPathString()); - - EvaluationResult rootResult = - evaluator.evaluate( - ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); - if (rootResult.hasError()) { - fail(rootResult.getError().toString()); - } - RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); - - ImmutableMap depGraph = - ImmutableMap.of(ModuleKey.ROOT, InterimModule.toModule(rootValue.getModule(), null, null)); - - UpdateLockFileKey key = - UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); - EvaluationResult result = - evaluator.evaluate(ImmutableList.of(key), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - - result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - - BazelLockFileValue value = result.get(BazelLockFileValue.KEY); - assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); - assertThat(value.getLocalOverrideHashes()).isNotEmpty(); - } - - @Test - public void fullModule() throws Exception { - scratch.overwriteFile( - rootDirectory.getRelative("MODULE.bazel").getPathString(), - "module(name='my_root', version='1.0')", - "register_toolchains('//my:toolchain', '//my:toolchain2')", - "ext1 = use_extension('//:defs.bzl','ext_1')", - "use_repo(ext1,'myrepo')", - "ext2 = use_extension('@ext//:defs.bzl','ext_2')", - "ext2.tag(file='@myrepo//:Hello1.txt')", - "ext2.tag(file='@myrepo//:Hello2.txt')", - "use_repo(ext2,'ext_repo')"); - - EvaluationResult rootResult = - evaluator.evaluate( - ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); - if (rootResult.hasError()) { - fail(rootResult.getError().toString()); - } - RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); - - ImmutableMap depGraph = - ImmutableMap.of(ModuleKey.ROOT, InterimModule.toModule(rootValue.getModule(), null, null)); - - UpdateLockFileKey key = - UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); - EvaluationResult result = - evaluator.evaluate(ImmutableList.of(key), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - - result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - - BazelLockFileValue value = result.get(BazelLockFileValue.KEY); - assertThat(value.getModuleDepGraph()).isEqualTo(depGraph); - } - - @Test - public void invalidLockfileEmptyFile() throws Exception { - scratch.overwriteFile( - rootDirectory.getRelative("MODULE.bazel").getPathString(), - "module(name='my_root', version='1.0')"); - - EvaluationResult rootResult = - evaluator.evaluate( - ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); - if (rootResult.hasError()) { - fail(rootResult.getError().toString()); - } - RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); - - ImmutableMap depGraph = - ImmutableMap.of(ModuleKey.ROOT, InterimModule.toModule(rootValue.getModule(), null, null)); - - UpdateLockFileKey key = - UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); - EvaluationResult result = - evaluator.evaluate(ImmutableList.of(key), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - - scratch.overwriteFile( - rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), - "{\"lockFileVersion\": " + BazelLockFileValue.LOCK_FILE_VERSION + "}"); - - result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); - if (!result.hasError()) { - fail("expected error about missing field in the lockfile, but succeeded"); - } - assertThat(result.getError().toString()) - .contains( - "Failed to read and parse the MODULE.bazel.lock file with error: " - + "java.lang.IllegalStateException: Missing required properties: moduleFileHash " - + "flags localOverrideHashes moduleDepGraph. Try deleting it and rerun the build."); - } - - @Test - public void invalidLockfileNullFlag() throws Exception { - scratch.overwriteFile( - rootDirectory.getRelative("MODULE.bazel").getPathString(), - "module(name='my_root', version='1.0')"); - - EvaluationResult rootResult = - evaluator.evaluate( - ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); - if (rootResult.hasError()) { - fail(rootResult.getError().toString()); - } - RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); - - ImmutableMap depGraph = - ImmutableMap.of(ModuleKey.ROOT, InterimModule.toModule(rootValue.getModule(), null, null)); - - UpdateLockFileKey key = - UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); - EvaluationResult result = - evaluator.evaluate(ImmutableList.of(key), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - - JsonObject jsonObject = - (JsonObject) JsonParser.parseString(scratch.readFile("MODULE.bazel.lock")); - jsonObject.get("flags").getAsJsonObject().remove("directDependenciesMode"); - scratch.overwriteFile( - rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), jsonObject.toString()); - - result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); - if (!result.hasError()) { - fail("expected error about missing field in the lockfile, but succeeded"); - } - assertThat(result.getError().toString()) - .contains( - "Failed to read and parse the MODULE.bazel.lock file with error: Null" - + " directDependenciesMode. Try deleting it and rerun the build."); - } - - @Test - public void invalidLockfileMalformed() throws Exception { - scratch.overwriteFile( - rootDirectory.getRelative("MODULE.bazel").getPathString(), - "module(name='my_root', version='1.0')"); - - EvaluationResult rootResult = - evaluator.evaluate( - ImmutableList.of(ModuleFileValue.KEY_FOR_ROOT_MODULE), evaluationContext); - if (rootResult.hasError()) { - fail(rootResult.getError().toString()); - } - RootModuleFileValue rootValue = rootResult.get(ModuleFileValue.KEY_FOR_ROOT_MODULE); - - ImmutableMap depGraph = - ImmutableMap.of(ModuleKey.ROOT, InterimModule.toModule(rootValue.getModule(), null, null)); - - UpdateLockFileKey key = - UpdateLockFileKey.create("moduleHash", depGraph, rootValue.getOverrides()); - EvaluationResult result = - evaluator.evaluate(ImmutableList.of(key), evaluationContext); - if (result.hasError()) { - fail(result.getError().toString()); - } - - JsonObject jsonObject = - (JsonObject) JsonParser.parseString(scratch.readFile("MODULE.bazel.lock")); - jsonObject.get("flags").getAsJsonObject().addProperty("allowedYankedVersions", "string!"); - scratch.overwriteFile( - rootDirectory.getRelative("MODULE.bazel.lock").getPathString(), jsonObject.toString()); - - result = evaluator.evaluate(ImmutableList.of(BazelLockFileValue.KEY), evaluationContext); - if (!result.hasError()) { - fail("expected error about invalid field value in the lockfile, but succeeded"); - } - Pattern expectedExceptionMessage = - Pattern.compile( - Pattern.quote( - "Failed to read and parse the MODULE.bazel.lock file with error:" - + " java.lang.IllegalStateException: Expected BEGIN_ARRAY but was STRING at" - + " line 1 column 129 path $.flags.allowedYankedVersions") - + ".*" - + Pattern.quote("Try deleting it and rerun the build."), - Pattern.DOTALL); - assertThat(result.getError().toString()).containsMatch(expectedExceptionMessage); - } - - @AutoValue - abstract static class UpdateLockFileKey implements SkyKey { - - abstract String moduleHash(); - - abstract ImmutableMap depGraph(); - - abstract ImmutableMap overrides(); - - static UpdateLockFileKey create( - String moduleHash, - ImmutableMap depGraph, - ImmutableMap overrides) { - return new AutoValue_BazelLockFileFunctionTest_UpdateLockFileKey( - moduleHash, depGraph, overrides); - } - - @Override - public SkyFunctionName functionName() { - return updateLockfileFunction; - } - } -} diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java index 2c4a7b3099c6d6..38ee7f4d338e29 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelModuleResolutionFunctionTest.java @@ -146,7 +146,9 @@ public void setup() throws Exception { .put(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction()) .put(SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(rootDirectory)) .put(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction()) - .put(SkyFunctions.REGISTRY, new RegistryFunction(registryFactory)) + .put( + SkyFunctions.REGISTRY, + new RegistryFunction(registryFactory, directories.getWorkspace())) .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) .put(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction()) .put( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java index 444ba83ce719d1..9c98532eec437b 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodRepoRuleFunctionTest.java @@ -134,7 +134,9 @@ public void setup() throws Exception { ruleClassProvider.getBazelStarlarkEnvironment(), workspaceRoot, ImmutableMap.of())) - .put(SkyFunctions.REGISTRY, new RegistryFunction(registryFactory)) + .put( + SkyFunctions.REGISTRY, + new RegistryFunction(registryFactory, directories.getWorkspace())) .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) .put(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction()) .put( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java index 81e4685a404dbe..15a5fb4e2b572c 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/DiscoveryTest.java @@ -32,6 +32,7 @@ import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.InterimModuleBuilder; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule; import com.google.devtools.build.lib.clock.BlazeClock; @@ -204,7 +205,9 @@ private void setUpWithBuiltinModules(ImmutableMap b .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) - .put(SkyFunctions.REGISTRY, new RegistryFunction(registryFactory)) + .put( + SkyFunctions.REGISTRY, + new RegistryFunction(registryFactory, directories.getWorkspace())) .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) .put(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction()) .put( @@ -231,6 +234,7 @@ private void setUpWithBuiltinModules(ImmutableMap b ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); } @Test diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java index 1b31b80f82087c..9fe40a91905fd4 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java @@ -20,6 +20,7 @@ import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.events.ExtendedEventHandler; import com.google.errorprone.annotations.CanIgnoreReturnValue; @@ -86,7 +87,7 @@ public RepoSpec getRepoSpec(ModuleKey key, ExtendedEventHandler eventHandler) { "%s/modules/%s/%s/source.json" .formatted(url, key.getName(), key.getVersion().toString()), Optional.of( - GsonTypeAdapterUtil.createSingleExtensionUsagesValueHashGson() + GsonTypeAdapterUtil.SINGLE_EXTENSION_USAGES_VALUE_GSON .toJson(repoSpec) .getBytes(UTF_8)))); return repoSpec; @@ -98,6 +99,12 @@ public Optional> getYankedVersions( return Optional.ofNullable(yankedVersionMap.get(moduleName)); } + @Override + public Optional tryGetYankedVersionsFromLockfile( + ModuleKey selectedModuleKey) { + return Optional.empty(); + } + @Override public boolean equals(Object other) { return other instanceof FakeRegistry @@ -126,7 +133,10 @@ public FakeRegistry newFakeRegistry(String rootPath) { @Override public Registry createRegistry( - String url, ImmutableMap> fileHashes) { + String url, + LockfileMode lockfileMode, + ImmutableMap> fileHashes, + ImmutableMap previouslySelectedYankedVersions) { return Preconditions.checkNotNull(registries.get(url), "unknown registry url: %s", url); } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java index f44daa4d364be1..e76f499ff020b1 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java @@ -32,12 +32,12 @@ import com.google.devtools.build.lib.authandtls.Netrc; import com.google.devtools.build.lib.authandtls.NetrcCredentials; import com.google.devtools.build.lib.authandtls.NetrcParser; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; import com.google.devtools.build.lib.bazel.repository.downloader.Checksum; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; import com.google.devtools.build.lib.testutil.FoundationTestCase; -import com.google.devtools.build.lib.vfs.Path; import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; @@ -85,12 +85,10 @@ public ImmutableMap> getRecordedHashes() { public void setUp() throws Exception { eventRecorder = new EventRecorder(); eventBus.register(eventRecorder); - Path workspaceRoot = scratch.dir("/ws"); repositoryCache = new RepositoryCache(); downloadManager = new DownloadManager(repositoryCache, new HttpDownloader()); registryFactory = - new RegistryFactoryImpl( - workspaceRoot, downloadManager, Suppliers.ofInstance(ImmutableMap.of())); + new RegistryFactoryImpl(downloadManager, Suppliers.ofInstance(ImmutableMap.of())); } @Test @@ -99,7 +97,8 @@ public void testHttpUrl() throws Exception { server.start(); Registry registry = - registryFactory.createRegistry(server.getUrl() + "/myreg", ImmutableMap.of()); + registryFactory.createRegistry( + server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -116,7 +115,8 @@ public void testHttpUrlWithNetrcCreds() throws Exception { new ByteArrayInputStream( "machine [::1] login rinne password rinnepass\n".getBytes(UTF_8))); Registry registry = - registryFactory.createRegistry(server.getUrl() + "/myreg", ImmutableMap.of()); + registryFactory.createRegistry( + server.getUrl() + "/myreg", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); var e = assertThrows( @@ -146,7 +146,10 @@ public void testFileUrl() throws Exception { Registry registry = registryFactory.createRegistry( - new File(tempFolder.getRoot(), "fakereg").toURI().toString(), ImmutableMap.of()); + new File(tempFolder.getRoot(), "fakereg").toURI().toString(), + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue(ModuleFile.create("lol".getBytes(UTF_8), file.toURI().toString())); assertThat(registry.getModuleFile(createModuleKey("bar", "1.0"), reporter)).isEmpty(); @@ -182,7 +185,9 @@ public void testGetArchiveRepoSpec() throws Exception { "}"); server.start(); - Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); + Registry registry = + registryFactory.createRegistry( + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -226,7 +231,9 @@ public void testGetLocalPathRepoSpec() throws Exception { "}"); server.start(); - Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); + Registry registry = + registryFactory.createRegistry( + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() @@ -248,7 +255,9 @@ public void testGetRepoInvalidRegistryJsonSpec() throws Exception { " \"strip_prefix\": \"pref\"", "}"); - Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); + Registry registry = + registryFactory.createRegistry( + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -279,7 +288,9 @@ public void testGetRepoInvalidModuleJsonSpec() throws Exception { "}"); server.start(); - Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); + Registry registry = + registryFactory.createRegistry( + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); } @@ -306,7 +317,9 @@ public void testGetYankedVersion() throws Exception { + " }\n" + "}"); server.start(); - Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); + Registry registry = + registryFactory.createRegistry( + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); Optional> yankedVersion = registry.getYankedVersions("red-pill", reporter); assertThat(yankedVersion) @@ -327,7 +340,9 @@ public void testArchiveWithExplicitType() throws Exception { "}"); server.start(); - Registry registry = registryFactory.createRegistry(server.getUrl(), ImmutableMap.of()); + Registry registry = + registryFactory.createRegistry( + server.getUrl(), LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of()); assertThat(registry.getRepoSpec(createModuleKey("archive_type", "1.0"), reporter)) .isEqualTo( new ArchiveRepoSpecBuilder() @@ -354,7 +369,9 @@ public void testGetModuleFileChecksums() throws Exception { Optional.of(sha256("old")), server.getUrl() + "/myreg/modules/unused/1.0/MODULE.bazel", Optional.of(sha256("unused"))); - Registry registry = registryFactory.createRegistry(server.getUrl() + "/myreg", knownFiles); + Registry registry = + registryFactory.createRegistry( + server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); assertThat(registry.getModuleFile(createModuleKey("foo", "1.0"), reporter)) .hasValue( ModuleFile.create( @@ -378,7 +395,9 @@ public void testGetModuleFileChecksums() throws Exception { Optional.empty()) .inOrder(); - registry = registryFactory.createRegistry(server.getUrl() + "/myreg", recordedChecksums); + registry = + registryFactory.createRegistry( + server.getUrl() + "/myreg", LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/myreg/modules/foo/1.0/MODULE.bazel"); @@ -406,7 +425,9 @@ public void testGetModuleFileChecksumMismatch() throws Exception { ImmutableMap.of( server.getUrl() + "/myreg/modules/foo/1.0/MODULE.bazel", Optional.of(sha256("original"))); - Registry registry = registryFactory.createRegistry(server.getUrl() + "/myreg", knownFiles); + Registry registry = + registryFactory.createRegistry( + server.getUrl() + "/myreg", LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); var e = assertThrows( IOException.class, @@ -445,7 +466,9 @@ public void testGetRepoSpecChecksum() throws Exception { var knownFiles = ImmutableMap.of( server.getUrl() + "/modules/foo/2.0/source.json", Optional.of(sha256("unused"))); - Registry registry = registryFactory.createRegistry(server.getUrl(), knownFiles); + Registry registry = + registryFactory.createRegistry( + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); assertThat(registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)) .isEqualTo( RepoSpec.builder() @@ -463,7 +486,9 @@ public void testGetRepoSpecChecksum() throws Exception { server.getUrl() + "/modules/foo/1.0/source.json", Optional.of(sha256(sourceJson).toString())); - registry = registryFactory.createRegistry(server.getUrl(), recordedChecksums); + registry = + registryFactory.createRegistry( + server.getUrl(), LockfileMode.UPDATE, recordedChecksums, ImmutableMap.of()); // Test that the recorded hashes are used for repo cache hits even when the server content // changes. server.unserve("/bazel_registry.json"); @@ -505,7 +530,9 @@ public void testGetRepoSpecChecksumMismatch() throws Exception { Optional.of(sha256(registryJson)), server.getUrl() + "/modules/foo/1.0/source.json", Optional.of(sha256(sourceJson))); - Registry registry = registryFactory.createRegistry(server.getUrl(), knownFiles); + Registry registry = + registryFactory.createRegistry( + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); var e = assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); @@ -547,7 +574,9 @@ public void testBazelRegistryChecksumMismatch() throws Exception { Optional.of(sha256(registryJson)), server.getUrl() + "/modules/foo/1.0/source.json", Optional.of(sha256(sourceJson))); - Registry registry = registryFactory.createRegistry(server.getUrl(), knownFiles); + Registry registry = + registryFactory.createRegistry( + server.getUrl(), LockfileMode.UPDATE, knownFiles, ImmutableMap.of()); var e = assertThrows( IOException.class, () -> registry.getRepoSpec(createModuleKey("foo", "1.0"), reporter)); diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java index abacd9ee3006fe..1d38f1bbed43df 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleExtensionResolutionTest.java @@ -268,7 +268,9 @@ public void setup() throws Exception { .put( SkyFunctions.SINGLE_EXTENSION_EVAL, new SingleExtensionEvalFunction(directories, ImmutableMap::of, downloadManager)) - .put(SkyFunctions.REGISTRY, new RegistryFunction(registryFactory)) + .put( + SkyFunctions.REGISTRY, + new RegistryFunction(registryFactory, directories.getWorkspace())) .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) .put(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction()) .put( diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java index bf4d985c03e506..7908509f7e2e56 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/ModuleFileFunctionTest.java @@ -32,6 +32,7 @@ import com.google.devtools.build.lib.analysis.util.AnalysisMock; import com.google.devtools.build.lib.bazel.bzlmod.BzlmodTestUtil.InterimModuleBuilder; import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.starlark.StarlarkRepositoryModule; import com.google.devtools.build.lib.clock.BlazeClock; import com.google.devtools.build.lib.cmdline.RepositoryName; @@ -177,7 +178,9 @@ private void setUpWithBuiltinModules(ImmutableMap b .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) - .put(SkyFunctions.REGISTRY, new RegistryFunction(registryFactory)) + .put( + SkyFunctions.REGISTRY, + new RegistryFunction(registryFactory, directories.getWorkspace())) .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) .put(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction()) .put( @@ -204,6 +207,7 @@ private void setUpWithBuiltinModules(ImmutableMap b ModuleFileFunction.IGNORE_DEV_DEPS.set(differencer, false); ModuleFileFunction.MODULE_OVERRIDES.set(differencer, ImmutableMap.of()); YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.set(differencer, ImmutableList.of()); + BazelLockFileFunction.LOCKFILE_MODE.set(differencer, LockfileMode.UPDATE); } @Test @@ -753,13 +757,15 @@ public void testModuleExtensions_good() throws Exception { .setExtensionName("myext1") .setIsolationKey(Optional.empty()) .setUsingModule(myMod) - .setLocation( - Location.fromFileLineColumn( - "fake:0/modules/mymod/1.0/MODULE.bazel", 2, 23)) - .setImports(ImmutableBiMap.of("repo1", "repo1")) - .setDevImports(ImmutableSet.of()) - .setHasDevUseExtension(false) - .setHasNonDevUseExtension(true) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 2, 23)) + .setDevDependency(false) + .setProxyName("myext1") + .setImports(ImmutableBiMap.of("repo1", "repo1")) + .build()) .addTag( Tag.builder() .setTagName("tag") @@ -780,13 +786,16 @@ public void testModuleExtensions_good() throws Exception { .setExtensionName("myext2") .setIsolationKey(Optional.empty()) .setUsingModule(myMod) - .setLocation( - Location.fromFileLineColumn( - "fake:0/modules/mymod/1.0/MODULE.bazel", 5, 23)) - .setImports(ImmutableBiMap.of("other_repo1", "repo1", "repo2", "repo2")) - .setDevImports(ImmutableSet.of()) - .setHasDevUseExtension(false) - .setHasNonDevUseExtension(true) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 5, 23)) + .setDevDependency(false) + .setProxyName("myext2") + .setImports( + ImmutableBiMap.of("other_repo1", "repo1", "repo2", "repo2")) + .build()) .addTag( Tag.builder() .setTagName("tag1") @@ -820,14 +829,17 @@ public void testModuleExtensions_good() throws Exception { .setExtensionName("maven") .setIsolationKey(Optional.empty()) .setUsingModule(myMod) - .setLocation( - Location.fromFileLineColumn( - "fake:0/modules/mymod/1.0/MODULE.bazel", 10, 22)) - .setImports( - ImmutableBiMap.of("mvn", "maven", "junit", "junit", "guava", "guava")) - .setDevImports(ImmutableSet.of()) - .setHasDevUseExtension(false) - .setHasNonDevUseExtension(true) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 10, 22)) + .setDevDependency(false) + .setProxyName("maven") + .setImports( + ImmutableBiMap.of( + "mvn", "maven", "junit", "junit", "guava", "guava")) + .build()) .addTag( Tag.builder() .setTagName("dep") @@ -893,14 +905,38 @@ public void testModuleExtensions_duplicateProxy_asRoot() throws Exception { .setExtensionName("myext") .setIsolationKey(Optional.empty()) .setUsingModule(ModuleKey.ROOT) - .setLocation(Location.fromFileLineColumn("/workspace/MODULE.bazel", 1, 23)) - .setImports( - ImmutableBiMap.of( - "alpha", "alpha", "beta", "beta", "gamma", "gamma", "delta", - "delta")) - .setDevImports(ImmutableSet.of("alpha", "gamma")) - .setHasDevUseExtension(true) - .setHasNonDevUseExtension(true) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn("/workspace/MODULE.bazel", 1, 23)) + .setDevDependency(true) + .setProxyName("myext1") + .addImport("alpha", "alpha") + .build()) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn("/workspace/MODULE.bazel", 4, 23)) + .setDevDependency(false) + .setProxyName("myext2") + .addImport("beta", "beta") + .build()) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn("/workspace/MODULE.bazel", 7, 23)) + .setDevDependency(true) + .setProxyName("myext3") + .addImport("gamma", "gamma") + .build()) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn("/workspace/MODULE.bazel", 10, 23)) + .setDevDependency(false) + .setProxyName("myext4") + .addImport("delta", "delta") + .build()) .addTag( Tag.builder() .setTagName("tag") @@ -993,13 +1029,24 @@ public void testModuleExtensions_duplicateProxy_asDep() throws Exception { .setExtensionName("myext") .setIsolationKey(Optional.empty()) .setUsingModule(myMod) - .setLocation( - Location.fromFileLineColumn( - "fake:0/modules/mymod/1.0/MODULE.bazel", 5, 23)) - .setImports(ImmutableBiMap.of("beta", "beta", "delta", "delta")) - .setDevImports(ImmutableSet.of()) - .setHasDevUseExtension(false) - .setHasNonDevUseExtension(true) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 5, 23)) + .setDevDependency(false) + .setProxyName("myext2") + .addImport("beta", "beta") + .build()) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn( + "fake:0/modules/mymod/1.0/MODULE.bazel", 11, 23)) + .setDevDependency(false) + .setProxyName("myext4") + .addImport("delta", "delta") + .build()) .addTag( Tag.builder() .setTagName("tag") @@ -1098,13 +1145,27 @@ public void testModuleExtensions_innate() throws Exception { .setExtensionName("_repo_rules") .setIsolationKey(Optional.empty()) .setUsingModule(ModuleKey.ROOT) - .setLocation(Location.fromFile("/workspace/MODULE.bazel")) - .setImports( - ImmutableBiMap.of( - "repo_name", "repo_name", "guava", "guava", "vuaga", "vuaga")) - .setDevImports(ImmutableSet.of("vuaga")) - .setHasDevUseExtension(true) - .setHasNonDevUseExtension(true) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn("/workspace/MODULE.bazel", 2, 5)) + .setDevDependency(false) + .addImport("repo_name", "repo_name") + .build()) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn("/workspace/MODULE.bazel", 4, 13)) + .setDevDependency(false) + .addImport("guava", "guava") + .build()) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation( + Location.fromFileLineColumn("/workspace/MODULE.bazel", 5, 13)) + .setDevDependency(true) + .addImport("vuaga", "vuaga") + .build()) .addTag( Tag.builder() .setTagName("//:repo.bzl%repo") diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java index c56ad709836890..cc963f233e5e3e 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFactoryTest.java @@ -20,11 +20,10 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableMap; +import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode; import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache; import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager; import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader; -import com.google.devtools.build.lib.testutil.FoundationTestCase; -import com.google.devtools.build.lib.vfs.Path; import java.net.URISyntaxException; import org.junit.Test; import org.junit.runner.RunWith; @@ -32,34 +31,34 @@ /** Tests for {@link RegistryFactory}. */ @RunWith(JUnit4.class) -public class RegistryFactoryTest extends FoundationTestCase { +public class RegistryFactoryTest { @Test - public void badSchemes() throws Exception { - Path workspaceRoot = scratch.dir("/ws"); + public void badSchemes() { RegistryFactory registryFactory = new RegistryFactoryImpl( - workspaceRoot, new DownloadManager(new RepositoryCache(), new HttpDownloader()), Suppliers.ofInstance(ImmutableMap.of())); Throwable exception = assertThrows( URISyntaxException.class, - () -> registryFactory.createRegistry("/home/www", ImmutableMap.of())); + () -> + registryFactory.createRegistry( + "/home/www", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of())); assertThat(exception).hasMessageThat().contains("Registry URL has no scheme"); exception = assertThrows( URISyntaxException.class, - () -> registryFactory.createRegistry("foo://bar", ImmutableMap.of())); + () -> + registryFactory.createRegistry( + "foo://bar", LockfileMode.UPDATE, ImmutableMap.of(), ImmutableMap.of())); assertThat(exception).hasMessageThat().contains("Unrecognized registry URL protocol"); } @Test - public void badPath() throws Exception { - Path workspaceRoot = scratch.dir("/ws"); + public void badPath() { RegistryFactory registryFactory = new RegistryFactoryImpl( - workspaceRoot, new DownloadManager(new RepositoryCache(), new HttpDownloader()), Suppliers.ofInstance(ImmutableMap.of())); Throwable exception = @@ -67,7 +66,10 @@ public void badPath() throws Exception { URISyntaxException.class, () -> registryFactory.createRegistry( - "file:c:/path/to/workspace/registry", ImmutableMap.of())); + "file:c:/path/to/workspace/registry", + LockfileMode.UPDATE, + ImmutableMap.of(), + ImmutableMap.of())); assertThat(exception).hasMessageThat().contains("Registry URL path is not valid"); } } diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/StarlarkBazelModuleTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/StarlarkBazelModuleTest.java index e444e443814ff8..6fdb4f7afe1ee8 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/StarlarkBazelModuleTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/StarlarkBazelModuleTest.java @@ -23,10 +23,8 @@ import static com.google.devtools.build.lib.packages.Attribute.attr; import static org.junit.Assert.assertThrows; -import com.google.common.collect.ImmutableBiMap; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; import com.google.devtools.build.lib.cmdline.Label; import com.google.devtools.build.lib.packages.BuildType; import com.google.devtools.build.lib.packages.Type; @@ -48,12 +46,7 @@ private static ModuleExtensionUsage.Builder getBaseUsageBuilder() { .setExtensionBzlFile("//:rje.bzl") .setExtensionName("maven") .setIsolationKey(Optional.empty()) - .setUsingModule(ModuleKey.ROOT) - .setLocation(Location.BUILTIN) - .setImports(ImmutableBiMap.of()) - .setDevImports(ImmutableSet.of()) - .setHasDevUseExtension(false) - .setHasNonDevUseExtension(true); + .setUsingModule(ModuleKey.ROOT); } /** A builder for ModuleExtension that sets all the mandatory but irrelevant fields. */ diff --git a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModExecutorTest.java b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModExecutorTest.java index ab89f4a6f98891..c8d9518387dcb2 100644 --- a/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModExecutorTest.java +++ b/src/test/java/com/google/devtools/build/lib/bazel/bzlmod/modcommand/ModExecutorTest.java @@ -606,12 +606,13 @@ public void testExtensionsInfoTextAndGraph() throws Exception { ModuleExtensionUsage.builder() .setExtensionBzlFile("//extensions:extensions.bzl") .setExtensionName("maven") - .setLocation(Location.fromFileLineColumn("C@1.0/MODULE.bazel", 2, 23)) - .setImports(ImmutableBiMap.of("repo1", "repo1", "repo3", "repo3")) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation(Location.fromFileLineColumn("C@1.0/MODULE.bazel", 2, 23)) + .setImports(ImmutableBiMap.of("repo1", "repo1", "repo3", "repo3")) + .setDevDependency(false) + .build()) .setUsingModule(createModuleKey("C", "1.0")) - .setDevImports(ImmutableSet.of()) - .setHasDevUseExtension(false) - .setHasNonDevUseExtension(true) .build()) .put( mavenId, @@ -619,12 +620,13 @@ public void testExtensionsInfoTextAndGraph() throws Exception { ModuleExtensionUsage.builder() .setExtensionBzlFile("//extensions:extensions.bzl") .setExtensionName("maven") - .setLocation(Location.fromFileLineColumn("D@1.0/MODULE.bazel", 1, 10)) - .setImports(ImmutableBiMap.of("repo1", "repo1", "repo2", "repo2")) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation(Location.fromFileLineColumn("D@1.0/MODULE.bazel", 1, 10)) + .setImports(ImmutableBiMap.of("repo1", "repo1", "repo2", "repo2")) + .setDevDependency(false) + .build()) .setUsingModule(createModuleKey("D", "1.0")) - .setDevImports(ImmutableSet.of()) - .setHasDevUseExtension(false) - .setHasNonDevUseExtension(true) .build()) .put( gradleId, @@ -632,12 +634,13 @@ public void testExtensionsInfoTextAndGraph() throws Exception { ModuleExtensionUsage.builder() .setExtensionBzlFile("//extensions:extensions.bzl") .setExtensionName("gradle") - .setLocation(Location.fromFileLineColumn("Y@2.0/MODULE.bazel", 2, 13)) - .setImports(ImmutableBiMap.of("repo2", "repo2")) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation(Location.fromFileLineColumn("Y@2.0/MODULE.bazel", 2, 13)) + .setImports(ImmutableBiMap.of("repo2", "repo2")) + .setDevDependency(false) + .build()) .setUsingModule(createModuleKey("Y", "2.0")) - .setDevImports(ImmutableSet.of()) - .setHasDevUseExtension(false) - .setHasNonDevUseExtension(true) .build()) .put( mavenId, @@ -645,8 +648,12 @@ public void testExtensionsInfoTextAndGraph() throws Exception { ModuleExtensionUsage.builder() .setExtensionBzlFile("//extensions:extensions.bzl") .setExtensionName("maven") - .setLocation(Location.fromFileLineColumn("Y@2.0/MODULE.bazel", 13, 10)) - .setImports(ImmutableBiMap.of("myrepo", "repo5")) + .addProxy( + ModuleExtensionUsage.Proxy.builder() + .setLocation(Location.fromFileLineColumn("Y@2.0/MODULE.bazel", 13, 10)) + .setImports(ImmutableBiMap.of("myrepo", "repo5")) + .setDevDependency(false) + .build()) .addTag(buildTag("dep").addAttr("coord", "junit").build()) .addTag(buildTag("dep").addAttr("coord", "guava").build()) .addTag( @@ -656,9 +663,6 @@ public void testExtensionsInfoTextAndGraph() throws Exception { StarlarkList.immutableOf("//:pom.xml", "@bar//:pom.xml")) .build()) .setUsingModule(createModuleKey("Y", "2.0")) - .setDevImports(ImmutableSet.of()) - .setHasDevUseExtension(false) - .setHasNonDevUseExtension(true) .build()) .buildOrThrow(); diff --git a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java index b6f888e61eccdd..bc176a5b776d94 100644 --- a/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java +++ b/src/test/java/com/google/devtools/build/lib/rules/repository/RepositoryDelegatorTest.java @@ -264,7 +264,9 @@ public void setupDelegator() throws Exception { .put( BzlmodRepoRuleValue.BZLMOD_REPO_RULE, new BzlmodRepoRuleFunction(ruleClassProvider, directories)) - .put(SkyFunctions.REGISTRY, new RegistryFunction(registryFactory)) + .put( + SkyFunctions.REGISTRY, + new RegistryFunction(registryFactory, directories.getWorkspace())) .put(SkyFunctions.REPO_SPEC, new RepoSpecFunction()) .put(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction()) .put( diff --git a/src/test/java/com/google/devtools/build/lib/testutil/BUILD b/src/test/java/com/google/devtools/build/lib/testutil/BUILD index 9a9b4f834ca164..f5d1477482c697 100644 --- a/src/test/java/com/google/devtools/build/lib/testutil/BUILD +++ b/src/test/java/com/google/devtools/build/lib/testutil/BUILD @@ -1,9 +1,9 @@ # Description: # A grab-bag of testing utilities. -load("//src/tools/bzlmod:utils.bzl", "get_canonical_repo_name") load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_java//java:defs.bzl", "java_library", "java_test") +load("//src/tools/bzlmod:blazel_utils.bzl", "get_canonical_repo_name") package( default_applicable_licenses = ["//:license"], diff --git a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py index 6118fb8c2affbc..d96e3735a382f3 100644 --- a/src/test/py/bazel/bzlmod/bazel_lockfile_test.py +++ b/src/test/py/bazel/bzlmod/bazel_lockfile_test.py @@ -64,6 +64,32 @@ def tearDown(self): self.main_registry.stop() test_base.TestBase.tearDown(self) + def testInvalidLockfile(self): + self.ScratchFile('BUILD', ['filegroup(name = "hello")']) + self.RunBazel(['build', '//:all']) + + with open(self.Path('MODULE.bazel.lock'), 'r') as f: + lockfile = json.loads(f.read().strip()) + lockfile['registryFileHashes'] = 'I am a string' + + with open(self.Path('MODULE.bazel.lock'), 'w') as f: + f.write(json.dumps(lockfile)) + + exit_code, _, stderr = self.RunBazel( + ['build', '//:all'], allow_failure=True + ) + stderr = '\n'.join(stderr) + self.AssertExitCode(exit_code, 48, stderr) + self.assertRegex( + stderr, + ( + 'ERROR: Error computing the main repository mapping: Failed to read' + ' and parse the MODULE\\.bazel\\.lock file with error:' + ' java\\.lang\\.IllegalStateException: Expected BEGIN_OBJECT but' + ' was STRING .*\\. Try deleting it and rerun the build.' + ), + ) + def testChangeModuleInRegistryWithoutLockfile(self): # Add module 'sss' to the registry with dep on 'aaa' self.main_registry.createCcModule('sss', '1.3', {'aaa': '1.1'}) @@ -218,18 +244,18 @@ def testLockfileErrorMode(self): [ 'build', '--nobuild', - '--check_direct_dependencies=warning', '//:all', ], ) - # Run with updated module and a different flag - self.ScratchFile('MODULE.bazel', ['module(name="lala")']) + # Run with updated module + self.ScratchFile( + 'MODULE.bazel', ['bazel_dep(name="does_not_exist",version="0")'] + ) exit_code, _, stderr = self.RunBazel( [ 'build', '--nobuild', - '--check_direct_dependencies=error', '--lockfile_mode=error', '//:all', ], @@ -239,58 +265,16 @@ def testLockfileErrorMode(self): self.assertIn( ( 'ERROR: Error computing the main repository mapping:' - ' MODULE.bazel.lock is no longer up-to-date because: the root' - ' MODULE.bazel has been modified, the value of' - ' --check_direct_dependencies flag has been modified. Please run' + ' Missing checksum for registry file {}'.format( + self.main_registry.getURL() + ) + + '/modules/does_not_exist/0/MODULE.bazel not permitted with' + ' --lockfile_mode=error. Please run' ' `bazel mod deps --lockfile_mode=update` to update your lockfile.' ), stderr, ) - def testLocalOverrideWithErrorMode(self): - self.ScratchFile( - 'MODULE.bazel', - [ - 'module(name="lala")', - 'bazel_dep(name="bar")', - 'local_path_override(module_name="bar",path="bar")', - ], - ) - self.ScratchFile('BUILD', ['filegroup(name = "hello")']) - self.ScratchFile('bar/MODULE.bazel', ['module(name="bar")']) - self.ScratchFile('bar/BUILD', ['filegroup(name = "hello from bar")']) - self.RunBazel( - [ - 'build', - '--nobuild', - '//:all', - ], - ) - - # Run with updated module and a different flag - self.ScratchFile( - 'bar/MODULE.bazel', - [ - 'module(name="bar")', - 'bazel_dep(name="hmmm")', - ], - ) - exit_code, _, stderr = self.RunBazel( - ['build', '--nobuild', '--lockfile_mode=error', '//:all'], - allow_failure=True, - ) - self.AssertExitCode(exit_code, 48, stderr) - self.assertIn( - ( - 'ERROR: Error computing the main repository mapping:' - ' MODULE.bazel.lock is no longer up-to-date because: The' - ' MODULE.bazel file has changed for the overriden module: bar.' - ' Please run `bazel mod deps --lockfile_mode=update` to update your' - ' lockfile.' - ), - stderr, - ) - def testModuleExtension(self): self.ScratchFile( 'MODULE.bazel', @@ -493,7 +477,6 @@ def testModuleExtensionsInDifferentBuilds(self): with open(self.Path('MODULE.bazel.lock'), 'r') as f: lockfile = json.loads(f.read().strip()) - self.assertGreater(len(lockfile['moduleDepGraph']), 0) ext_keys = list(lockfile['moduleExtensions'].keys()) self.assertIn('//:extension.bzl%extA', ext_keys) self.assertIn('//:extension.bzl%extB', ext_keys) @@ -863,10 +846,10 @@ def testOldVersion(self): self.AssertExitCode(exit_code, 48, stderr) self.assertIn( ( - 'ERROR: Error computing the main repository mapping:' - ' MODULE.bazel.lock is no longer up-to-date because: the version of' - ' the lockfile is not compatible with the current Bazel. Please run' - ' `bazel mod deps --lockfile_mode=update` to update your lockfile.' + 'ERROR: Error computing the main repository mapping: The version' + ' of MODULE.bazel.lock is not supported by this version of Bazel.' + ' Please run `bazel mod deps --lockfile_mode=update` to update your' + ' lockfile.' ), stderr, ) @@ -1335,9 +1318,10 @@ def testExtensionEvaluationOnlyRerunOnRelevantUsagesChanges(self): '', 'def _ext_1_impl(ctx):', ' print("Ext 1 is being evaluated")', - ' num_tags = len([', - ' tag for mod in ctx.modules for tag in mod.tags.tag', - ' ])', + ( + ' num_tags = len([tag for mod in ctx.modules for tag in' + ' mod.tags.tag])' + ), ' repo_rule(name="dep", value="Ext 1 saw %s tags" % num_tags)', '', 'ext_1 = module_extension(', @@ -1347,9 +1331,10 @@ def testExtensionEvaluationOnlyRerunOnRelevantUsagesChanges(self): '', 'def _ext_2_impl(ctx):', ' print("Ext 2 is being evaluated")', - ' num_tags = len([', - ' tag for mod in ctx.modules for tag in mod.tags.tag', - ' ])', + ( + ' num_tags = len([tag for mod in ctx.modules for tag in' + ' mod.tags.tag])' + ), ' repo_rule(name="dep", value="Ext 2 saw %s tags" % num_tags)', '', 'ext_2 = module_extension(', @@ -1359,9 +1344,10 @@ def testExtensionEvaluationOnlyRerunOnRelevantUsagesChanges(self): '', 'def _ext_3_impl(ctx):', ' print("Ext 3 is being evaluated")', - ' num_tags = len([', - ' tag for mod in ctx.modules for tag in mod.tags.tag', - ' ])', + ( + ' num_tags = len([tag for mod in ctx.modules for tag in' + ' mod.tags.tag])' + ), ' repo_rule(name="dep", value="Ext 3 saw %s tags" % num_tags)', '', 'ext_3 = module_extension(', @@ -1536,17 +1522,9 @@ def testLockfileWithNoUserSpecificPath(self): ['build', '--registry=file:///%workspace%/registry', '//:lala'] ) - with open('MODULE.bazel.lock', 'r') as json_file: - lockfile = json.load(json_file) - ss_dep = lockfile['moduleDepGraph']['ss@1.3-1'] - remote_patches = ss_dep['repoSpec']['attributes']['remote_patches'] - ext_usage_location = ss_dep['extensionUsages'][0]['location']['file'] - - self.assertNotIn(self.my_registry.getURL(), ext_usage_location) - self.assertIn('%workspace%', ext_usage_location) - for key in remote_patches.keys(): - self.assertNotIn(self.my_registry.getURL(), key) - self.assertIn('%workspace%', key) + with open('MODULE.bazel.lock', 'r') as f: + self.assertNotIn(self.my_registry.getURL(), f.read()) + finally: self.my_registry.stop() diff --git a/src/test/py/bazel/bzlmod/bazel_yanked_versions_test.py b/src/test/py/bazel/bzlmod/bazel_yanked_versions_test.py index 352e7b948edd02..3cf795726b621c 100644 --- a/src/test/py/bazel/bzlmod/bazel_yanked_versions_test.py +++ b/src/test/py/bazel/bzlmod/bazel_yanked_versions_test.py @@ -16,6 +16,7 @@ import os import tempfile + from absl.testing import absltest from src.test.py.bazel import test_base from src.test.py.bazel.bzlmod.test_utils import BazelRegistry @@ -40,6 +41,8 @@ def setUp(self): 'ddd', '1.0', {'yanked1': '1.0', 'yanked2': '1.0'} ).createCcModule( 'eee', '1.0', {'yanked1': '1.0'} + ).createCcModule( + 'fff', '1.0' ).createCcModule( 'yanked1', '1.0' ).createCcModule( @@ -199,6 +202,42 @@ def testAllowedYankedDepsByEnvVar(self): ''.join(stderr), ) + def testAllowedYankedDepsByEnvVarErrorMode(self): + self.writeBazelrcFile(allow_yanked_versions=False) + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "ddd", version = "1.0")', + ], + ) + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = ["@ddd//:lib_ddd"],', + ')', + ], + ) + self.RunBazel( + ['build', '--nobuild', '//:main'], + env_add={'BZLMOD_ALLOW_YANKED_VERSIONS': 'yanked1@1.0,yanked2@1.0'}, + ) + + # Test changing the env var, the build should fail again. + exit_code, _, stderr = self.RunBazel( + ['build', '--nobuild', '--lockfile_mode=error', '//:main'], + env_add={'BZLMOD_ALLOW_YANKED_VERSIONS': 'yanked2@1.0'}, + allow_failure=True, + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + 'Yanked version detected in your resolved dependency graph: ' + + 'yanked1@1.0, for the reason: dodgy.', + ''.join(stderr), + ) + def testAllowedYankedDepsSuccessMix(self): self.writeBazelrcFile(allow_yanked_versions=False) self.ScratchFile( @@ -227,6 +266,70 @@ def testAllowedYankedDepsSuccessMix(self): env_add={'BZLMOD_ALLOW_YANKED_VERSIONS': 'yanked2@1.0'}, ) + def testYankedVersionsFetchedIncrementally(self): + self.writeBazelrcFile(allow_yanked_versions=False) + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "aaa", version = "1.0")', + ], + ) + self.ScratchFile( + 'BUILD', + [ + 'cc_binary(', + ' name = "main",', + ' srcs = ["main.cc"],', + ' deps = ["@aaa//:lib_aaa"],', + ')', + ], + ) + self.RunBazel(['build', '--nobuild', '//:main']) + + # Yank aaa@1.0 and aaa@1.1. + self.main_registry.addMetadata( + 'aaa', yanked_versions={'1.0': 'already dodgy', '1.1': 'still dodgy'} + ) + + # Without any changes, both a cold and a warm build still pass. + self.RunBazel(['build', '--nobuild', '//:main']) + self.RunBazel(['shutdown']) + self.RunBazel(['build', '--nobuild', '//:main']) + + # Adding an unrelated dependency should not cause yanked versions to be + # fetched again. + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "aaa", version = "1.0")', + 'bazel_dep(name = "fff", version = "1.0")', + ], + ) + self.RunBazel(['build', '--nobuild', '//:main']) + self.RunBazel(['shutdown']) + self.RunBazel(['build', '--nobuild', '//:main']) + + # If a new version of aaa is selected, yanked versions should be fetched + # again. + self.ScratchFile( + 'MODULE.bazel', + [ + 'bazel_dep(name = "aaa", version = "1.0")', + 'bazel_dep(name = "fff", version = "1.0")', + # Depends on aaa@1.1. + 'bazel_dep(name = "bbb", version = "1.1")', + ], + ) + exit_code, _, stderr = self.RunBazel( + ['build', '--nobuild', '//:main'], allow_failure=True + ) + self.AssertExitCode(exit_code, 48, stderr) + self.assertIn( + 'Yanked version detected in your resolved dependency graph: ' + + 'aaa@1.1, for the reason: still dodgy.', + ''.join(stderr), + ) + if __name__ == '__main__': absltest.main() diff --git a/src/test/shell/bazel/remote_helpers.sh b/src/test/shell/bazel/remote_helpers.sh index e8181d210ff7db..14bbb63f36ac67 100755 --- a/src/test/shell/bazel/remote_helpers.sh +++ b/src/test/shell/bazel/remote_helpers.sh @@ -294,7 +294,15 @@ function setup_credential_helper() { cat > "${TEST_TMPDIR}/credhelper" <<'EOF' #!/usr/bin/env python3 +import json import os +import sys + +# Neither count nor add headers to requests to the BCR. +uri = json.load(sys.stdin)["uri"] +if uri.startswith("https://bcr.bazel.build/"): + print("{}") + sys.exit(0) path = os.path.join(os.environ["TEST_TMPDIR"], "credhelper.callcount") fd = os.open(path, os.O_WRONLY|os.O_CREAT|os.O_APPEND) diff --git a/src/test/shell/bazel/starlark_repository_test.sh b/src/test/shell/bazel/starlark_repository_test.sh index 5742bb5ad35629..a53803f14d8486 100755 --- a/src/test/shell/bazel/starlark_repository_test.sh +++ b/src/test/shell/bazel/starlark_repository_test.sh @@ -2327,9 +2327,6 @@ genrule( ) EOF - # Ensure that all Bzlmod-related downloads have happened before disabling - # downloads. - bazel mod deps || fail "Failed to cache Bazel modules" bazel build --repository_disable_download //:it > "${TEST_log}" 2>&1 \ && fail "Expected failure" || : expect_log "Failed to download repository @.*: download is disabled" diff --git a/src/test/tools/bzlmod/BUILD b/src/test/tools/bzlmod/BUILD index ccaa79a657c45e..a0bbf6bc15bbbb 100644 --- a/src/test/tools/bzlmod/BUILD +++ b/src/test/tools/bzlmod/BUILD @@ -23,6 +23,5 @@ sh_test( "//src:bazel", "//src/test/shell/bazel:test-deps", ], - tags = ["requires-network"], deps = ["@bazel_tools//tools/bash/runfiles"], ) diff --git a/src/test/tools/bzlmod/MODULE.bazel.lock b/src/test/tools/bzlmod/MODULE.bazel.lock index 2f9dd37c7da39f..9c1ffa728df6ef 100644 --- a/src/test/tools/bzlmod/MODULE.bazel.lock +++ b/src/test/tools/bzlmod/MODULE.bazel.lock @@ -1,1062 +1,5 @@ { - "lockFileVersion": 8, - "moduleFileHash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", - "flags": { - "cmdRegistries": [ - "https://bcr.bazel.build/" - ], - "cmdModuleOverrides": {}, - "allowedYankedVersions": [], - "envVarAllowedYankedVersions": "", - "ignoreDevDependency": false, - "directDependenciesMode": "WARNING", - "compatibilityMode": "ERROR" - }, - "localOverrideHashes": { - "bazel_tools": "2db28a52a5aed3ab881eae06856ddfbe3d47c08108fd8002e0f8e615764f4f76" - }, - "moduleDepGraph": { - "": { - "name": "", - "version": "", - "key": "", - "repoName": "", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - } - }, - "bazel_tools@_": { - "name": "bazel_tools", - "version": "", - "key": "bazel_tools@_", - "repoName": "bazel_tools", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "@local_config_cc_toolchains//:all", - "@local_config_sh//:local_sh_toolchain" - ], - "extensionUsages": [ - { - "extensionBzlFile": "@bazel_tools//tools/cpp:cc_configure.bzl", - "extensionName": "cc_configure_extension", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 18, - "column": 29 - }, - "imports": { - "local_config_cc": "local_config_cc", - "local_config_cc_toolchains": "local_config_cc_toolchains" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@bazel_tools//tools/osx:xcode_configure.bzl", - "extensionName": "xcode_configure_extension", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 22, - "column": 32 - }, - "imports": { - "local_config_xcode": "local_config_xcode" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@rules_java//java:extensions.bzl", - "extensionName": "toolchains", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 25, - "column": 32 - }, - "imports": { - "local_jdk": "local_jdk", - "remote_java_tools": "remote_java_tools", - "remote_java_tools_linux": "remote_java_tools_linux", - "remote_java_tools_windows": "remote_java_tools_windows", - "remote_java_tools_darwin_x86_64": "remote_java_tools_darwin_x86_64", - "remote_java_tools_darwin_arm64": "remote_java_tools_darwin_arm64" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@bazel_tools//tools/sh:sh_configure.bzl", - "extensionName": "sh_configure_extension", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 36, - "column": 39 - }, - "imports": { - "local_config_sh": "local_config_sh" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@bazel_tools//tools/test:extensions.bzl", - "extensionName": "remote_coverage_tools_extension", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 40, - "column": 48 - }, - "imports": { - "remote_coverage_tools": "remote_coverage_tools" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@bazel_tools//tools/android:android_extensions.bzl", - "extensionName": "remote_android_tools_extensions", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 43, - "column": 42 - }, - "imports": { - "android_gmaven_r8": "android_gmaven_r8", - "android_tools": "android_tools" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@buildozer//:buildozer_binary.bzl", - "extensionName": "buildozer_binary", - "usingModule": "bazel_tools@_", - "location": { - "file": "@@bazel_tools//:MODULE.bazel", - "line": 47, - "column": 33 - }, - "imports": { - "buildozer_binary": "buildozer_binary" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "rules_cc": "rules_cc@0.0.9", - "rules_java": "rules_java@7.5.0", - "rules_license": "rules_license@0.0.7", - "rules_proto": "rules_proto@5.3.0-21.7", - "rules_python": "rules_python@0.22.1", - "buildozer": "buildozer@7.1.1.1", - "platforms": "platforms@0.0.9", - "com_google_protobuf": "protobuf@21.7", - "zlib": "zlib@1.3", - "build_bazel_apple_support": "apple_support@1.5.0", - "local_config_platform": "local_config_platform@_" - } - }, - "local_config_platform@_": { - "name": "local_config_platform", - "version": "", - "key": "local_config_platform@_", - "repoName": "local_config_platform", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "platforms": "platforms@0.0.9", - "bazel_tools": "bazel_tools@_" - } - }, - "rules_cc@0.0.9": { - "name": "rules_cc", - "version": "0.0.9", - "key": "rules_cc@0.0.9", - "repoName": "rules_cc", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "@local_config_cc_toolchains//:all" - ], - "extensionUsages": [ - { - "extensionBzlFile": "@bazel_tools//tools/cpp:cc_configure.bzl", - "extensionName": "cc_configure_extension", - "usingModule": "rules_cc@0.0.9", - "location": { - "file": "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel", - "line": 9, - "column": 29 - }, - "imports": { - "local_config_cc_toolchains": "local_config_cc_toolchains" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "platforms": "platforms@0.0.9", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/rules_cc/releases/download/0.0.9/rules_cc-0.0.9.tar.gz" - ], - "integrity": "sha256-IDeHW5pEVtzkp50RKorohbvEqtlo5lh9ym5k86CQDN8=", - "strip_prefix": "rules_cc-0.0.9", - "remote_patches": { - "https://bcr.bazel.build/modules/rules_cc/0.0.9/patches/module_dot_bazel_version.patch": "sha256-mM+qzOI0SgAdaJBlWOSMwMPKpaA9b7R37Hj/tp5bb4g=" - }, - "remote_patch_strip": 0 - } - } - }, - "rules_java@7.5.0": { - "name": "rules_java", - "version": "7.5.0", - "key": "rules_java@7.5.0", - "repoName": "rules_java", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "//toolchains:all", - "@local_jdk//:runtime_toolchain_definition", - "@local_jdk//:bootstrap_runtime_toolchain_definition", - "@remotejdk11_linux_toolchain_config_repo//:all", - "@remotejdk11_linux_aarch64_toolchain_config_repo//:all", - "@remotejdk11_linux_ppc64le_toolchain_config_repo//:all", - "@remotejdk11_linux_s390x_toolchain_config_repo//:all", - "@remotejdk11_macos_toolchain_config_repo//:all", - "@remotejdk11_macos_aarch64_toolchain_config_repo//:all", - "@remotejdk11_win_toolchain_config_repo//:all", - "@remotejdk11_win_arm64_toolchain_config_repo//:all", - "@remotejdk17_linux_toolchain_config_repo//:all", - "@remotejdk17_linux_aarch64_toolchain_config_repo//:all", - "@remotejdk17_linux_ppc64le_toolchain_config_repo//:all", - "@remotejdk17_linux_s390x_toolchain_config_repo//:all", - "@remotejdk17_macos_toolchain_config_repo//:all", - "@remotejdk17_macos_aarch64_toolchain_config_repo//:all", - "@remotejdk17_win_toolchain_config_repo//:all", - "@remotejdk17_win_arm64_toolchain_config_repo//:all", - "@remotejdk21_linux_toolchain_config_repo//:all", - "@remotejdk21_linux_aarch64_toolchain_config_repo//:all", - "@remotejdk21_linux_ppc64le_toolchain_config_repo//:all", - "@remotejdk21_linux_s390x_toolchain_config_repo//:all", - "@remotejdk21_macos_toolchain_config_repo//:all", - "@remotejdk21_macos_aarch64_toolchain_config_repo//:all", - "@remotejdk21_win_toolchain_config_repo//:all", - "@remotejdk21_win_arm64_toolchain_config_repo//:all" - ], - "extensionUsages": [ - { - "extensionBzlFile": "@rules_java//java:extensions.bzl", - "extensionName": "toolchains", - "usingModule": "rules_java@7.5.0", - "location": { - "file": "https://bcr.bazel.build/modules/rules_java/7.5.0/MODULE.bazel", - "line": 19, - "column": 27 - }, - "imports": { - "remote_java_tools": "remote_java_tools", - "remote_java_tools_linux": "remote_java_tools_linux", - "remote_java_tools_windows": "remote_java_tools_windows", - "remote_java_tools_darwin_x86_64": "remote_java_tools_darwin_x86_64", - "remote_java_tools_darwin_arm64": "remote_java_tools_darwin_arm64", - "local_jdk": "local_jdk", - "remotejdk11_linux_toolchain_config_repo": "remotejdk11_linux_toolchain_config_repo", - "remotejdk11_linux_aarch64_toolchain_config_repo": "remotejdk11_linux_aarch64_toolchain_config_repo", - "remotejdk11_linux_ppc64le_toolchain_config_repo": "remotejdk11_linux_ppc64le_toolchain_config_repo", - "remotejdk11_linux_s390x_toolchain_config_repo": "remotejdk11_linux_s390x_toolchain_config_repo", - "remotejdk11_macos_toolchain_config_repo": "remotejdk11_macos_toolchain_config_repo", - "remotejdk11_macos_aarch64_toolchain_config_repo": "remotejdk11_macos_aarch64_toolchain_config_repo", - "remotejdk11_win_toolchain_config_repo": "remotejdk11_win_toolchain_config_repo", - "remotejdk11_win_arm64_toolchain_config_repo": "remotejdk11_win_arm64_toolchain_config_repo", - "remotejdk17_linux_toolchain_config_repo": "remotejdk17_linux_toolchain_config_repo", - "remotejdk17_linux_aarch64_toolchain_config_repo": "remotejdk17_linux_aarch64_toolchain_config_repo", - "remotejdk17_linux_ppc64le_toolchain_config_repo": "remotejdk17_linux_ppc64le_toolchain_config_repo", - "remotejdk17_linux_s390x_toolchain_config_repo": "remotejdk17_linux_s390x_toolchain_config_repo", - "remotejdk17_macos_toolchain_config_repo": "remotejdk17_macos_toolchain_config_repo", - "remotejdk17_macos_aarch64_toolchain_config_repo": "remotejdk17_macos_aarch64_toolchain_config_repo", - "remotejdk17_win_toolchain_config_repo": "remotejdk17_win_toolchain_config_repo", - "remotejdk17_win_arm64_toolchain_config_repo": "remotejdk17_win_arm64_toolchain_config_repo", - "remotejdk21_linux_toolchain_config_repo": "remotejdk21_linux_toolchain_config_repo", - "remotejdk21_linux_aarch64_toolchain_config_repo": "remotejdk21_linux_aarch64_toolchain_config_repo", - "remotejdk21_linux_ppc64le_toolchain_config_repo": "remotejdk21_linux_ppc64le_toolchain_config_repo", - "remotejdk21_linux_s390x_toolchain_config_repo": "remotejdk21_linux_s390x_toolchain_config_repo", - "remotejdk21_macos_toolchain_config_repo": "remotejdk21_macos_toolchain_config_repo", - "remotejdk21_macos_aarch64_toolchain_config_repo": "remotejdk21_macos_aarch64_toolchain_config_repo", - "remotejdk21_win_toolchain_config_repo": "remotejdk21_win_toolchain_config_repo", - "remotejdk21_win_arm64_toolchain_config_repo": "remotejdk21_win_arm64_toolchain_config_repo" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "platforms": "platforms@0.0.9", - "rules_cc": "rules_cc@0.0.9", - "bazel_skylib": "bazel_skylib@1.3.0", - "rules_proto": "rules_proto@5.3.0-21.7", - "rules_license": "rules_license@0.0.7", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/rules_java/releases/download/7.5.0/rules_java-7.5.0.tar.gz" - ], - "integrity": "sha256-TaN2H2hVrZFlaOK/6GITum0mN/Vrg2BTin+2Elq/ZRg=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, - "rules_license@0.0.7": { - "name": "rules_license", - "version": "0.0.7", - "key": "rules_license@0.0.7", - "repoName": "rules_license", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/rules_license/releases/download/0.0.7/rules_license-0.0.7.tar.gz" - ], - "integrity": "sha256-RTHezLkTY5ww5cdRKgVNXYdWmNrrddjPkPKEN1/nw2A=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, - "rules_proto@5.3.0-21.7": { - "name": "rules_proto", - "version": "5.3.0-21.7", - "key": "rules_proto@5.3.0-21.7", - "repoName": "rules_proto", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "bazel_skylib": "bazel_skylib@1.3.0", - "com_google_protobuf": "protobuf@21.7", - "rules_cc": "rules_cc@0.0.9", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/rules_proto/archive/refs/tags/5.3.0-21.7.tar.gz" - ], - "integrity": "sha256-3D+yBqLLNEG0heseQjFlsjEjWh6psDG0Qzz3vB+kYN0=", - "strip_prefix": "rules_proto-5.3.0-21.7", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, - "rules_python@0.22.1": { - "name": "rules_python", - "version": "0.22.1", - "key": "rules_python@0.22.1", - "repoName": "rules_python", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "@bazel_tools//tools/python:autodetecting_toolchain" - ], - "extensionUsages": [ - { - "extensionBzlFile": "@rules_python//python/extensions/private:internal_deps.bzl", - "extensionName": "internal_deps", - "usingModule": "rules_python@0.22.1", - "location": { - "file": "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel", - "line": 14, - "column": 30 - }, - "imports": { - "pypi__build": "pypi__build", - "pypi__click": "pypi__click", - "pypi__colorama": "pypi__colorama", - "pypi__importlib_metadata": "pypi__importlib_metadata", - "pypi__installer": "pypi__installer", - "pypi__more_itertools": "pypi__more_itertools", - "pypi__packaging": "pypi__packaging", - "pypi__pep517": "pypi__pep517", - "pypi__pip": "pypi__pip", - "pypi__pip_tools": "pypi__pip_tools", - "pypi__setuptools": "pypi__setuptools", - "pypi__tomli": "pypi__tomli", - "pypi__wheel": "pypi__wheel", - "pypi__zipp": "pypi__zipp", - "pypi__coverage_cp310_aarch64-apple-darwin": "pypi__coverage_cp310_aarch64-apple-darwin", - "pypi__coverage_cp310_aarch64-unknown-linux-gnu": "pypi__coverage_cp310_aarch64-unknown-linux-gnu", - "pypi__coverage_cp310_x86_64-apple-darwin": "pypi__coverage_cp310_x86_64-apple-darwin", - "pypi__coverage_cp310_x86_64-unknown-linux-gnu": "pypi__coverage_cp310_x86_64-unknown-linux-gnu", - "pypi__coverage_cp311_aarch64-unknown-linux-gnu": "pypi__coverage_cp311_aarch64-unknown-linux-gnu", - "pypi__coverage_cp311_x86_64-apple-darwin": "pypi__coverage_cp311_x86_64-apple-darwin", - "pypi__coverage_cp311_x86_64-unknown-linux-gnu": "pypi__coverage_cp311_x86_64-unknown-linux-gnu", - "pypi__coverage_cp38_aarch64-apple-darwin": "pypi__coverage_cp38_aarch64-apple-darwin", - "pypi__coverage_cp38_aarch64-unknown-linux-gnu": "pypi__coverage_cp38_aarch64-unknown-linux-gnu", - "pypi__coverage_cp38_x86_64-apple-darwin": "pypi__coverage_cp38_x86_64-apple-darwin", - "pypi__coverage_cp38_x86_64-unknown-linux-gnu": "pypi__coverage_cp38_x86_64-unknown-linux-gnu", - "pypi__coverage_cp39_aarch64-apple-darwin": "pypi__coverage_cp39_aarch64-apple-darwin", - "pypi__coverage_cp39_aarch64-unknown-linux-gnu": "pypi__coverage_cp39_aarch64-unknown-linux-gnu", - "pypi__coverage_cp39_x86_64-apple-darwin": "pypi__coverage_cp39_x86_64-apple-darwin", - "pypi__coverage_cp39_x86_64-unknown-linux-gnu": "pypi__coverage_cp39_x86_64-unknown-linux-gnu" - }, - "devImports": [], - "tags": [ - { - "tagName": "install", - "attributeValues": {}, - "devDependency": false, - "location": { - "file": "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel", - "line": 15, - "column": 22 - } - } - ], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@rules_python//python/extensions:python.bzl", - "extensionName": "python", - "usingModule": "rules_python@0.22.1", - "location": { - "file": "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel", - "line": 50, - "column": 23 - }, - "imports": { - "pythons_hub": "pythons_hub" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "platforms": "platforms@0.0.9", - "bazel_skylib": "bazel_skylib@1.3.0", - "rules_proto": "rules_proto@5.3.0-21.7", - "com_google_protobuf": "protobuf@21.7", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/rules_python/releases/download/0.22.1/rules_python-0.22.1.tar.gz" - ], - "integrity": "sha256-pWQP3dS+sD6MH95e1xYMC6a9R359BIZhwwwGk2om/WM=", - "strip_prefix": "rules_python-0.22.1", - "remote_patches": { - "https://bcr.bazel.build/modules/rules_python/0.22.1/patches/module_dot_bazel_version.patch": "sha256-3+VLDH9gYDzNI4eOW7mABC/LKxh1xqF6NhacLbNTucs=" - }, - "remote_patch_strip": 1 - } - } - }, - "buildozer@7.1.1.1": { - "name": "buildozer", - "version": "7.1.1.1", - "key": "buildozer@7.1.1.1", - "repoName": "buildozer", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [ - { - "extensionBzlFile": "@buildozer//:buildozer_binary.bzl", - "extensionName": "buildozer_binary", - "usingModule": "buildozer@7.1.1.1", - "location": { - "file": "https://bcr.bazel.build/modules/buildozer/7.1.1.1/MODULE.bazel", - "line": 7, - "column": 33 - }, - "imports": { - "buildozer_binary": "buildozer_binary" - }, - "devImports": [], - "tags": [ - { - "tagName": "buildozer", - "attributeValues": { - "sha256": { - "darwin-amd64": "6286bbbcb5314d675fcb4ede30edf397a68c611005412c937b2cc0c4bf4b714b", - "darwin-arm64": "0a2f70aa7b864de9bf71d1aac39659017b7a84169506e350d9ec77c609265212", - "linux-amd64": "9a7424aca7ca911c85cfedeadf065f0d95c492e80f0e29bd07ea98f6eb087259", - "linux-arm64": "07c8ed5ca3efea7e20756e9060660b7e658c781953c60650a1b99cd8bb857fcf", - "windows-amd64": "edad2e85cc691b064de9675db8e732b2c5a4ada0b32282ecdf4b18692cc8d3fe" - }, - "version": "7.1.1" - }, - "devDependency": false, - "location": { - "file": "https://bcr.bazel.build/modules/buildozer/7.1.1.1/MODULE.bazel", - "line": 8, - "column": 27 - } - } - ], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/fmeum/buildozer/releases/download/v7.1.1.1/buildozer-v7.1.1.1.tar.gz" - ], - "integrity": "sha256-wjmMsVBR96yT+WOlwN8g1gZRkc/VSPPnTjckHmIs9m8=", - "strip_prefix": "buildozer-7.1.1.1", - "remote_patches": { - "https://bcr.bazel.build/modules/buildozer/7.1.1.1/patches/module_dot_bazel_version.patch": "sha256-dOIwFLCmsm5pLgHK9aPnVXTjEsvn3xC4aHTXpbyrR2o=" - }, - "remote_patch_strip": 1 - } - } - }, - "platforms@0.0.9": { - "name": "platforms", - "version": "0.0.9", - "key": "platforms@0.0.9", - "repoName": "platforms", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [ - { - "extensionBzlFile": "@platforms//host:extension.bzl", - "extensionName": "host_platform", - "usingModule": "platforms@0.0.9", - "location": { - "file": "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel", - "line": 9, - "column": 30 - }, - "imports": { - "host_platform": "host_platform" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "rules_license": "rules_license@0.0.7", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/platforms/releases/download/0.0.9/platforms-0.0.9.tar.gz" - ], - "integrity": "sha256-XtpTnIQSZQMcL4LYrno6ZJC9YhduDAOPxGnqv5H2FJs=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, - "protobuf@21.7": { - "name": "protobuf", - "version": "21.7", - "key": "protobuf@21.7", - "repoName": "protobuf", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [ - { - "extensionBzlFile": "@rules_jvm_external//:extensions.bzl", - "extensionName": "maven", - "usingModule": "protobuf@21.7", - "location": { - "file": "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel", - "line": 22, - "column": 22 - }, - "imports": { - "maven": "maven" - }, - "devImports": [], - "tags": [ - { - "tagName": "install", - "attributeValues": { - "name": "maven", - "artifacts": [ - "com.google.code.findbugs:jsr305:3.0.2", - "com.google.code.gson:gson:2.8.9", - "com.google.errorprone:error_prone_annotations:2.3.2", - "com.google.j2objc:j2objc-annotations:1.3", - "com.google.guava:guava:31.1-jre", - "com.google.guava:guava-testlib:31.1-jre", - "com.google.truth:truth:1.1.2", - "junit:junit:4.13.2", - "org.mockito:mockito-core:4.3.1" - ] - }, - "devDependency": false, - "location": { - "file": "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel", - "line": 24, - "column": 14 - } - } - ], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "bazel_skylib": "bazel_skylib@1.3.0", - "rules_python": "rules_python@0.22.1", - "rules_cc": "rules_cc@0.0.9", - "rules_proto": "rules_proto@5.3.0-21.7", - "rules_java": "rules_java@7.5.0", - "rules_pkg": "rules_pkg@0.7.0", - "com_google_abseil": "abseil-cpp@20211102.0", - "zlib": "zlib@1.3", - "upb": "upb@0.0.0-20220923-a547704", - "rules_jvm_external": "rules_jvm_external@4.4.2", - "com_google_googletest": "googletest@1.11.0", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/protocolbuffers/protobuf/releases/download/v21.7/protobuf-all-21.7.zip" - ], - "integrity": "sha256-VJOiH17T/FAuZv7GuUScBqVRztYwAvpIkDxA36jeeko=", - "strip_prefix": "protobuf-21.7", - "remote_patches": { - "https://bcr.bazel.build/modules/protobuf/21.7/patches/add_module_dot_bazel.patch": "sha256-q3V2+eq0v2XF0z8z+V+QF4cynD6JvHI1y3kI/+rzl5s=", - "https://bcr.bazel.build/modules/protobuf/21.7/patches/add_module_dot_bazel_for_examples.patch": "sha256-O7YP6s3lo/1opUiO0jqXYORNHdZ/2q3hjz1QGy8QdIU=", - "https://bcr.bazel.build/modules/protobuf/21.7/patches/relative_repo_names.patch": "sha256-RK9RjW8T5UJNG7flIrnFiNE9vKwWB+8uWWtJqXYT0w4=", - "https://bcr.bazel.build/modules/protobuf/21.7/patches/add_missing_files.patch": "sha256-Hyne4DG2u5bXcWHNxNMirA2QFAe/2Cl8oMm1XJdkQIY=" - }, - "remote_patch_strip": 1 - } - } - }, - "zlib@1.3": { - "name": "zlib", - "version": "1.3", - "key": "zlib@1.3", - "repoName": "zlib", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "platforms": "platforms@0.0.9", - "rules_cc": "rules_cc@0.0.9", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/madler/zlib/releases/download/v1.3/zlib-1.3.tar.gz" - ], - "integrity": "sha256-/wukwpIBPbwnUws6geH5qBPNOd4Byl4Pi/NVcC76WT4=", - "strip_prefix": "zlib-1.3", - "remote_patches": { - "https://bcr.bazel.build/modules/zlib/1.3/patches/add_build_file.patch": "sha256-Ei+FYaaOo7A3jTKunMEodTI0Uw5NXQyZEcboMC8JskY=", - "https://bcr.bazel.build/modules/zlib/1.3/patches/module_dot_bazel.patch": "sha256-fPWLM+2xaF/kuy+kZc1YTfW6hNjrkG400Ho7gckuyJk=" - }, - "remote_patch_strip": 0 - } - } - }, - "apple_support@1.5.0": { - "name": "apple_support", - "version": "1.5.0", - "key": "apple_support@1.5.0", - "repoName": "build_bazel_apple_support", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "@local_config_apple_cc_toolchains//:all" - ], - "extensionUsages": [ - { - "extensionBzlFile": "@build_bazel_apple_support//crosstool:setup.bzl", - "extensionName": "apple_cc_configure_extension", - "usingModule": "apple_support@1.5.0", - "location": { - "file": "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel", - "line": 17, - "column": 35 - }, - "imports": { - "local_config_apple_cc": "local_config_apple_cc", - "local_config_apple_cc_toolchains": "local_config_apple_cc_toolchains" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "bazel_skylib": "bazel_skylib@1.3.0", - "platforms": "platforms@0.0.9", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/apple_support/releases/download/1.5.0/apple_support.1.5.0.tar.gz" - ], - "integrity": "sha256-miM41vja0yRPgj8txghKA+TQ+7J8qJLclw5okNW0gYQ=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, - "bazel_skylib@1.3.0": { - "name": "bazel_skylib", - "version": "1.3.0", - "key": "bazel_skylib@1.3.0", - "repoName": "bazel_skylib", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [ - "//toolchains/unittest:cmd_toolchain", - "//toolchains/unittest:bash_toolchain" - ], - "extensionUsages": [], - "deps": { - "platforms": "platforms@0.0.9", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz" - ], - "integrity": "sha256-dNVE2W9KW7Yw1GXKi7z+Ix41lOWq5X4e2/F6brPKJQY=", - "strip_prefix": "", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, - "rules_pkg@0.7.0": { - "name": "rules_pkg", - "version": "0.7.0", - "key": "rules_pkg@0.7.0", - "repoName": "rules_pkg", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "rules_python": "rules_python@0.22.1", - "bazel_skylib": "bazel_skylib@1.3.0", - "rules_license": "rules_license@0.0.7", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz" - ], - "integrity": "sha256-iimOgydi7aGDBZfWT+fbWBeKqEzVkm121bdE1lWJQcI=", - "strip_prefix": "", - "remote_patches": { - "https://bcr.bazel.build/modules/rules_pkg/0.7.0/patches/module_dot_bazel.patch": "sha256-4OaEPZwYF6iC71ZTDg6MJ7LLqX7ZA0/kK4mT+4xKqiE=" - }, - "remote_patch_strip": 0 - } - } - }, - "abseil-cpp@20211102.0": { - "name": "abseil-cpp", - "version": "20211102.0", - "key": "abseil-cpp@20211102.0", - "repoName": "abseil-cpp", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "rules_cc": "rules_cc@0.0.9", - "platforms": "platforms@0.0.9", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.tar.gz" - ], - "integrity": "sha256-3PcbnLqNwMqZQMSzFqDHlr6Pq0KwcLtrfKtitI8OZsQ=", - "strip_prefix": "abseil-cpp-20211102.0", - "remote_patches": { - "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/patches/module_dot_bazel.patch": "sha256-4izqopgGCey4jVZzl/w3M2GVPNohjh2B5TmbThZNvPY=" - }, - "remote_patch_strip": 0 - } - } - }, - "upb@0.0.0-20220923-a547704": { - "name": "upb", - "version": "0.0.0-20220923-a547704", - "key": "upb@0.0.0-20220923-a547704", - "repoName": "upb", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "bazel_skylib": "bazel_skylib@1.3.0", - "rules_proto": "rules_proto@5.3.0-21.7", - "com_google_protobuf": "protobuf@21.7", - "com_google_absl": "abseil-cpp@20211102.0", - "platforms": "platforms@0.0.9", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/protocolbuffers/upb/archive/a5477045acaa34586420942098f5fecd3570f577.tar.gz" - ], - "integrity": "sha256-z39x6v+QskwaKLSWRan/A6mmwecTQpHOcJActj5zZLU=", - "strip_prefix": "upb-a5477045acaa34586420942098f5fecd3570f577", - "remote_patches": { - "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/patches/module_dot_bazel.patch": "sha256-wH4mNS6ZYy+8uC0HoAft/c7SDsq2Kxf+J8dUakXhaB0=" - }, - "remote_patch_strip": 0 - } - } - }, - "rules_jvm_external@4.4.2": { - "name": "rules_jvm_external", - "version": "4.4.2", - "key": "rules_jvm_external@4.4.2", - "repoName": "rules_jvm_external", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [ - { - "extensionBzlFile": "@rules_jvm_external//:non-module-deps.bzl", - "extensionName": "non_module_deps", - "usingModule": "rules_jvm_external@4.4.2", - "location": { - "file": "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel", - "line": 9, - "column": 32 - }, - "imports": { - "io_bazel_rules_kotlin": "io_bazel_rules_kotlin" - }, - "devImports": [], - "tags": [], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - }, - { - "extensionBzlFile": "@rules_jvm_external//:extensions.bzl", - "extensionName": "maven", - "usingModule": "rules_jvm_external@4.4.2", - "location": { - "file": "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel", - "line": 16, - "column": 22 - }, - "imports": { - "rules_jvm_external_deps": "rules_jvm_external_deps" - }, - "devImports": [], - "tags": [ - { - "tagName": "install", - "attributeValues": { - "name": "rules_jvm_external_deps", - "artifacts": [ - "com.google.cloud:google-cloud-core:1.93.10", - "com.google.cloud:google-cloud-storage:1.113.4", - "com.google.code.gson:gson:2.9.0", - "org.apache.maven:maven-artifact:3.8.6", - "software.amazon.awssdk:s3:2.17.183" - ], - "lock_file": "@rules_jvm_external//:rules_jvm_external_deps_install.json" - }, - "devDependency": false, - "location": { - "file": "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel", - "line": 18, - "column": 14 - } - } - ], - "hasDevUseExtension": false, - "hasNonDevUseExtension": true - } - ], - "deps": { - "bazel_skylib": "bazel_skylib@1.3.0", - "io_bazel_stardoc": "stardoc@0.5.1", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/rules_jvm_external/archive/refs/tags/4.4.2.zip" - ], - "integrity": "sha256-c1YC9QgT6y6pPKP15DsZWb2AshO4NqB6YqKddXZwt3s=", - "strip_prefix": "rules_jvm_external-4.4.2", - "remote_patches": {}, - "remote_patch_strip": 0 - } - } - }, - "googletest@1.11.0": { - "name": "googletest", - "version": "1.11.0", - "key": "googletest@1.11.0", - "repoName": "googletest", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "com_google_absl": "abseil-cpp@20211102.0", - "platforms": "platforms@0.0.9", - "rules_cc": "rules_cc@0.0.9", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/google/googletest/archive/refs/tags/release-1.11.0.tar.gz" - ], - "integrity": "sha256-tIcL8SH/d5W6INILzdhie44Ijy0dqymaAxwQNO3ck9U=", - "strip_prefix": "googletest-release-1.11.0", - "remote_patches": { - "https://bcr.bazel.build/modules/googletest/1.11.0/patches/module_dot_bazel.patch": "sha256-HuahEdI/n8KCI071sN3CEziX+7qP/Ec77IWayYunLP0=" - }, - "remote_patch_strip": 0 - } - } - }, - "stardoc@0.5.1": { - "name": "stardoc", - "version": "0.5.1", - "key": "stardoc@0.5.1", - "repoName": "stardoc", - "executionPlatformsToRegister": [], - "toolchainsToRegister": [], - "extensionUsages": [], - "deps": { - "bazel_skylib": "bazel_skylib@1.3.0", - "rules_java": "rules_java@7.5.0", - "bazel_tools": "bazel_tools@_", - "local_config_platform": "local_config_platform@_" - }, - "repoSpec": { - "bzlFile": "@@bazel_tools//tools/build_defs/repo:http.bzl", - "ruleClassName": "http_archive", - "attributes": { - "urls": [ - "https://github.com/bazelbuild/stardoc/releases/download/0.5.1/stardoc-0.5.1.tar.gz" - ], - "integrity": "sha256-qoFNrgrEALurLoiB+ZFcb0fElmS/CHxAmhX5BDjSwj4=", - "strip_prefix": "", - "remote_patches": { - "https://bcr.bazel.build/modules/stardoc/0.5.1/patches/module_dot_bazel.patch": "sha256-UAULCuTpJE7SG0YrR9XLjMfxMRmbP+za3uW9ONZ5rjI=" - }, - "remote_patch_strip": 0 - } - } - } - }, + "lockFileVersion": 10, "registryFileHashes": { "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", @@ -1114,11 +57,12 @@ "https://bcr.bazel.build/modules/zlib/1.3/MODULE.bazel": "6a9c02f19a24dcedb05572b2381446e27c272cd383aed11d41d99da9e3167a72", "https://bcr.bazel.build/modules/zlib/1.3/source.json": "b6b43d0737af846022636e6e255fd4a96fee0d34f08f3830e6e0bac51465c37c" }, + "selectedYankedVersions": {}, "moduleExtensions": { "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "pMLFCYaRPkgXPQ8vtuNkMfiHfPmRBy6QJfnid4sWfv0=", - "usagesDigest": "fe2odiE5nvy4BXvMJSkQEP0BFhJ/bMKhVBFkh+VCfqU=", + "usagesDigest": "aLmqbvowmHkkBPve05yyDNGN7oh7QE9kBADr3QIZTZs=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1146,7 +90,7 @@ "@@bazel_tools//tools/android:android_extensions.bzl%remote_android_tools_extensions": { "general": { "bzlTransitiveDigest": "OBuiEhPTsmGJLplgSLRwqDPybt0jns4dOHdzYzOJ1aM=", - "usagesDigest": "dw7C7ErPIkjYn3yY10kLuOatAlp+XNoJ1RbOr/1/3Tg=", + "usagesDigest": "pfoI3DAAesYHdp38xIvu9Ry0AM3EIPGwvav/Cyv23Kw=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1174,7 +118,7 @@ "@@bazel_tools//tools/cpp:cc_configure.bzl%cc_configure_extension": { "general": { "bzlTransitiveDigest": "PHpT2yqMGms2U4L3E/aZ+WcQalmZWm+ILdP3yiLsDhA=", - "usagesDigest": "gWYd8gJQoq+NpmwavSUrR13PCKHYkpwE7EvqssERMc8=", + "usagesDigest": "6zVI5vmN6Evleo1nJgTcfkJmgPFqhlf1Wl94WarCpa0=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1202,7 +146,7 @@ "@@bazel_tools//tools/osx:xcode_configure.bzl%xcode_configure_extension": { "general": { "bzlTransitiveDigest": "Qh2bWTU6QW6wkrd87qrU4YeY+SG37Nvw3A0PR4Y0L2Y=", - "usagesDigest": "6IVLzvb1OpngnM2xlJMGcF4ORDez1NantK5MY6HVXRo=", + "usagesDigest": "hl3K6oDsXyui9k2oT4OTT6CdW6L/lSrBDSOWIZiUT1Q=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1222,7 +166,7 @@ "@@bazel_tools//tools/sh:sh_configure.bzl%sh_configure_extension": { "general": { "bzlTransitiveDigest": "hp4NgmNjEg5+xgvzfh6L83bt9/aiiWETuNpwNuF1MSU=", - "usagesDigest": "OFE3XiZmPfwxbY+NUdJN9QP5yiQTMoPN6zW5aV3n4fY=", + "usagesDigest": "gYz9qiTxGtMhXZhrQTBErBunPVrPWNhiFJAP9Nf18GM=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1239,7 +183,7 @@ "@@bazel_tools//tools/test:extensions.bzl%remote_coverage_tools_extension": { "general": { "bzlTransitiveDigest": "yd81gjOpH9Vd/KF71OifnQucGLfgWXfE+PqqamoczPk=", - "usagesDigest": "fsG10p+ckw98K4MfcQu0IYsMwidhFcUesWaQcPdj7vY=", + "usagesDigest": "5AzD10fWCl4vQ0i0Zl96Ial7tU8raC84LTK6DeHefuk=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1261,7 +205,7 @@ "@@buildozer~//:buildozer_binary.bzl%buildozer_binary": { "general": { "bzlTransitiveDigest": "EleDU/FQ1+e/RgkW3aIDmdaxZEthvoWQhsqFTxiSgMI=", - "usagesDigest": "z0uLgf24hCzSZAiM0N9nal6UjWKM4JAmNbm56LP0vU0=", + "usagesDigest": "hxw0DOI5uj9VGmLCZhN8VXkjHv/p+F6B1t1NyQTGnO8=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1287,7 +231,7 @@ "@@platforms//host:extension.bzl%host_platform": { "general": { "bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=", - "usagesDigest": "/wi7fMPr4cwiSnBi3fXdHVOtA47+TSmMiv8XIxSTiTE=", + "usagesDigest": "meSzxn3DUCcYEhq4HQwExWkWtU4EjriRBQLsZN+Q0SU=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1304,7 +248,7 @@ "@@rules_java~//java:extensions.bzl%toolchains": { "general": { "bzlTransitiveDigest": "luQz5wg3waFxw7wF3Jevqd/dQVN9RQxuHioSnhWMWbU=", - "usagesDigest": "eBHBwdDAsDtvJ8WuWgXTHIEveUYrhzF+zdOY4oGoDP8=", + "usagesDigest": "km/MmwGP92TBuE8sy/Jk2BmLrfOADnezet92ElEDuSA=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -1870,7 +814,7 @@ "@@rules_jvm_external~//:extensions.bzl%maven": { "general": { "bzlTransitiveDigest": "SLfIGbNRszU0h8ge0IMO3/eI3rU4GFZKXkA2TRilJmw=", - "usagesDigest": "zel/el3Hu7qA5ChuCSQc5Vo46WXLZmMwpH8aUfhD5wM=", + "usagesDigest": "UPebZtX4g40+QepdK3oMHged0o0tq6ojKbW84wE6XRA=", "recordedFileInputs": { "@@rules_jvm_external~//rules_jvm_external_deps_install.json": "10442a5ae27d9ff4c2003e5ab71643bf0d8b48dcf968b4173fa274c3232a8c06" }, @@ -2894,7 +1838,7 @@ "@@rules_jvm_external~//:non-module-deps.bzl%non_module_deps": { "general": { "bzlTransitiveDigest": "oHEz4DXnwk2aTh/PnIWUE3E1lOge1JPTJOlZOSOr3wI=", - "usagesDigest": "auoSltLJ8wMZ3iqVKYWQsyC9CgcCDt85yHgTXFuQEvE=", + "usagesDigest": "bTG4ItERqhG1LeSs62hQ01DiMarFsflWgpZaghM5qik=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2922,7 +1866,7 @@ "@@rules_python~//python/extensions:python.bzl%python": { "general": { "bzlTransitiveDigest": "8RLGOLaDCviFdE0nDT5TmqaQd1w6Cm2OXWVzHbcIcxQ=", - "usagesDigest": "gGVuRK4jgFR/T75mVSEYMFObc6AY82n+c1P5B+5hUkE=", + "usagesDigest": "7vjNHuEgQORYN9+9/77Q4zw1kawobM2oCQb9p0uhL68=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, @@ -2952,7 +1896,7 @@ "@@rules_python~//python/extensions/private:internal_deps.bzl%internal_deps": { "general": { "bzlTransitiveDigest": "HyR82V0QIYQqdGYFWjm85H/5Up3KvA+wg1c684HcFpE=", - "usagesDigest": "/sVN6+OrINK3/oVrgWGYeYt+N1f1J1KnGaHR2AUoIMY=", + "usagesDigest": "b+nMDqtqPCBxiMBewNNde3aNjzKqZyvJuN5/49xB62s=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, diff --git a/src/tools/bzlmod/blazel_utils.bzl b/src/tools/bzlmod/blazel_utils.bzl new file mode 100644 index 00000000000000..9cd5a2723afb33 --- /dev/null +++ b/src/tools/bzlmod/blazel_utils.bzl @@ -0,0 +1,21 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Helper functions for Bzlmod build that are safe to use both internally and externally.""" + +def get_canonical_repo_name(apparent_repo_name): + """Returns the canonical repo name for the given apparent repo name seen by the module this bzl file belongs to.""" + if not apparent_repo_name.startswith("@"): + apparent_repo_name = "@" + apparent_repo_name + return Label(apparent_repo_name).workspace_name diff --git a/src/tools/bzlmod/utils.bzl b/src/tools/bzlmod/utils.bzl index c9bfcdd270eed9..c8b22a93bd26db 100644 --- a/src/tools/bzlmod/utils.bzl +++ b/src/tools/bzlmod/utils.bzl @@ -14,11 +14,10 @@ """Helper functions for Bzlmod build""" -def get_canonical_repo_name(apparent_repo_name): - """Returns the canonical repo name for the given apparent repo name seen by the module this bzl file belongs to.""" - if not apparent_repo_name.startswith("@"): - apparent_repo_name = "@" + apparent_repo_name - return Label(apparent_repo_name).workspace_name +load("@buildozer//:buildozer.bzl", "BUILDOZER_LABEL") +load(":blazel_utils.bzl", _get_canonical_repo_name = "get_canonical_repo_name") + +get_canonical_repo_name = _get_canonical_repo_name def extract_url(attributes): """Extracts the url from the given attributes. @@ -52,26 +51,56 @@ def parse_http_artifacts(ctx, lockfile_path, required_repos): lockfile = json.decode(ctx.read(lockfile_path)) http_artifacts = [] found_repos = [] - for _, module in lockfile["moduleDepGraph"].items(): - if "repoSpec" in module and module["repoSpec"]["ruleClassName"] == "http_archive": - repo_spec = module["repoSpec"] - attributes = repo_spec["attributes"] - repo_name = _module_repo_name(module) + if "moduleDepGraph" in lockfile: + # TODO: Remove this branch after Bazel is built with 7.2.0. + for _, module in lockfile["moduleDepGraph"].items(): + if "repoSpec" in module and module["repoSpec"]["ruleClassName"] == "http_archive": + repo_spec = module["repoSpec"] + attributes = repo_spec["attributes"] + repo_name = _module_repo_name(module) + if repo_name not in required_repos: + continue + found_repos.append(repo_name) + + http_artifacts.append({ + "integrity": attributes["integrity"], + "url": extract_url(attributes), + }) + if "remote_patches" in attributes: + for patch, integrity in attributes["remote_patches"].items(): + http_artifacts.append({ + "integrity": integrity, + "url": patch, + }) + else: + for url, sha256 in lockfile["registryFileHashes"].items(): + if not url.endswith("/source.json"): + continue + segments = url.split("/") + module = { + "name": segments[-3], + "version": segments[-2], + } + repo_name = _module_repo_name(module) if repo_name not in required_repos: continue found_repos.append(repo_name) + ctx.delete("./tempfile") + ctx.download(url, "./tempfile", executable = False, sha256 = sha256) + source_json = json.decode(ctx.read("./tempfile")) + http_artifacts.append({ - "integrity": attributes["integrity"], - "url": extract_url(attributes), + "integrity": source_json["integrity"], + "url": source_json["url"], }) - if "remote_patches" in attributes: - for patch, integrity in attributes["remote_patches"].items(): - http_artifacts.append({ - "integrity": integrity, - "url": patch, - }) + + for patch, integrity in source_json.get("patches", {}).items(): + http_artifacts.append({ + "integrity": integrity, + "url": url.rsplit("/", 1)[0] + "/patches/" + patch, + }) for extension_id, extension_entry in lockfile["moduleExtensions"].items(): if extension_id.startswith("@@"): @@ -109,6 +138,83 @@ def parse_http_artifacts(ctx, lockfile_path, required_repos): return http_artifacts +BCR_URL_SCHEME = "https://bcr.bazel.build/modules/{name}/{version}/{file}" + +def parse_registry_files(ctx, lockfile_path, module_files): + """Parses the registry files referenced by the given lockfile and returns them in http_file form. + + Args: + ctx: the repository / module extension ctx object. + lockfile_path: The path of the lockfile to extract the registry files from. + module_files: The paths of non-registry module files to use during fake module resolution. + + Returns: + A list of http artifacts in the form of + [{"sha256": , "url": }, ...] + """ + lockfile = json.decode(ctx.read(lockfile_path)) + registry_file_hashes = lockfile.get("registryFileHashes", {}) + if registry_file_hashes: + return [ + {"sha256": sha256, "url": url} + for url, sha256 in registry_file_hashes.items() + ] + + # TODO: Remove the following code after Bazel is built with 7.2.0. + registry_files = ["https://bcr.bazel.build/bazel_registry.json"] + + # 1. Collect all source.json files of selected module versions. + for module in lockfile["moduleDepGraph"].values(): + if module["version"]: + registry_files.append(BCR_URL_SCHEME.format( + name = module["name"], + version = module["version"], + file = "source.json", + )) + + # 2. Download registry files to compute their hashes. + registry_file_artifacts = [] + downloads = { + url: ctx.download(url, "./tempdir/{}".format(i), executable = False, block = False) + for i, url in enumerate(registry_files) + } + for url, download in downloads.items(): + hash = download.wait() + registry_file_artifacts.append({"url": url, "sha256": hash.sha256}) + + # 3. Perform module resolution in Starlark to get the MODULE.bazel file URLs + # of all module versions relevant during resolution. The lockfile only + # contains the selected module versions. + module_file_stack = [ctx.path(module_file) for module_file in module_files] + seen_deps = {} + for _ in range(1000000): + if not module_file_stack: + break + bazel_deps = _extract_bazel_deps(ctx, module_file_stack.pop()) + downloads = {} + for dep in bazel_deps: + if dep in seen_deps: + continue + url = BCR_URL_SCHEME.format( + name = dep.name, + version = dep.version, + file = "MODULE.bazel", + ) + path = ctx.path("./tempdir/modules/{name}/{version}/MODULE.bazel".format( + name = dep.name, + version = dep.version, + )) + module_file_stack.append(path) + seen_deps[dep] = None + downloads[url] = ctx.download(url, path, executable = False, block = False) + + for url, download in downloads.items(): + hash = download.wait() + registry_file_artifacts.append({"url": url, "sha256": hash.sha256}) + + ctx.delete("./tempdir") + return registry_file_artifacts + def parse_bazel_module_repos(ctx, lockfile_path): """Parse repo names of http_archive backed Bazel modules from the given lockfile. @@ -122,13 +228,17 @@ def parse_bazel_module_repos(ctx, lockfile_path): lockfile = json.decode(ctx.read(lockfile_path)) repos = [] - for _, module in lockfile["moduleDepGraph"].items(): - if "repoSpec" in module and module["repoSpec"]["ruleClassName"] == "http_archive": - repo_spec = module["repoSpec"] - attributes = repo_spec["attributes"] - repo_name = _module_repo_name(module) - repos.append(repo_name) - return repos + for url in lockfile["registryFileHashes"].keys(): + if not url.endswith("/source.json"): + continue + segments = url.split("/") + module = { + "name": segments[-3], + "version": segments[-2], + } + repo_name = _module_repo_name(module) + repos.append(repo_name) + return {repo: None for repo in repos}.keys() # Keep in sync with ModuleKey. _WELL_KNOWN_MODULES = ["bazel_tools", "local_config_platform", "platforms"] @@ -143,3 +253,25 @@ def _module_repo_name(module): return "{}~".format(module_name) return "{}~{}".format(module_name, module["version"]) + +def _extract_bazel_deps(ctx, module_file): + buildozer = ctx.path(BUILDOZER_LABEL) + temp_path = "tempdir/buildozer/MODULE.bazel" + ctx.delete(temp_path) + ctx.symlink(module_file, temp_path) + result = ctx.execute([buildozer, "print name version dev_dependency", temp_path + ":%bazel_dep"]) + if result.return_code != 0: + fail("Failed to extract bazel_dep from {}:\n{}".format(module_file, result.stderr)) + deps = [] + for line in result.stdout.splitlines(): + if " " in line: + # The dep doesn't have a version specified, which is only valid in + # the root module. Ignore it. + continue + if line.endswith(" True"): + # The dep is a dev_dependency, ignore it. + continue + name, version, _ = line.split(" ") + deps.append(struct(name = name, version = version)) + + return deps