diff --git a/flake.nix b/flake.nix index 1f663b1db..685b8f2fc 100644 --- a/flake.nix +++ b/flake.nix @@ -76,14 +76,11 @@ llvmPackages = pkgs.llvmPackages_18; - customStdenv = import ./tools/llvmStdenv.nix {inherit pkgs llvmPackages;}; + customStdenv = pkgs.callPackage ./tools/llvmStdenv.nix {inherit llvmPackages;}; # TODO(aaronmondal): This doesn't work with rules_rust yet. # Tracked in https://github.com/TraceMachina/nativelink/issues/477. - customClang = pkgs.callPackage ./tools/customClang.nix { - inherit pkgs; - stdenv = customStdenv; - }; + customClang = pkgs.callPackage ./tools/customClang.nix {stdenv = customStdenv;}; craneLib = if pkgs.stdenv.isDarwin @@ -132,15 +129,15 @@ cargoExtraArgs = "--features enable_tokio_console"; }); - hooks = import ./tools/pre-commit-hooks.nix {inherit pkgs nightly-rust;}; + publish-ghcr = pkgs.callPackage ./tools/publish-ghcr.nix {}; - publish-ghcr = import ./tools/publish-ghcr.nix {inherit pkgs;}; + local-image-test = pkgs.callPackage ./tools/local-image-test.nix {}; - local-image-test = import ./tools/local-image-test.nix {inherit pkgs nativelink;}; + nativelink-is-executable-test = pkgs.callPackage ./tools/nativelink-is-executable-test.nix {inherit nativelink;}; - nativelink-is-executable-test = import ./tools/nativelink-is-executable-test.nix {inherit pkgs nativelink;}; + rbe-configs-gen = pkgs.callPackage ./local-remote-execution/rbe-configs-gen.nix {}; - generate-toolchains = import ./tools/generate-toolchains.nix {inherit pkgs;}; + generate-toolchains = pkgs.callPackage ./tools/generate-toolchains.nix {inherit rbe-configs-gen;}; native-cli = pkgs.callPackage ./native-cli/default.nix {}; @@ -174,8 +171,11 @@ nativelink-worker-init = pkgs.callPackage ./tools/nativelink-worker-init.nix {inherit buildImage self nativelink-image;}; - rbe-autogen = import ./local-remote-execution/rbe-autogen.nix {inherit pkgs nativelink buildImage llvmPackages;}; - createWorker = import ./tools/create-worker.nix {inherit pkgs buildImage self;}; + rbe-autogen = pkgs.callPackage ./local-remote-execution/rbe-autogen.nix { + inherit buildImage; + stdenv = customStdenv; + }; + createWorker = pkgs.callPackage ./tools/create-worker.nix {inherit buildImage self;}; siso-chromium = buildImage { name = "siso-chromium"; fromImage = pullImage { @@ -187,7 +187,10 @@ os = "linux"; }; }; - lre-cc = import ./local-remote-execution/lre-cc.nix {inherit pkgs buildImage llvmPackages;}; + lre-cc = pkgs.callPackage ./local-remote-execution/lre-cc.nix { + inherit customClang buildImage; + stdenv = customStdenv; + }; toolchain-drake = buildImage { name = "toolchain-drake"; # imageDigest and sha256 are generated by toolchain-drake.sh for non-reproducible builds. @@ -232,7 +235,7 @@ rbe-autogen-lre-cc = rbe-autogen lre-cc; nativelink-worker-lre-cc = createWorker lre-cc; - lre-java = import ./local-remote-execution/lre-java.nix {inherit pkgs buildImage;}; + lre-java = pkgs.callPackage ./local-remote-execution/lre-java.nix {inherit buildImage;}; rbe-autogen-lre-java = rbe-autogen lre-java; nativelink-worker-lre-java = createWorker lre-java; nativelink-worker-siso-chromium = createWorker siso-chromium; @@ -249,7 +252,11 @@ # partitionType = "count"; # }); }; - pre-commit.settings = {inherit hooks;}; + pre-commit.settings = { + hooks = import ./tools/pre-commit-hooks.nix { + inherit pkgs nightly-rust; + }; + }; local-remote-execution.settings = { Env = if pkgs.stdenv.isDarwin diff --git a/local-remote-execution/lre-cc.nix b/local-remote-execution/lre-cc.nix index bab041961..7938e315f 100644 --- a/local-remote-execution/lre-cc.nix +++ b/local-remote-execution/lre-cc.nix @@ -1,14 +1,9 @@ { - pkgs, buildImage, - llvmPackages, + customClang, + stdenv, + pkgs, }: let - customStdenv = import ../tools/llvmStdenv.nix {inherit pkgs llvmPackages;}; - customClang = pkgs.callPackage ../tools/customClang.nix { - inherit pkgs; - stdenv = customStdenv; - }; - # This environment is shared between toolchain autogen images and the final # toolchain image. Env = [ @@ -17,9 +12,9 @@ # binary identical toolchains during local and remote execution. ("PATH=" + (pkgs.lib.strings.concatStringsSep ":" [ - "${customStdenv.cc.bintools}/bin" + "${stdenv.cc.bintools}/bin" "${customClang}/bin" - "${customStdenv}/bin" + "${stdenv}/bin" "${pkgs.coreutils}/bin" "${pkgs.findutils}/bin" "${pkgs.gnutar}/bin" diff --git a/local-remote-execution/lre-java.nix b/local-remote-execution/lre-java.nix index c73d09d16..c1d461503 100644 --- a/local-remote-execution/lre-java.nix +++ b/local-remote-execution/lre-java.nix @@ -1,6 +1,10 @@ { - pkgs, buildImage, + coreutils, + findutils, + gnutar, + jdk17_headless, + lib, ... }: let # This config is shared between toolchain autogen images and the final @@ -10,12 +14,12 @@ # paths instead of `/bin` or `/usr/bin`. This way we're guaranteed to use # binary identical toolchains during local and remote execution. ("PATH=" - + (pkgs.lib.strings.concatStringsSep ":" [ - "${pkgs.coreutils}/bin" - "${pkgs.findutils}/bin" - "${pkgs.gnutar}/bin" + + (lib.strings.concatStringsSep ":" [ + "${coreutils}/bin" + "${findutils}/bin" + "${gnutar}/bin" ])) - "JAVA_HOME=${pkgs.jdk17_headless}/lib/openjdk" + "JAVA_HOME=${jdk17_headless}/lib/openjdk" ]; in buildImage { diff --git a/local-remote-execution/rbe-autogen.nix b/local-remote-execution/rbe-autogen.nix index 9ef120cc4..9af521466 100644 --- a/local-remote-execution/rbe-autogen.nix +++ b/local-remote-execution/rbe-autogen.nix @@ -1,34 +1,40 @@ { - pkgs, + bash, + bazel_7, + buildEnv, buildImage, - llvmPackages, - ... + cacert, + coreutils, + findutils, + gnutar, + lib, + runCommand, + runtimeShell, + stdenv, }: let - customStdenv = import ../tools/llvmStdenv.nix {inherit pkgs llvmPackages;}; - # These dependencies are needed to generate the toolchain configurations but # aren't required during remote execution. autogenDeps = [ # Required to generate toolchain configs. - pkgs.bazel_7 + bazel_7 # Required for communication with trusted sources. - pkgs.cacert + cacert # Tools that we would usually forward from the host. - pkgs.bash - pkgs.coreutils + bash + coreutils # We need these tools to generate the RBE autoconfiguration. - pkgs.findutils - pkgs.gnutar + findutils + gnutar - customStdenv.cc.bintools + stdenv.cc.bintools ]; # A temporary directory. Note that this doesn't set any permissions. Those # need to be added explicitly in the final image arguments. - mkTmp = pkgs.runCommand "mkTmp" {} '' + mkTmp = runCommand "mkTmp" {} '' mkdir -p $out/tmp ''; @@ -42,7 +48,7 @@ }; # Enable the shebang `#!/usr/bin/env bash`. - mkEnvSymlink = pkgs.runCommand "mkEnvSymlink" {} '' + mkEnvSymlink = runCommand "mkEnvSymlink" {} '' mkdir -p $out/usr/bin ln -s /bin/env $out/usr/bin/env ''; @@ -52,10 +58,10 @@ uid = "1000"; gid = "1000"; - mkUser = pkgs.runCommand "mkUser" {} '' + mkUser = runCommand "mkUser" {} '' mkdir -p $out/etc/pam.d - echo "root:x:0:0::/root:${pkgs.runtimeShell}" > $out/etc/passwd + echo "root:x:0:0::/root:${runtimeShell}" > $out/etc/passwd echo "${user}:x:${uid}:${gid}:::" >> $out/etc/passwd echo "root:!x:::::::" > $out/etc/shadow @@ -83,8 +89,8 @@ path = mkUser; regex = "/home/${user}"; mode = "0755"; - uid = pkgs.lib.toInt uid; - gid = pkgs.lib.toInt gid; + uid = lib.toInt uid; + gid = lib.toInt gid; uname = user; gname = group; }; @@ -98,7 +104,7 @@ in mkUser mkTmp mkEnvSymlink - (pkgs.buildEnv { + (buildEnv { name = "${image.imageName}-buildEnv"; paths = autogenDeps; pathsToLink = ["/bin"]; diff --git a/local-remote-execution/rbe-configs-gen.nix b/local-remote-execution/rbe-configs-gen.nix index db0299bea..748b8598b 100644 --- a/local-remote-execution/rbe-configs-gen.nix +++ b/local-remote-execution/rbe-configs-gen.nix @@ -1,5 +1,9 @@ -{pkgs, ...}: -pkgs.buildGoModule rec { +{ + buildGoModule, + fetchFromGitHub, + lib, +}: +buildGoModule rec { pname = "bazel-toolchains"; version = "5.1.3-rc1"; @@ -7,7 +11,7 @@ pkgs.buildGoModule rec { ./rbe_configs_gen_adjustments.diff ]; - src = pkgs.fetchFromGitHub { + src = fetchFromGitHub { owner = "bazelbuild"; repo = "bazel-toolchains"; rev = "v${version}"; @@ -16,7 +20,7 @@ pkgs.buildGoModule rec { vendorHash = "sha256-E6PylI2prXCXqOUYgYi5nZ4qptqOqbcaOquDfEkhaQ4="; - meta = with pkgs.lib; { + meta = with lib; { description = "Generate Bazel toolchain configs for remote execution."; homepage = "https://github.com/bazelbuild/bazel-toolchains"; license = licenses.asl20; diff --git a/tools/create-worker.nix b/tools/create-worker.nix index 57374c866..40212ef2f 100644 --- a/tools/create-worker.nix +++ b/tools/create-worker.nix @@ -1,12 +1,16 @@ { - pkgs, + bash, + buildEnv, buildImage, + coreutils, + lib, + runCommand, + runtimeShell, self, - ... }: let # A temporary directory. Note that this doesn't set any permissions. Those # need to be added explicitly in the final image arguments. - mkTmp = pkgs.runCommand "mkTmp" {} '' + mkTmp = runCommand "mkTmp" {} '' mkdir -p $out/tmp ''; @@ -20,7 +24,7 @@ }; # Enable the shebang `#!/usr/bin/env bash`. - mkEnvSymlink = pkgs.runCommand "mkEnvSymlink" {} '' + mkEnvSymlink = runCommand "mkEnvSymlink" {} '' mkdir -p $out/usr/bin ln -s /bin/env $out/usr/bin/env ''; @@ -30,10 +34,10 @@ uid = "1000"; gid = "1000"; - mkUser = pkgs.runCommand "mkUser" {} '' + mkUser = runCommand "mkUser" {} '' mkdir -p $out/etc/pam.d - echo "root:x:0:0::/root:${pkgs.runtimeShell}" > $out/etc/passwd + echo "root:x:0:0::/root:${runtimeShell}" > $out/etc/passwd echo "${user}:x:${uid}:${gid}:::" >> $out/etc/passwd echo "root:!x:::::::" > $out/etc/shadow @@ -61,8 +65,8 @@ path = mkUser; regex = "/home/${user}"; mode = "0755"; - uid = pkgs.lib.toInt uid; - gid = pkgs.lib.toInt gid; + uid = lib.toInt uid; + gid = lib.toInt gid; uname = user; gname = group; }; @@ -79,9 +83,9 @@ in mkUser mkTmp mkEnvSymlink - (pkgs.buildEnv { + (buildEnv { name = "${image.imageName}-buildEnv"; - paths = [pkgs.coreutils pkgs.bash]; + paths = [coreutils bash]; pathsToLink = ["/bin"]; }) ]; diff --git a/tools/customClang.nix b/tools/customClang.nix index 1827af3e0..c0dfef90d 100644 --- a/tools/customClang.nix +++ b/tools/customClang.nix @@ -1,6 +1,6 @@ { - pkgs, stdenv, + writeShellScriptBin, }: # Bazel expects a single frontend for both C and C++. That works for GCC but # not for clang. This wrapper selects `clang` or `clang++` depending on file @@ -8,7 +8,7 @@ # TODO(aaronmondal): The necessity of this is a bug. # See: https://github.com/NixOS/nixpkgs/issues/216047 # and https://github.com/NixOS/nixpkgs/issues/150655 -pkgs.writeShellScriptBin "customClang" '' +writeShellScriptBin "customClang" '' #! ${stdenv.shell} function isCxx() { if [ $# -eq 0 ]; then false diff --git a/tools/generate-toolchains.nix b/tools/generate-toolchains.nix index 2dd2c7115..190c960c4 100644 --- a/tools/generate-toolchains.nix +++ b/tools/generate-toolchains.nix @@ -1,71 +1,70 @@ -{pkgs}: let - rbeConfigsGen = import ../local-remote-execution/rbe-configs-gen.nix { - inherit pkgs; - }; -in - pkgs.writeShellScriptBin "generate-toolchains" '' - #!{pkgs.bash}/bin/bash - set -xeuo pipefail +{ + bazel_7, + writeShellScriptBin, + rbe-configs-gen, +}: +writeShellScriptBin "generate-toolchains" '' + set -xeuo pipefail - SRC_ROOT=$(git rev-parse --show-toplevel)/local-remote-execution + SRC_ROOT=$(git rev-parse --show-toplevel)/local-remote-execution - cd "''${SRC_ROOT}" + cd "''${SRC_ROOT}" - LRE_CC_IMAGE_TAG=$(nix eval .#lre-cc.imageTag --raw) + LRE_CC_IMAGE_TAG=$(nix eval .#lre-cc.imageTag --raw) - nix run .#rbe-autogen-lre-cc.copyTo \ - docker-daemon:rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} -L + nix run .#rbe-autogen-lre-cc.copyTo \ + docker-daemon:rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} -L - ${rbeConfigsGen}/bin/rbe_configs_gen \ - --toolchain_container=rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} \ - --exec_os=linux \ - --target_os=linux \ - --bazel_version=${pkgs.bazel_7.version} \ - --output_src_root=''${SRC_ROOT} \ - --output_config_path=generated-cc \ - --generate_java_configs=false \ - --generate_cpp_configs=true \ - --bazel_path=${pkgs.bazel_7}/bin/bazel \ - --cpp_env_json=cpp_env.json + ${rbe-configs-gen}/bin/rbe_configs_gen \ + --toolchain_container=rbe-autogen-lre-cc:''${LRE_CC_IMAGE_TAG} \ + --exec_os=linux \ + --target_os=linux \ + --bazel_version=${bazel_7.version} \ + --output_src_root=''${SRC_ROOT} \ + --output_config_path=generated-cc \ + --generate_java_configs=false \ + --generate_cpp_configs=true \ + --bazel_path=${bazel_7}/bin/bazel \ + --cpp_env_json=cpp_env.json - # The rbe_configs_gen tool automatically sets the exec_properties of the - # generated platform to the generator container name and tag. For efficiency - # reasons the actual deployment won't be the same as this generator - # container, so we modify this in the generated configuration. - sed -i \ - 's|rbe-autogen-lre-cc|lre-cc|g' \ - ''${SRC_ROOT}/generated-cc/config/BUILD + # The rbe_configs_gen tool automatically sets the exec_properties of the + # generated platform to the generator container name and tag. For efficiency + # reasons the actual deployment won't be the same as this generator + # container, so we modify this in the generated configuration. + sed -i \ + 's|rbe-autogen-lre-cc|lre-cc|g' \ + ''${SRC_ROOT}/generated-cc/config/BUILD - chmod 644 \ - ''${SRC_ROOT}/generated-cc/LICENSE \ - ''${SRC_ROOT}/generated-cc/config/BUILD \ + chmod 644 \ + ''${SRC_ROOT}/generated-cc/LICENSE \ + ''${SRC_ROOT}/generated-cc/config/BUILD \ - LRE_JAVA_IMAGE_TAG=$(nix eval .#lre-java.imageTag --raw) + LRE_JAVA_IMAGE_TAG=$(nix eval .#lre-java.imageTag --raw) - nix run .#rbe-autogen-lre-java.copyTo \ - docker-daemon:rbe-autogen-lre-java:''${LRE_JAVA_IMAGE_TAG} -L + nix run .#rbe-autogen-lre-java.copyTo \ + docker-daemon:rbe-autogen-lre-java:''${LRE_JAVA_IMAGE_TAG} -L - ${rbeConfigsGen}/bin/rbe_configs_gen \ - --toolchain_container=rbe-autogen-lre-java:''${LRE_JAVA_IMAGE_TAG} \ - --exec_os=linux \ - --target_os=linux \ - --bazel_version=${pkgs.bazel_7.version} \ - --output_src_root=''${SRC_ROOT} \ - --output_config_path=generated-java \ - --generate_java_configs=true \ - --generate_cpp_configs=false \ - --bazel_path=${pkgs.bazel_7}/bin/bazel \ - --cpp_env_json=cpp_env.json + ${rbe-configs-gen}/bin/rbe_configs_gen \ + --toolchain_container=rbe-autogen-lre-java:''${LRE_JAVA_IMAGE_TAG} \ + --exec_os=linux \ + --target_os=linux \ + --bazel_version=${bazel_7.version} \ + --output_src_root=''${SRC_ROOT} \ + --output_config_path=generated-java \ + --generate_java_configs=true \ + --generate_cpp_configs=false \ + --bazel_path=${bazel_7}/bin/bazel \ + --cpp_env_json=cpp_env.json - # See comment above for C++. - sed -i \ - 's|rbe-autogen-lre-java|lre-java|g' \ - ''${SRC_ROOT}/generated-java/config/BUILD + # See comment above for C++. + sed -i \ + 's|rbe-autogen-lre-java|lre-java|g' \ + ''${SRC_ROOT}/generated-java/config/BUILD - chmod 644 \ - ''${SRC_ROOT}/generated-java/LICENSE \ - ''${SRC_ROOT}/generated-java/config/BUILD \ - ''${SRC_ROOT}/generated-java/java/BUILD + chmod 644 \ + ''${SRC_ROOT}/generated-java/LICENSE \ + ''${SRC_ROOT}/generated-java/config/BUILD \ + ''${SRC_ROOT}/generated-java/java/BUILD - pre-commit run -a - '' + pre-commit run -a +'' diff --git a/tools/llvmStdenv.nix b/tools/llvmStdenv.nix index 5eedf9d77..d3115f04b 100644 --- a/tools/llvmStdenv.nix +++ b/tools/llvmStdenv.nix @@ -1,10 +1,11 @@ { - pkgs, + stdenv, + overrideCC, + useMoldLinker, llvmPackages, - ... }: let llvmToolchain = - pkgs.overrideCC ( + overrideCC ( llvmPackages.libcxxStdenv.override { targetPlatform.useLLVM = true; } @@ -12,9 +13,9 @@ llvmPackages.clangUseLLVM; toolchain = - if pkgs.stdenv.isDarwin + if stdenv.isDarwin then llvmToolchain # Mold doesn't support darwin. - else pkgs.useMoldLinker llvmToolchain; + else useMoldLinker llvmToolchain; in # This toolchain uses Clang as compiler, Mold as linker, libc++ as C++ # standard library and compiler-rt as compiler runtime. Resulting rust diff --git a/tools/local-image-test.nix b/tools/local-image-test.nix index fd5731379..ae0293f04 100644 --- a/tools/local-image-test.nix +++ b/tools/local-image-test.nix @@ -1,5 +1,9 @@ -{pkgs, ...}: -pkgs.writeShellScriptBin "local-image-test" '' +{ + dive, + trivy, + writeShellScriptBin, +}: +writeShellScriptBin "local-image-test" '' set -xeuo pipefail echo "Testing image: $1" @@ -16,9 +20,9 @@ pkgs.writeShellScriptBin "local-image-test" '' docker-daemon:''${IMAGE_NAME}:''${IMAGE_TAG} # Ensure that the image has minimal closure size. - CI=1 ${pkgs.dive}/bin/dive \ + CI=1 ${dive}/bin/dive \ ''${IMAGE_NAME}:''${IMAGE_TAG} \ --highestWastedBytes=0 - ${pkgs.trivy}/bin/trivy image ''${IMAGE_NAME}:''${IMAGE_TAG} + ${trivy}/bin/trivy image ''${IMAGE_NAME}:''${IMAGE_TAG} '' diff --git a/tools/nativelink-is-executable-test.nix b/tools/nativelink-is-executable-test.nix index 4734caf05..97cf52904 100644 --- a/tools/nativelink-is-executable-test.nix +++ b/tools/nativelink-is-executable-test.nix @@ -1,9 +1,8 @@ { - pkgs, nativelink, - ... + writeShellScriptBin, }: -pkgs.writeShellScriptBin "is-executable-test" '' +writeShellScriptBin "is-executable-test" '' set -xuo pipefail nativelink_output="$(${nativelink}/bin/nativelink 2>&1)" diff --git a/tools/publish-ghcr.nix b/tools/publish-ghcr.nix index 015c45637..4c9d39278 100644 --- a/tools/publish-ghcr.nix +++ b/tools/publish-ghcr.nix @@ -1,8 +1,13 @@ -{pkgs, ...}: -pkgs.writeShellScriptBin "publish-ghcr" '' +{ + writeShellScriptBin, + skopeo, + cosign, + trivy, +}: +writeShellScriptBin "publish-ghcr" '' set -xeuo pipefail - echo $GHCR_PASSWORD | ${pkgs.skopeo}/bin/skopeo \ + echo $GHCR_PASSWORD | ${skopeo}/bin/skopeo \ login \ --username=$GHCR_USERNAME \ --password-stdin \ @@ -22,23 +27,23 @@ pkgs.writeShellScriptBin "publish-ghcr" '' nix run .#$1.copyTo docker://''${TAGGED_IMAGE} - echo $GHCR_PASSWORD | ${pkgs.cosign}/bin/cosign \ + echo $GHCR_PASSWORD | ${cosign}/bin/cosign \ login \ --username=$GHCR_USERNAME \ --password-stdin \ ghcr.io - ${pkgs.cosign}/bin/cosign \ + ${cosign}/bin/cosign \ sign \ --yes \ ''${GHCR_REGISTRY,,}/''${IMAGE_NAME}@$( \ - ${pkgs.skopeo}/bin/skopeo \ + ${skopeo}/bin/skopeo \ inspect \ --format "{{ .Digest }}" \ docker://''${TAGGED_IMAGE} \ ) - ${pkgs.trivy}/bin/trivy \ + ${trivy}/bin/trivy \ image \ --format sarif \ ''${TAGGED_IMAGE} \