Transition the coverage-linux64 pipeline to Buildkite (JuliaLang#41238

)

* Transition the `coverage-linux64` pipeline to Buildkite

* Simplify, run inside of a sandbox

* Upload coverage reports to Codecov and Coveralls

* Add `COVERALLS_TOKEN`

Co-authored-by: Elliot Saba <[email protected]>

.buildkite/coverage-linux64/0_webui.yml

@@ -0,0 +1,16 @@
+# This file represents what is put into the webUI.
+# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect.
+# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps.
+agents:
+  queue: "julia"
+  sandbox.jl: "true"
+
+steps:
+  - label: ":unlock: Unlock secrets, launch pipelines"
+    plugins:
+      - staticfloat/cryptic:
+          # Our signed pipelines must have a `signature` or `signature_file` parameter that
+          # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
+          signed_pipelines:
+            - pipeline: .buildkite/coverage-linux64/pipeline.yml
+              signature: "U2FsdGVkX18eQWpd3hMYLO5Kd+6K+oBoLk1I6J3qIw7lc6g5/jaeWyq/wralosZCfTzyjS4NstNKFvhQf3KDPEBVElipNvTxoWOjVLRVOrfBqqvTkQN4xVosY/r026Gy"

.buildkite/coverage-linux64/README.md

@@ -0,0 +1,6 @@
+# Coverage pipeline
+
+We run coverage on a separate pipeline, that uses a scheduled build rather than webhooks.
+The pipeline is here: https://buildkite.com/julialang/julia-coverage-linux64
+
+It contains [its own webui steps](0_webuiy.ml) (listed here in this repository for clarity) and its own [pipeline.yml](pipeline.yml).

.buildkite/coverage-linux64/pipeline.yml

@@ -0,0 +1,44 @@
+# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
+# since we need nestable sandboxing.  The rootfs images being used here are built from
+# the `.buildkite/rootfs_images/llvm-passes.jl` file.
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+
+steps:
+  - label: ":unlock: :coverage: Run coverage test"
+    plugins:
+      - staticfloat/cryptic:
+          variables:
+            - CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA=="
+            - COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw=="
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz
+          rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8"
+          uid: 1000
+          gid: 1000
+    commands: |
+      echo "--- Build Julia from source"
+      make -j 6
+
+      echo "--- Print Julia version info"
+      ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
+      ./julia -e '@info "" Sys.CPU_THREADS'
+      # this is necessary to make sure that the LibGit2 tests passes
+      git config --global init.defaultBranch master
+
+      echo "--- Run Julia tests with code coverage enabled"
+      # Run the actual tests
+      ./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_base.jl
+
+      echo "--- Process and upload coverage information"
+      ./julia .buildkite/coverage-linux64/upload_coverage.jl
+    timeout_in_minutes: 600 # 600 minutes = 10 hours
+
+# We must accept the signed job id secret in order to propagate secrets
+env:
+  BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}

.buildkite/coverage-linux64/run_tests_base.jl

@@ -0,0 +1,36 @@
+# When running this file, make sure to set the `--code-coverage=all` command-line flag.
+
+# Important note: even if one or more tests fail, we will still exit with status code 0.
+
+# The reason for this is that we always want to upload code coverage, even if some of the
+# tests fail. Therefore, even if the `coverage-linux64` pipeline passes, you should not
+# assume that all of the tests passed. If you want to know if all of the tests are passing,
+# please look at the status of the `tester_linux64` pipeline.
+
+const include_tests = String[]
+
+const exclude_tests = String[]
+
+empty!(Base.DEPOT_PATH)
+push!(Base.DEPOT_PATH, mktempdir(; cleanup = true))
+
+module ChooseTests
+    include(joinpath(dirname(dirname(@__DIR__)), "test", "choosetests.jl"))
+end
+
+const tests = ChooseTests.choosetests() |>
+              first |>
+              x -> setdiff(x, exclude_tests) |>
+              x -> vcat(x, include_tests) |>
+              unique |>
+              sort
+
+const ncores = min(Sys.CPU_THREADS, Threads.nthreads())
+
+@info "" ncores Sys.CPU_THREADS Threads.nthreads()
+
+try
+    Base.runtests(tests; ncores)
+catch ex
+    @error "" exception=(ex, catch_backtrace())
+end

.buildkite/coverage-linux64/upload_coverage.jl

@@ -0,0 +1,115 @@
+empty!(Base.DEPOT_PATH)
+push!(Base.DEPOT_PATH, mktempdir(; cleanup = true))
+
+import Pkg
+import Logging
+import TOML
+
+Pkg.add(; name = "Coverage", uuid = "a2441757-f6aa-5fb2-8edb-039e3f45d037", version = "1")
+Pkg.precompile()
+
+import Coverage
+
+function get_external_stdlib_names(stdlib_dir::AbstractString)
+    filename_list = filter(x -> isfile(joinpath(stdlib_dir, x)), readdir(stdlib_dir))
+    # find all of the files like `Pkg.version`, `Statistics.version`, etc.
+    regex_matches_or_nothing = match.(Ref(r"^([\w].*?)\.version$"), filename_list)
+    regex_matches = filter(x -> x !== nothing, regex_matches_or_nothing)
+    # get the names of the external stdlibs, like `Pkg`, `Statistics`, etc.
+    external_stdlib_names = only.(regex_matches)
+    unique!(external_stdlib_names)
+    sort!(external_stdlib_names)
+    @info "# Begin list of external stdlibs"
+    for (i, x) in enumerate(external_stdlib_names)
+        @info "$(i). $(x)"
+    end
+    @info "# End list of external stdlibs"
+    return external_stdlib_names
+end
+
+function get_external_stdlib_prefixes(stdlib_dir::AbstractString)
+    external_stdlib_names = get_external_stdlib_names(stdlib_dir)
+    prefixes_1 = joinpath.(Ref(stdlib_dir), external_stdlib_names, Ref(""))
+    prefixes_2 = joinpath.(Ref(stdlib_dir), string.(external_stdlib_names, Ref("-")))
+    prefixes = vcat(prefixes_1, prefixes_2)
+    unique!(prefixes)
+    sort!(prefixes)
+    # example of what `prefixes` might look like:
+    # 4-element Vector{String}:
+    # "stdlib/Pkg-"
+    # "stdlib/Pkg/"
+    # "stdlib/Statistics-"
+    # "stdlib/Statistics/"
+    return prefixes
+end
+
+function print_coverage_summary(fc::Coverage.FileCoverage)
+    cov_lines, tot_lines = Coverage.get_summary(fc)
+    if cov_lines == tot_lines == 0
+        cov_pct = 0
+    else
+        cov_pct = floor(Int, cov_lines/tot_lines * 100)
+    end
+    pad_1 = 71
+    pad_2 = 15
+    pad_3 = 15
+    col_1 = rpad(fc.filename, pad_1)
+    col_2 = rpad(string(cov_pct, " %"), pad_2)
+    col_3 = string(
+        rpad(string(cov_lines), pad_3),
+        string(tot_lines),
+    )
+    @info "$(col_1) $(col_2) $(col_3)"
+    return nothing
+end
+
+function print_coverage_summary(
+        fcs::Vector{Coverage.FileCoverage}, description::AbstractString,
+    )
+    cov_lines, tot_lines = Coverage.get_summary(fcs)
+    if cov_lines == tot_lines == 0
+        cov_pct = 0
+    else
+        cov_pct = floor(Int, cov_lines/tot_lines * 100)
+    end
+    @info "$(description): $(cov_pct)% ($(cov_lines)/$(tot_lines))"
+    return nothing
+end
+
+# `Coverage.process_folder` will have a LOT of `@info` statements that will make the log
+# way too long. So before we run `Coverage.process_folder`, we disable logging for `@info`
+# statements. After we run `Coverage.process_folder`, we re-enable logging for `@info`
+# statements.
+Logging.disable_logging(Logging.Info)
+const fcs = Coverage.merge_coverage_counts(
+    Coverage.process_folder("base"),
+    Coverage.process_folder("stdlib"),
+);
+Logging.disable_logging(Logging.Debug)
+
+# Only include source code files. Exclude test files, benchmarking files, etc.
+filter!(fcs) do fc
+    occursin(r"^base\/", fc.filename) || occursin("/src/", fc.filename)
+end;
+
+# Exclude all external stdlibs (stdlibs that live in external repos).
+const external_stdlib_prefixes = get_external_stdlib_prefixes("stdlib")
+filter!(fcs) do fc
+    all(x -> !startswith(fc.filename, x), external_stdlib_prefixes)
+end;
+
+# Exclude all stdlib JLLs (stdlibs of the form `stdlib/*_jll/`).
+filter!(fcs) do fc
+    !occursin(r"^stdlib\/[A-Za-z0-9]*?_jll\/", fc.filename)
+end
+
+sort!(fcs; by = fc -> fc.filename)
+
+print_coverage_summary.(fcs);
+print_coverage_summary(fcs, "Total")
+
+# In order to upload to Codecov, you need to have the `CODECOV_TOKEN` environment variable defined.
+Coverage.Codecov.submit_local(fcs)
+
+# In order to upload to Coveralls, you need to have the `COVERALLS_TOKEN` environment variable defined.
+Coverage.Coveralls.submit_local(fcs)