CI (Buildkite): Add package_linux and tester_linux for more Linux…

… architectures
JuliaLang · Sep 17, 2021 · 0f5ae4b · 0f5ae4b
1 parent c5f3487
commit 0f5ae4b
Show file tree

Hide file tree

Showing 19 changed files with 457 additions and 186 deletions.
diff --git a/.buildkite/pipelines/experimental/0_webui.yml b/.buildkite/pipelines/experimental/0_webui.yml
@@ -4,7 +4,6 @@
 agents:
   queue: "julia"
   sandbox.jl: "true"
-
 steps:
   - label: ":unlock: Unlock secrets, launch pipelines"
     plugins:
@@ -16,9 +15,8 @@ steps:
           # pipelines.
           unsigned_pipelines:
             - .buildkite/pipelines/experimental/launch_unsigned_builders.yml
-
           # Our signed pipelines must have a `signature` or `signature_file` parameter that
           # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
           # signed_pipelines:
           #   - pipeline: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml
-          #     signature: "my_signature"
+          #     signature_file: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml.signature
diff --git a/.buildkite/pipelines/experimental/misc/sanitizers.yml b/.buildkite/pipelines/experimental/misc/sanitizers.yml
@@ -3,10 +3,9 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "asan"
-    key: asan
+    key: "asan"
     plugins:
       - JuliaCI/julia#v1:
           version: 1.6
@@ -17,13 +16,10 @@ steps:
           gid: 1000
           workspaces:
             - "/cache/repos:/cache/repos"
-      # `contrib/check-asan.jl` needs a `julia` binary:
+      # `contrib/check-asan.jl` needs a `julia` binary inside the inner sandbox:
       - JuliaCI/julia#v1:
           version: 1.6
     timeout_in_minutes: 120
-    # notify:
-    #   - github_commit_status:
-    #       context: "asan"
     commands: |
       echo "--- Build julia-debug with ASAN"
-      contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_NUM_CORES} debug
+      contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug
diff --git a/.buildkite/pipelines/main/0_webui.yml b/.buildkite/pipelines/main/0_webui.yml
@@ -4,7 +4,6 @@
 agents:
   queue: "julia"
   sandbox.jl: "true"
-
 steps:
   - label: ":unlock: Unlock secrets, launch pipelines"
     plugins:

diff --git a/.buildkite/pipelines/main/launch_unsigned_builders.yml b/.buildkite/pipelines/main/launch_unsigned_builders.yml
@@ -15,15 +15,19 @@
 steps:
   - label: ":buildkite: Launch unsigned builders"
     commands: |
-      # First, we launch the `whitespace` builder, because we want that builder to finish as quickly as possible.
+      # Launch the `whitespace` builder first, because we want that builder to finish as quickly as possible.
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
 
-      # Next, we launch the miscellaneous builders in alphabetical order.
+      # Launch the miscellaneous builders in alphabetical order.
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
 
-      # Finally, we launch the platform builders (`package_*`) and (`tester_*`) in alphabetical order.
-      buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux64.yml
+      # Launch all of the platform builders.
+      bash .buildkite/pipelines/main/platforms/platforms.sh package_linux
+      bash .buildkite/pipelines/main/platforms/platforms.sh tester_linux
+
+      # Launch the "dummy job" for creating grouped commit statuses.
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/commit_statuses.yml
     agents:
       queue: julia
diff --git a/.buildkite/pipelines/main/misc/commit_statuses.yml b/.buildkite/pipelines/main/misc/commit_statuses.yml
@@ -0,0 +1,99 @@
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+steps:
+  - label: "analyze"
+    key: "analyze"
+    depends_on:
+      - "analyzegc"
+      - "asan"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
+          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            # Include `/cache/repos` so that our `git` version introspection works.
+            - "/cache/repos:/cache/repos"
+    timeout_in_minutes: 10
+    notify:
+      - github_commit_status:
+          context: "analyze"
+    commands: |
+      echo "All `analyze` jobs completed successfully."
+  - label: "docs"
+    key: "docs"
+    depends_on:
+      - "doctest"
+      # - "docdeploy"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
+          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            # Include `/cache/repos` so that our `git` version introspection works.
+            - "/cache/repos:/cache/repos"
+    timeout_in_minutes: 10
+    notify:
+      - github_commit_status:
+          context: "docs"
+    commands: |
+      echo "All `docs` jobs completed successfully."
+  - label: "package"
+    key: "package"
+    depends_on:
+      - "package_linux32"
+      - "package_linux64"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
+          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            # Include `/cache/repos` so that our `git` version introspection works.
+            - "/cache/repos:/cache/repos"
+    timeout_in_minutes: 10
+    notify:
+      - github_commit_status:
+          context: "package"
+    commands: |
+      echo "All `package` jobs completed successfully."
+  - label: "test"
+    key: "test"
+    depends_on:
+      - "embedding"
+      - "llvmpasses"
+      - "tester_linux32"
+      - "tester_linux64_st"
+      - "tester_linux64_mt"
+      - "tester_linux64_st_rr"
+      - "tester_linux64_mt_rr"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
+          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            # Include `/cache/repos` so that our `git` version introspection works.
+            - "/cache/repos:/cache/repos"
+    timeout_in_minutes: 10
+    notify:
+      - github_commit_status:
+          context: "test"
+    commands: |
+      echo "All `test` jobs completed successfully."
diff --git a/.buildkite/pipelines/main/misc/doctest.yml b/.buildkite/pipelines/main/misc/doctest.yml
@@ -3,7 +3,6 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "doctest"
     key: doctest
@@ -31,6 +30,3 @@ steps:
       echo "--- Run Julia doctests"
       JULIA_NUM_THREADS=1 make -C doc doctest=true
     timeout_in_minutes: 45
-    notify:
-      - github_commit_status:
-          context: "doctest"
diff --git a/.buildkite/pipelines/main/misc/embedding.yml b/.buildkite/pipelines/main/misc/embedding.yml
@@ -3,7 +3,6 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "embedding"
     key: "embedding"
@@ -20,15 +19,11 @@ steps:
             - "/cache/repos:/cache/repos"
     commands: |
       prefix="/tmp/prefix"
-      echo "+++ Build julia, deploy to $${prefix}"
-      make -j$${JULIA_NUM_CORES} JULIA_PRECOMPILE=0 prefix=$${prefix} install
+      echo "+++ Build julia, deploy to $${prefix:?}"
+      make -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install
 
       embedding_output="/tmp/embedding-test"
-      echo "+++ Run embedding tests, deploy to $${embedding_output}"
-      mkdir -p "$${embedding_output}"
-      make -j$${JULIA_NUM_CORES} -C test/embedding JULIA="$${prefix}/bin/julia" BIN="$${embedding_output}"
-
+      echo "+++ Run embedding tests, deploy to $${embedding_output:?}"
+      mkdir -p "$${embedding_output:?}"
+      make -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}"
     timeout_in_minutes: 60
-    notify:
-      - github_commit_status:
-          context: "embedding"
diff --git a/.buildkite/pipelines/main/misc/llvmpasses.yml b/.buildkite/pipelines/main/misc/llvmpasses.yml
@@ -3,7 +3,6 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "analyzegc"
     key: "analyzegc"
@@ -18,15 +17,11 @@ steps:
             - "/cache/repos:/cache/repos"
     commands: |
       echo "--- Install in-tree LLVM dependencies"
-      make -j$${JULIA_NUM_CORES} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind
+      make -j$${JULIA_CPU_THREADS:?} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind
       echo "+++ run clangsa/analyzegc"
-      make -j$${JULIA_NUM_CORES} -C test/clangsa
-      make -j$${JULIA_NUM_CORES} -C src analyzegc
+      make -j$${JULIA_CPU_THREADS:?} -C test/clangsa
+      make -j$${JULIA_CPU_THREADS:?} -C src analyzegc
     timeout_in_minutes: 60
-    notify:
-      - github_commit_status:
-          context: "analyzegc"
-
   - label: "llvmpasses"
     key: "llvmpasses"
     plugins:
@@ -41,12 +36,9 @@ steps:
             - "/cache/repos:/cache/repos"
     commands: |
       echo "--- make release"
-      make -j$${JULIA_NUM_CORES} release JULIA_PRECOMPILE=0
+      make -j$${JULIA_CPU_THREADS:?} release JULIA_PRECOMPILE=0
       echo "--- make src/install-analysis-deps"
-      make -j$${JULIA_NUM_CORES} -C src install-analysis-deps
+      make -j$${JULIA_CPU_THREADS:?} -C src install-analysis-deps
       echo "+++ make test/llvmpasses"
-      make -j$${JULIA_NUM_CORES} -C test/llvmpasses
+      make -j$${JULIA_CPU_THREADS:?} -C test/llvmpasses
     timeout_in_minutes: 60
-    notify:
-      - github_commit_status:
-          context: "llvmpasses"
diff --git a/.buildkite/pipelines/main/misc/whitespace.yml b/.buildkite/pipelines/main/misc/whitespace.yml
@@ -3,7 +3,6 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "whitespace"
     key: "whitespace"
@@ -15,9 +14,9 @@ steps:
           rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
           workspaces:
             - "/cache/repos:/cache/repos"
-    commands: |
-      make -j$${JULIA_NUM_CORES} check-whitespace
     timeout_in_minutes: 10
     notify:
       - github_commit_status:
           context: "whitespace"
+    commands: |
+      make -j$${JULIA_CPU_THREADS:?} check-whitespace
diff --git a/.buildkite/pipelines/main/platforms/linux64.yml b/.buildkite/pipelines/main/platforms/linux64.yml
diff --git a/.buildkite/pipelines/main/platforms/package_linux.arches b/.buildkite/pipelines/main/platforms/package_linux.arches
@@ -0,0 +1,6 @@
+# ARCH      ARCH_LABEL      ROOTFS_ARCH      TIMEOUT      ROOTFS_TAG      ROOTFS_TREE
+# aarch64   aarch64         aarch64          60           v3.2            0566841e29f0f9880541c26a6595fd5ce0beb5ff
+# armv7l    armv7l          armv7l           60           v3.2            fb359370b052a47ce5c84cc6b4a7a03ed7053b25
+32          32              i686             60           v3.2            209c4db679a515befd7fb50ecc6bfbecf7ec3d32
+# ppc64le   ppc64le         powerpc64le      60           v3.2            c03a0158b19d48ac84b426834fce0d3584cdd0c7
+64          64              x86_64           60           v3.2            474bf61a926b2d7fcf202284d59d4b11a04601d7