release-1.6: Backports for julia-1.6.4

.buildkite/cryptic_repo_keys/.gitignore

@@ -0,0 +1,5 @@
+# Ignore the unencrypted repo_key
+repo_key
+
+# Ignore any agent keys (public or private) we have stored
+agent_key*

.buildkite/pipelines/experimental/0_webui.yml

@@ -4,7 +4,6 @@
 agents:
   queue: "julia"
   sandbox.jl: "true"
-
 steps:
   - label: ":unlock: Unlock secrets, launch pipelines"
     plugins:
@@ -16,9 +15,8 @@ steps:
           # pipelines.
           unsigned_pipelines:
             - .buildkite/pipelines/experimental/launch_unsigned_builders.yml
-
           # Our signed pipelines must have a `signature` or `signature_file` parameter that
           # verifies the treehash of the pipeline itself and the inputs listed in `inputs`
           # signed_pipelines:
           #   - pipeline: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml
-          #     signature: "my_signature"
+          #     signature_file: .buildkite/pipelines/experimental/misc/foo_bar_baz.yml.signature

.buildkite/pipelines/experimental/launch_unsigned_builders.yml

@@ -1,6 +1,6 @@
 steps:
   - label: ":buildkite: Launch unsigned pipelines"
     commands: |
-      buildkite-agent pipeline upload .buildkite/pipelines/experimental/misc/sanitizers.yml
+      true
     agents:
       queue: julia

.buildkite/pipelines/main/0_webui.yml

@@ -4,7 +4,6 @@
 agents:
   queue: "julia"
   sandbox.jl: "true"
-
 steps:
   - label: ":unlock: Unlock secrets, launch pipelines"
     plugins:

.buildkite/pipelines/main/launch_unsigned_builders.yml

@@ -13,17 +13,21 @@
 # something about the privileged steps.
 
 steps:
-  - label: ":buildkite: Launch unsigned builders"
+  - label: ":buildkite: Launch unsigned jobs"
     commands: |
-      # First, we launch the `whitespace` builder, because we want that builder to finish as quickly as possible.
-      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
-
-      # Next, we launch the miscellaneous builders in alphabetical order.
+      # Launch the miscellaneous jobs in alphabetical order.
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
       buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml
 
-      # Finally, we launch the platform builders (`package_*`) and (`tester_*`) in alphabetical order.
-      buildkite-agent pipeline upload .buildkite/pipelines/main/platforms/linux64.yml
+      # Launch all of the platform jobs.
+      bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml
+      bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml
+
+      # Launch the `whitespace` job last. Uploading it last actually causes it to start
+      # first. We want this job to start first because we want it to finish as quickly
+      # as possible.
+      buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml
     agents:
       queue: julia

.buildkite/pipelines/main/misc/doctest.yml

@@ -3,24 +3,25 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "doctest"
     key: doctest
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
-          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
           uid: 1000
           gid: 1000
           workspaces:
             # Include `/cache/repos` so that our `git` version introspection works.
             - "/cache/repos:/cache/repos"
     commands: |
       echo "--- Build Julia from source"
-      make -j 6
+      make --output-sync -j 6
 
       echo "--- Print Julia version info"
       ./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
@@ -31,6 +32,3 @@ steps:
       echo "--- Run Julia doctests"
       JULIA_NUM_THREADS=1 make -C doc doctest=true
     timeout_in_minutes: 45
-    notify:
-      - github_commit_status:
-          context: "doctest"

.buildkite/pipelines/main/misc/embedding.yml

@@ -3,32 +3,29 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "embedding"
     key: "embedding"
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
-          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
           uid: 1000
           gid: 1000
           workspaces:
             # Include `/cache/repos` so that our `git` version introspection works.
             - "/cache/repos:/cache/repos"
     commands: |
       prefix="/tmp/prefix"
-      echo "+++ Build julia, deploy to $${prefix}"
-      make -j$${JULIA_NUM_CORES} JULIA_PRECOMPILE=0 prefix=$${prefix} install
+      echo "+++ Build julia, deploy to $${prefix:?}"
+      make --output-sync -j$${JULIA_CPU_THREADS:?} JULIA_PRECOMPILE=0 prefix=$${prefix:?} install
 
       embedding_output="/tmp/embedding-test"
-      echo "+++ Run embedding tests, deploy to $${embedding_output}"
-      mkdir -p "$${embedding_output}"
-      make -j$${JULIA_NUM_CORES} -C test/embedding JULIA="$${prefix}/bin/julia" BIN="$${embedding_output}"
-
+      echo "+++ Run embedding tests, deploy to $${embedding_output:?}"
+      mkdir -p "$${embedding_output:?}"
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/embedding JULIA="$${prefix:?}/bin/julia" BIN="$${embedding_output:?}"
     timeout_in_minutes: 60
-    notify:
-      - github_commit_status:
-          context: "embedding"

.buildkite/pipelines/main/misc/llvmpasses.yml

@@ -3,50 +3,46 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "analyzegc"
     key: "analyzegc"
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/llvm_passes.x86_64.tar.gz
-          rootfs_treehash: "9dd715500b117a16fcfa419ea0bca0c0ca902cee"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz
+          rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd"
           workspaces:
             # Include `/cache/repos` so that our `git` version introspection works.
             - "/cache/repos:/cache/repos"
     commands: |
       echo "--- Install in-tree LLVM dependencies"
-      make -j$${JULIA_NUM_CORES} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C deps install-llvm install-clang install-llvm-tools install-libuv install-utf8proc install-unwind
       echo "+++ run clangsa/analyzegc"
-      make -j$${JULIA_NUM_CORES} -C test/clangsa
-      make -j$${JULIA_NUM_CORES} -C src analyzegc
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/clangsa
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C src analyzegc
     timeout_in_minutes: 60
-    notify:
-      - github_commit_status:
-          context: "analyzegc"
-
   - label: "llvmpasses"
     key: "llvmpasses"
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.8/package_linux.x86_64.tar.gz
-          rootfs_treehash: "84a323ae8fcc724f8ea5aca5901bbbf4bda3e519"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
           uid: 1000
           gid: 1000
           workspaces:
             - "/cache/repos:/cache/repos"
     commands: |
       echo "--- make release"
-      make -j$${JULIA_NUM_CORES} release JULIA_PRECOMPILE=0
+      make --output-sync -j$${JULIA_CPU_THREADS:?} release JULIA_PRECOMPILE=0
       echo "--- make src/install-analysis-deps"
-      make -j$${JULIA_NUM_CORES} -C src install-analysis-deps
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C src install-analysis-deps
       echo "+++ make test/llvmpasses"
-      make -j$${JULIA_NUM_CORES} -C test/llvmpasses
+      make --output-sync -j$${JULIA_CPU_THREADS:?} -C test/llvmpasses
     timeout_in_minutes: 60
-    notify:
-      - github_commit_status:
-          context: "llvmpasses"

.buildkite/pipelines/main/misc/sanitizers.yml

@@ -0,0 +1,47 @@
+agents:
+  queue: "julia"
+  # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
+  sandbox.jl: "true"
+  os: "linux"
+steps:
+  - label: "asan"
+    key: "asan"
+    plugins:
+      - JuliaCI/julia#v1:
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz
+          rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            - "/cache/repos:/cache/repos"
+    timeout_in_minutes: 120
+    if: | # We only run the `asan` job on Julia 1.8 and later.
+      (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7")
+    soft_fail: true # TODO: delete this line (and thus disallow failures) once JuliaLang/julia#42540 is fixed
+    commands: |
+      echo "--- Build julia-debug with ASAN"
+      contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug
+  - label: "tsan"
+    key: "tsan"
+    plugins:
+      - JuliaCI/julia#v1:
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/llvm_passes.x86_64.tar.gz
+          rootfs_treehash: "c7a289a8cc544b234b1e2d7cbcce3e6815359ecd"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            - "/cache/repos:/cache/repos"
+    timeout_in_minutes: 120
+    if: | # We only run the `tsan` job on Julia 1.8 and later.
+      (pipeline.slug != "julia-release-1-dot-6") && (pipeline.slug != "julia-release-1-dot-7")
+    commands: |
+      echo "--- Build julia-debug runtime with TSAN"
+      contrib/tsan/build.sh ./tmp/test-tsan -j$${JULIA_CPU_THREADS:?} julia-src-debug

.buildkite/pipelines/main/misc/signed_pipeline_test.yml

@@ -5,13 +5,14 @@ agents:
 ## pipeline that showcases decryption of environment variable
 steps:
   - label: ":lock: :rocket: Signed pipeline test"
+    # We must accept the signed job id secret in order to propagate secrets
+    env:
+      BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}
+    depends_on:
     plugins:
       - staticfloat/cryptic#v1:
           variables:
             - SECRET_KEY="U2FsdGVkX18tb7st0SuQAvh4Yv4xENxOAu8q9XkmOeDVKBNY4FngEwK3xmiKUqaS"
     commands: |
       echo "SECRET_KEY: $${SECRET_KEY}"
 
-# We must accept the signed job id secret in order to propagate secrets
-env:
-  BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}

.buildkite/pipelines/main/misc/signed_pipeline_test.yml.signature

@@ -1 +1 @@
-Salted__��NE""�;��יܜ��3%���0-R�`�>t��QΪZ6-�N�����He��_5��[�d��zn�@�@B9���H�ۮ���'
+Salted__���J0�Q?���rۀ�g�~�d��ۛŧ�ө��o���Ujʀ���p�)�$�U$����y@gZM}{�m��,۠�K��e�r�

.buildkite/pipelines/main/misc/whitespace.yml

@@ -3,21 +3,22 @@ agents:
   # Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
   sandbox.jl: "true"
   os: "linux"
-
 steps:
   - label: "whitespace"
     key: "whitespace"
     plugins:
       - JuliaCI/julia#v1:
-          version: 1.6
+          # Drop default "registries" directory, so it is not persisted from execution to execution
+          persist_depot_dirs: packages,artifacts,compiled
+          version: '1.6'
       - staticfloat/sandbox#v1:
-          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/package_linux.x86_64.tar.gz
-          rootfs_treehash: "8c33c341a864852629b8aac01a6eb6a79b73570e"
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v4.8/package_linux.x86_64.tar.gz
+          rootfs_treehash: "2a058481b567f0e91b9aa3ce4ad4f09e6419355a"
           workspaces:
             - "/cache/repos:/cache/repos"
-    commands: |
-      make -j$${JULIA_NUM_CORES} check-whitespace
     timeout_in_minutes: 10
     notify:
       - github_commit_status:
           context: "whitespace"
+    commands: |
+      make --output-sync -j$${JULIA_CPU_THREADS:?} check-whitespace