From dcd5024a04b73757bd48570f47361cf6d710e6cb Mon Sep 17 00:00:00 2001
From: Takafumi Arakaki <aka.tkf@gmail.com>
Date: Thu, 30 Sep 2021 23:24:27 -0400
Subject: [PATCH] Setup CI for TSAN

---
 .../experimental/misc/sanitizers.yml          | 18 ++++++
 Make.user.bak                                 |  1 +
 contrib/tsan/Make.user.tsan                   | 16 +++++
 contrib/tsan/build.sh                         | 61 +++++++++++++++++++
 tsan-suppressions                             |  1 +
 5 files changed, 97 insertions(+)
 create mode 120000 Make.user.bak
 create mode 100644 contrib/tsan/Make.user.tsan
 create mode 100755 contrib/tsan/build.sh
 create mode 100644 tsan-suppressions

diff --git a/.buildkite/pipelines/experimental/misc/sanitizers.yml b/.buildkite/pipelines/experimental/misc/sanitizers.yml
index 79af4905416fcc..31c9253cd1b746 100644
--- a/.buildkite/pipelines/experimental/misc/sanitizers.yml
+++ b/.buildkite/pipelines/experimental/misc/sanitizers.yml
@@ -23,3 +23,21 @@ steps:
     commands: |
       echo "--- Build julia-debug with ASAN"
       contrib/asan/build.sh ./tmp/test-asan -j$${JULIA_CPU_THREADS:?} debug
+  - label: "tsan"
+    key: "tsan"
+    plugins:
+      - JuliaCI/julia#v1:
+          version: 1.6
+      - staticfloat/sandbox#v1:
+          rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v3.1/llvm_passes.x86_64.tar.gz
+          rootfs_treehash: "9dd715500b117a16fcfa419ea0bca0c0ca902cee"
+          uid: 1000
+          gid: 1000
+          workspaces:
+            - "/cache/repos:/cache/repos"
+    timeout_in_minutes: 120
+    commands: |
+      echo "--- Build julia-debug runtime with TSAN"
+      contrib/tsan/build.sh ./tmp/test-tsan -j$${JULIA_CPU_THREADS:?} -C src debug
+      echo "--- BROKEN: Build julia-debug with TSAN"
+      ! make -C ./tmp/test-tsan/tsan -j$${JULIA_CPU_THREADS:?} debug
diff --git a/Make.user.bak b/Make.user.bak
new file mode 120000
index 00000000000000..4d1fd96deffb27
--- /dev/null
+++ b/Make.user.bak
@@ -0,0 +1 @@
+/home/arakaki/repos/hack/dot-repos/amdci2/home/arakaki/repos/watch/_wt/julia/tsan/Make.user
\ No newline at end of file
diff --git a/contrib/tsan/Make.user.tsan b/contrib/tsan/Make.user.tsan
new file mode 100644
index 00000000000000..3773832aafeb89
--- /dev/null
+++ b/contrib/tsan/Make.user.tsan
@@ -0,0 +1,16 @@
+TOOLCHAIN=$(BUILDROOT)/../toolchain
+BINDIR=$(TOOLCHAIN)/usr/bin
+TOOLDIR=$(TOOLCHAIN)/usr/tools
+
+# use our new toolchain
+USECLANG=1
+override CC=$(BINDIR)/clang
+override CXX=$(TOOLDIR)/clang++
+
+USE_BINARYBUILDER_LLVM=1
+
+override SANITIZE=1
+override SANITIZE_THREAD=1
+
+# default to a debug build for better line number reporting
+override JULIA_BUILD_MODE=debug
diff --git a/contrib/tsan/build.sh b/contrib/tsan/build.sh
new file mode 100755
index 00000000000000..67956d2dcaa68b
--- /dev/null
+++ b/contrib/tsan/build.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+#
+# Usage:
+#     contrib/tsan/build.sh <path> [<make_targets>...]
+#
+# Build TSAN-enabled julia.  Given a workspace directory <path>, build
+# TSAN-enabled julia in <path>/tsan.  Required toolss are install under
+# <path>/toolchain.  Note that the same <path> passed to `contrib/asan/build.sh`
+# can be used to share the toolchain used for ASAN.  This scripts also takes
+# optional <make_targets> arguments which are passed to `make`.  The default
+# make target is `debug`.
+
+set -ue
+
+# `$WORKSPACE` is a directory in which we create `toolchain` and `tsan`
+# sub-directories.
+WORKSPACE="$1"
+shift
+if [ "$WORKSPACE" = "" ]; then
+    echo "Workspace directory must be specified as the first argument" >&2
+    exit 2
+fi
+
+mkdir -pv "$WORKSPACE"
+WORKSPACE="$(cd "$WORKSPACE" && pwd)"
+if [ "$WORKSPACE" = "" ]; then
+    echo "Failed to create the workspace directory." >&2
+    exit 2
+fi
+
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+JULIA_HOME="$HERE/../../"
+
+echo
+echo "Installing toolchain..."
+
+TOOLCHAIN="$WORKSPACE/toolchain"
+if [ ! -d "$TOOLCHAIN" ]; then
+    make -C "$JULIA_HOME" configure O=$TOOLCHAIN
+    cp "$HERE/../asan/Make.user.tools"  "$TOOLCHAIN/Make.user"
+fi
+
+make -C "$TOOLCHAIN/deps" install-clang install-llvm-tools
+
+# TODO: https://github.com/JuliaPackaging/Yggdrasil/issues/3359
+rm "$TOOLCHAIN/usr/tools/clang++"
+ln -s "$TOOLCHAIN/usr/bin/clang" "$TOOLCHAIN/usr/tools/clang++"
+
+echo
+echo "Building Julia..."
+
+BUILD="$WORKSPACE/tsan"
+if [ ! -d "$BUILD" ]; then
+    make -C "$JULIA_HOME" configure O="$BUILD"
+    cp "$HERE/Make.user.tsan"  "$BUILD/Make.user"
+fi
+
+cd "$BUILD"  # so that we can pass `-C src` to `make`
+make "$@"
diff --git a/tsan-suppressions b/tsan-suppressions
new file mode 100644
index 00000000000000..1a3c21513bbfbb
--- /dev/null
+++ b/tsan-suppressions
@@ -0,0 +1 @@
+deadlock:llvm