Add memcpm test

.github/new-prs-labeler.yml

@@ -90,6 +90,10 @@ compiler-rt:sanitizer:
   - compiler-rt/lib/scudo/**
   - compiler-rt/test/scudo/**
 
+compiler-rt:scudo:
+  - compiler-rt/lib/scudo/**
+  - compiler-rt/test/scudo/**
+
 xray:
   - llvm/tools/llvm-xray/**
   - compiler-rt/*/xray/**

.github/workflows/pr-code-format.yml

@@ -0,0 +1,54 @@
+name: "Check code formatting"
+on: pull_request_target
+permissions:
+  pull-requests: write
+
+jobs:
+  code_formatter:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fetch LLVM sources
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v39
+        with:
+          separator: ","
+          fetch_depth: 10 # Fetches only the last 10 commits
+
+      - name: "Listed files"
+        run: |
+          echo "Formatting files:"
+          echo "${{ steps.changed-files.outputs.all_changed_files }}"
+
+      - name: Install clang-format
+        uses: aminya/setup-cpp@v1
+        with:
+          clangformat: 16.0.6
+
+      - name: Setup Python env
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+          cache-dependency-path: 'llvm/utils/git/requirements_formatting.txt'
+
+      - name: Install python dependencies
+        run: pip install -r llvm/utils/git/requirements_formatting.txt
+
+      - name: Run code formatter
+        env:
+          GITHUB_PR_NUMBER: ${{ github.event.pull_request.number }}
+          START_REV: ${{ github.event.pull_request.base.sha }}
+          END_REV: ${{ github.event.pull_request.head.sha }}
+          CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+        run: |
+          python llvm/utils/git/code-format-helper.py \
+            --token ${{ secrets.GITHUB_TOKEN }} \
+            --issue-number $GITHUB_PR_NUMBER \
+            --start-rev $START_REV \
+            --end-rev $END_REV \
+            --changed-files "$CHANGED_FILES"

bolt/include/bolt/Core/MCPlusBuilder.h

@@ -29,6 +29,7 @@
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ErrorOr.h"
+#include "llvm/Support/RWMutex.h"
 #include <cassert>
 #include <cstdint>
 #include <map>
@@ -166,10 +167,15 @@ class MCPlusBuilder {
   /// Names of non-standard annotations.
   SmallVector<std::string, 8> AnnotationNames;
 
+  /// A mutex that is used to control parallel accesses to
+  /// AnnotationNameIndexMap and AnnotationsNames.
+  mutable llvm::sys::RWMutex AnnotationNameMutex;
+
   /// Allocate the TailCall annotation value. Clients of the target-specific
   /// MCPlusBuilder classes must use convert/lower/create* interfaces instead.
   void setTailCall(MCInst &Inst);
 
+public:
   /// Transfer annotations from \p SrcInst to \p DstInst.
   void moveAnnotations(MCInst &&SrcInst, MCInst &DstInst) const {
     assert(!getAnnotationInst(DstInst) &&
@@ -182,7 +188,6 @@ class MCPlusBuilder {
     removeAnnotationInst(SrcInst);
   }
 
-public:
   /// Return iterator range covering def operands.
   iterator_range<MCInst::iterator> defOperands(MCInst &Inst) const {
     return make_range(Inst.begin(),
@@ -621,6 +626,11 @@ class MCPlusBuilder {
     return Info->get(Inst.getOpcode()).mayStore();
   }
 
+  virtual bool isAArch64Exclusive(const MCInst &Inst) const {
+    llvm_unreachable("not implemented");
+    return false;
+  }
+
   virtual bool isCleanRegXOR(const MCInst &Inst) const {
     llvm_unreachable("not implemented");
     return false;
@@ -1775,6 +1785,7 @@ class MCPlusBuilder {
 
   /// Return annotation index matching the \p Name.
   std::optional<unsigned> getAnnotationIndex(StringRef Name) const {
+    std::shared_lock<llvm::sys::RWMutex> Lock(AnnotationNameMutex);
     auto AI = AnnotationNameIndexMap.find(Name);
     if (AI != AnnotationNameIndexMap.end())
       return AI->second;
@@ -1784,10 +1795,10 @@ class MCPlusBuilder {
   /// Return annotation index matching the \p Name. Create a new index if the
   /// \p Name wasn't registered previously.
   unsigned getOrCreateAnnotationIndex(StringRef Name) {
-    auto AI = AnnotationNameIndexMap.find(Name);
-    if (AI != AnnotationNameIndexMap.end())
-      return AI->second;
+    if (std::optional<unsigned> Index = getAnnotationIndex(Name))
+      return *Index;
 
+    std::unique_lock<llvm::sys::RWMutex> Lock(AnnotationNameMutex);
     const unsigned Index =
         AnnotationNameIndexMap.size() + MCPlus::MCAnnotation::kGeneric;
     AnnotationNameIndexMap.insert(std::make_pair(Name, Index));

bolt/lib/Passes/FixRISCVCallsPass.cpp

@@ -19,13 +19,15 @@ void FixRISCVCallsPass::runOnFunction(BinaryFunction &BF) {
         auto *Target = MIB->getTargetSymbol(*II);
         assert(Target && "Cannot find call target");
 
+        MCInst OldCall = *II;
         auto L = BC.scopeLock();
 
         if (MIB->isTailCall(*II))
           MIB->createTailCall(*II, Target, Ctx);
         else
           MIB->createCall(*II, Target, Ctx);
 
+        MIB->moveAnnotations(std::move(OldCall), *II);
         ++II;
         continue;
       }
@@ -39,8 +41,19 @@ void FixRISCVCallsPass::runOnFunction(BinaryFunction &BF) {
         auto *Target = MIB->getTargetSymbol(*II);
         assert(Target && "Cannot find call target");
 
+        MCInst OldCall = *NextII;
         auto L = BC.scopeLock();
         MIB->createCall(*II, Target, Ctx);
+        MIB->moveAnnotations(std::move(OldCall), *II);
+
+        // The original offset was set on the jalr of the auipc+jalr pair. Since
+        // the whole pair is replaced by a call, adjust the offset by -4 (the
+        // size of a auipc).
+        if (std::optional<uint32_t> Offset = MIB->getOffset(*II)) {
+          assert(*Offset >= 4 && "Illegal jalr offset");
+          MIB->setOffset(*II, *Offset - 4);
+        }
+
         II = BB.eraseInstruction(NextII);
         continue;
       }

bolt/lib/Passes/Instrumentation.cpp

@@ -13,6 +13,7 @@
 #include "bolt/Passes/Instrumentation.h"
 #include "bolt/Core/ParallelUtilities.h"
 #include "bolt/RuntimeLibs/InstrumentationRuntimeLibrary.h"
+#include "bolt/Utils/CommandLineOpts.h"
 #include "bolt/Utils/Utils.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/RWMutex.h"
@@ -85,6 +86,24 @@ cl::opt<bool> InstrumentCalls("instrument-calls",
 namespace llvm {
 namespace bolt {
 
+static bool hasAArch64ExclusiveMemop(BinaryFunction &Function) {
+  // FIXME ARMv8-a architecture reference manual says that software must avoid
+  // having any explicit memory accesses between exclusive load and associated
+  // store instruction. So for now skip instrumentation for functions that have
+  // these instructions, since it might lead to runtime deadlock.
+  BinaryContext &BC = Function.getBinaryContext();
+  for (const BinaryBasicBlock &BB : Function)
+    for (const MCInst &Inst : BB)
+      if (BC.MIB->isAArch64Exclusive(Inst)) {
+        if (opts::Verbosity >= 1)
+          outs() << "BOLT-INSTRUMENTER: Function " << Function
+                 << " has exclusive instructions, skip instrumentation\n";
+        return true;
+      }
+
+  return false;
+}
+
 uint32_t Instrumentation::getFunctionNameIndex(const BinaryFunction &Function) {
   auto Iter = FuncToStringIdx.find(&Function);
   if (Iter != FuncToStringIdx.end())
@@ -288,6 +307,9 @@ void Instrumentation::instrumentFunction(BinaryFunction &Function,
   if (BC.isMachO() && Function.hasName("___GLOBAL_init_65535/1"))
     return;
 
+  if (BC.isAArch64() && hasAArch64ExclusiveMemop(Function))
+    return;
+
   SplitWorklistTy SplitWorklist;
   SplitInstrsTy SplitInstrs;
 

bolt/lib/Passes/ReorderAlgorithm.cpp

@@ -531,21 +531,21 @@ void ExtTSPReorderAlgorithm::reorderBasicBlocks(BinaryFunction &BF,
   }
 
   // Initialize CFG edges
-  using JumpT = std::pair<uint64_t, uint64_t>;
-  std::vector<std::pair<JumpT, uint64_t>> JumpCounts;
+  std::vector<codelayout::EdgeCount> JumpCounts;
   for (BinaryBasicBlock *BB : BF.getLayout().blocks()) {
     auto BI = BB->branch_info_begin();
     for (BinaryBasicBlock *SuccBB : BB->successors()) {
       assert(BI->Count != BinaryBasicBlock::COUNT_NO_PROFILE &&
              "missing profile for a jump");
-      auto It = std::make_pair(BB->getLayoutIndex(), SuccBB->getLayoutIndex());
-      JumpCounts.push_back(std::make_pair(It, BI->Count));
+      JumpCounts.push_back(
+          {BB->getLayoutIndex(), SuccBB->getLayoutIndex(), BI->Count});
       ++BI;
     }
   }
 
   // Run the layout algorithm
-  auto Result = applyExtTspLayout(BlockSizes, BlockCounts, JumpCounts);
+  auto Result =
+      codelayout::computeExtTspLayout(BlockSizes, BlockCounts, JumpCounts);
   Order.reserve(BF.getLayout().block_size());
   for (uint64_t R : Result)
     Order.push_back(OrigOrder[R]);

bolt/lib/Passes/ReorderFunctions.cpp

@@ -331,23 +331,21 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
     // Initialize CFG nodes and their data
     std::vector<uint64_t> FuncSizes;
     std::vector<uint64_t> FuncCounts;
-    using JumpT = std::pair<uint64_t, uint64_t>;
-    std::vector<std::pair<JumpT, uint64_t>> CallCounts;
+    std::vector<codelayout::EdgeCount> CallCounts;
     std::vector<uint64_t> CallOffsets;
     for (NodeId F = 0; F < Cg.numNodes(); ++F) {
       FuncSizes.push_back(Cg.size(F));
       FuncCounts.push_back(Cg.samples(F));
       for (NodeId Succ : Cg.successors(F)) {
         const Arc &Arc = *Cg.findArc(F, Succ);
-        auto It = std::make_pair(F, Succ);
-        CallCounts.push_back(std::make_pair(It, Arc.weight()));
+        CallCounts.push_back({F, Succ, uint64_t(Arc.weight())});
         CallOffsets.push_back(uint64_t(Arc.avgCallOffset()));
       }
     }
 
     // Run the layout algorithm.
-    std::vector<uint64_t> Result =
-        applyCDSLayout(FuncSizes, FuncCounts, CallCounts, CallOffsets);
+    std::vector<uint64_t> Result = codelayout::computeCacheDirectedLayout(
+        FuncSizes, FuncCounts, CallCounts, CallOffsets);
 
     // Create a single cluster from the computed order of hot functions.
     std::vector<CallGraph::NodeId> NodeOrder(Result.begin(), Result.end());

bolt/lib/Target/AArch64/AArch64MCPlusBuilder.cpp

@@ -272,6 +272,34 @@ class AArch64MCPlusBuilder : public MCPlusBuilder {
     return isLDRB(Inst) || isLDRH(Inst) || isLDRW(Inst) || isLDRX(Inst);
   }
 
+  bool isAArch64Exclusive(const MCInst &Inst) const override {
+    return (Inst.getOpcode() == AArch64::LDXPX ||
+            Inst.getOpcode() == AArch64::LDXPW ||
+            Inst.getOpcode() == AArch64::LDXRX ||
+            Inst.getOpcode() == AArch64::LDXRW ||
+            Inst.getOpcode() == AArch64::LDXRH ||
+            Inst.getOpcode() == AArch64::LDXRB ||
+            Inst.getOpcode() == AArch64::STXPX ||
+            Inst.getOpcode() == AArch64::STXPW ||
+            Inst.getOpcode() == AArch64::STXRX ||
+            Inst.getOpcode() == AArch64::STXRW ||
+            Inst.getOpcode() == AArch64::STXRH ||
+            Inst.getOpcode() == AArch64::STXRB ||
+            Inst.getOpcode() == AArch64::LDAXPX ||
+            Inst.getOpcode() == AArch64::LDAXPW ||
+            Inst.getOpcode() == AArch64::LDAXRX ||
+            Inst.getOpcode() == AArch64::LDAXRW ||
+            Inst.getOpcode() == AArch64::LDAXRH ||
+            Inst.getOpcode() == AArch64::LDAXRB ||
+            Inst.getOpcode() == AArch64::STLXPX ||
+            Inst.getOpcode() == AArch64::STLXPW ||
+            Inst.getOpcode() == AArch64::STLXRX ||
+            Inst.getOpcode() == AArch64::STLXRW ||
+            Inst.getOpcode() == AArch64::STLXRH ||
+            Inst.getOpcode() == AArch64::STLXRB ||
+            Inst.getOpcode() == AArch64::CLREX);
+  }
+
   bool isLoadFromStack(const MCInst &Inst) const {
     if (!mayLoad(Inst))
       return false;

bolt/test/AArch64/exclusive-instrument.s

@@ -0,0 +1,39 @@
+// This test checks that the foo function having exclusive memory access
+// instructions won't be instrumented.
+
+// REQUIRES: system-linux,bolt-runtime,target=aarch64{{.*}}
+
+// RUN: llvm-mc -filetype=obj -triple aarch64-unknown-unknown \
+// RUN:   %s -o %t.o
+// RUN: %clang %cflags -fPIC -pie %t.o -o %t.exe -nostdlib -Wl,-q -Wl,-fini=dummy
+// RUN: llvm-bolt %t.exe -o %t.bolt -instrument -v=1 | FileCheck %s
+
+// CHECK: Function foo has exclusive instructions, skip instrumentation
+
+.global foo
+.type foo, %function
+foo:
+  ldaxr w9, [x10]
+  cbnz w9, .Lret
+  stlxr w12, w11, [x9]
+  cbz w12, foo
+  clrex
+.Lret:
+  ret
+.size foo, .-foo
+
+.global _start
+.type _start, %function
+_start:
+  cmp x0, #0
+  b.eq .Lexit
+  bl foo
+.Lexit:
+  ret
+.size _start, .-_start
+
+.global dummy
+.type dummy, %function
+dummy:
+  ret
+.size dummy, .-dummy