From e69fe9bf465ec8d74b8c2b207bf7a6931447d618 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 19 Feb 2022 19:55:02 -0500 Subject: [PATCH 1/4] [LLVM][PM] Make LowerExcHandlers NewPM compatible --- src/aotcompile.cpp | 4 ++ src/llvm-lower-handlers.cpp | 65 +++++++++++++++---------------- src/passes.h | 6 +++ test/llvmpasses/lower-handlers.ll | 1 + 4 files changed, 43 insertions(+), 33 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 6947d08600741..33b8b030263c7 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -899,6 +899,10 @@ static void registerCallbacks(PassBuilder &PB) { PM.addPass(PropagateJuliaAddrspacesPass()); return true; } + if (Name == "LowerExcHandlers") { + PM.addPass(LowerExcHandlers()); + return true; + } return false; }); diff --git a/src/llvm-lower-handlers.cpp b/src/llvm-lower-handlers.cpp index 045056805bddd..af9f3695dcf88 100644 --- a/src/llvm-lower-handlers.cpp +++ b/src/llvm-lower-handlers.cpp @@ -1,6 +1,7 @@ // This file is a part of Julia. License is MIT: https://julialang.org/license #include "llvm-version.h" +#include "passes.h" #include #include @@ -70,23 +71,8 @@ using namespace llvm; * handler structures to tell LLVM that it is free to re-use the stack slot * while the handler is not being used. */ -struct LowerExcHandlers : public FunctionPass { - static char ID; - LowerExcHandlers() : FunctionPass(ID) - {} - -private: - Function *except_enter_func; - Function *leave_func; - Function *jlenter_func; - Function *setjmp_func; - Function *lifetime_start; - Function *lifetime_end; - - bool doInitialization(Module &M) override; - bool runOnFunction(Function &F) override; -}; +namespace { /* * If the module doesn't have declarations for the jl_enter_handler and setjmp * functions, insert them. @@ -115,24 +101,19 @@ static void ensure_enter_function(Module &M) } } -bool LowerExcHandlers::doInitialization(Module &M) { - except_enter_func = M.getFunction("julia.except_enter"); +static bool lowerExcHandlers(Function &F) { + Module &M = *F.getParent(); + Function *except_enter_func = M.getFunction("julia.except_enter"); if (!except_enter_func) - return false; + return false; // No EH frames in this module ensure_enter_function(M); - leave_func = M.getFunction(XSTR(jl_pop_handler)); - jlenter_func = M.getFunction(XSTR(jl_enter_handler)); - setjmp_func = M.getFunction(jl_setjmp_name); + Function *leave_func = M.getFunction(XSTR(jl_pop_handler)); + Function *jlenter_func = M.getFunction(XSTR(jl_enter_handler)); + Function *setjmp_func = M.getFunction(jl_setjmp_name); auto T_pint8 = Type::getInt8PtrTy(M.getContext(), 0); - lifetime_start = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_start, { T_pint8 }); - lifetime_end = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_end, { T_pint8 }); - return true; -} - -bool LowerExcHandlers::runOnFunction(Function &F) { - if (!except_enter_func) - return false; // No EH frames in this module + Function *lifetime_start = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_start, { T_pint8 }); + Function *lifetime_end = Intrinsic::getDeclaration(&M, Intrinsic::lifetime_end, { T_pint8 }); /* Step 1: EH Depth Numbering */ std::map EnterDepth; @@ -237,14 +218,32 @@ bool LowerExcHandlers::runOnFunction(Function &F) { return true; } -char LowerExcHandlers::ID = 0; -static RegisterPass X("LowerExcHandlers", "Lower Julia Exception Handlers", +} // anonymous namespace + +PreservedAnalyses LowerExcHandlers::run(Function &F, FunctionAnalysisManager &AM) +{ + lowerExcHandlers(F); + return PreservedAnalyses::all(); +} + + +struct LowerExcHandlersLegacy : public FunctionPass { + static char ID; + LowerExcHandlersLegacy() : FunctionPass(ID) + {} + bool runOnFunction(Function &F) { + return lowerExcHandlers(F); + } +}; + +char LowerExcHandlersLegacy::ID = 0; +static RegisterPass X("LowerExcHandlers", "Lower Julia Exception Handlers", false /* Only looks at CFG */, false /* Analysis Pass */); Pass *createLowerExcHandlersPass() { - return new LowerExcHandlers(); + return new LowerExcHandlersLegacy(); } extern "C" JL_DLLEXPORT void LLVMExtraAddLowerExcHandlersPass_impl(LLVMPassManagerRef PM) diff --git a/src/passes.h b/src/passes.h index 57cf7dcef8e21..4b73eb2ff7684 100644 --- a/src/passes.h +++ b/src/passes.h @@ -26,11 +26,17 @@ struct LateLowerGC : PassInfoMixin { struct AllocOptPass : PassInfoMixin { PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); }; + struct PropagateJuliaAddrspacesPass : PassInfoMixin { PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); static bool isRequired() { return true; } }; +struct LowerExcHandlers : PassInfoMixin { + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + static bool isRequired() { return true; } +}; + // Module Passes struct CPUFeatures : PassInfoMixin { PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); diff --git a/test/llvmpasses/lower-handlers.ll b/test/llvmpasses/lower-handlers.ll index 80b6be683c024..01bc1ae728f15 100644 --- a/test/llvmpasses/lower-handlers.ll +++ b/test/llvmpasses/lower-handlers.ll @@ -1,4 +1,5 @@ ; RUN: opt -enable-new-pm=0 -load libjulia-codegen%shlibext -LowerExcHandlers -S %s | FileCheck %s +; RUN: opt -enable-new-pm=1 --load-pass-plugin=libjulia-codegen%shlibext -passes='function(LowerExcHandlers)' -S %s | FileCheck %s attributes #1 = { returns_twice } declare i32 @julia.except_enter() #1 From a292df3d716b4af8fd2a14d86f190c576cb31b6c Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 19 Feb 2022 20:14:23 -0500 Subject: [PATCH 2/4] [LLVM][PM] Make GCInvariantVerifier NewPM compatible --- src/aotcompile.cpp | 5 ++++ src/llvm-gc-invariant-verifier.cpp | 48 ++++++++++++++++++++---------- src/passes.h | 8 +++++ 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 33b8b030263c7..a99cc36e01ee2 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -903,6 +903,11 @@ static void registerCallbacks(PassBuilder &PB) { PM.addPass(LowerExcHandlers()); return true; } + if (Name == "GCInvariantVerifier") { + // TODO: Parse option and allow users to set `Strong` + PM.addPass(GCInvariantVerifierPass()); + return true; + } return false; }); diff --git a/src/llvm-gc-invariant-verifier.cpp b/src/llvm-gc-invariant-verifier.cpp index 4302f9021ec2c..0c6c7e27f50cf 100644 --- a/src/llvm-gc-invariant-verifier.cpp +++ b/src/llvm-gc-invariant-verifier.cpp @@ -4,6 +4,7 @@ // See the devdocs for a description of these invariants. #include "llvm-version.h" +#include "passes.h" #include #include @@ -33,11 +34,10 @@ using namespace llvm; -struct GCInvariantVerifier : public FunctionPass, public InstVisitor { - static char ID; +struct GCInvariantVerifier : public InstVisitor { bool Broken = false; bool Strong; - GCInvariantVerifier(bool Strong = false) : FunctionPass(ID), Strong(Strong) {} + GCInvariantVerifier(bool Strong = false) : Strong(Strong) {} private: void Check(bool Cond, const char *message, Value *Val) { @@ -48,12 +48,6 @@ struct GCInvariantVerifier : public FunctionPass, public InstVisitor X("GCInvariantVerifier", "GC Invariant Verification Pass", false, false); +struct GCInvariantVerifierLegacy : public FunctionPass { + static char ID; + bool Strong; + GCInvariantVerifierLegacy(bool Strong=false) : FunctionPass(ID), Strong(Strong) {} + +public: + void getAnalysisUsage(AnalysisUsage &AU) const override { + FunctionPass::getAnalysisUsage(AU); + AU.setPreservesAll(); + } + + bool runOnFunction(Function &F) override { + GCInvariantVerifier GIV(Strong); + GIV.visit(F); + if (GIV.Broken) { + abort(); + } + return false; + } +}; + +char GCInvariantVerifierLegacy::ID = 0; +static RegisterPass X("GCInvariantVerifier", "GC Invariant Verification Pass", false, false); Pass *createGCInvariantVerifierPass(bool Strong) { - return new GCInvariantVerifier(Strong); + return new GCInvariantVerifierLegacy(Strong); } extern "C" JL_DLLEXPORT void LLVMExtraAddGCInvariantVerifierPass_impl(LLVMPassManagerRef PM, LLVMBool Strong) diff --git a/src/passes.h b/src/passes.h index 4b73eb2ff7684..75c0f04d22b00 100644 --- a/src/passes.h +++ b/src/passes.h @@ -37,6 +37,14 @@ struct LowerExcHandlers : PassInfoMixin { static bool isRequired() { return true; } }; +struct GCInvariantVerifierPass : PassInfoMixin { + bool Strong; + GCInvariantVerifierPass(bool Strong = false) : Strong(Strong) {} + + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + static bool isRequired() { return true; } +}; + // Module Passes struct CPUFeatures : PassInfoMixin { PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); From 7ad142be90d26e2c4452193e35c85e8a2afc5502 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 19 Feb 2022 20:59:07 -0500 Subject: [PATCH 3/4] [LLVM][PM] Make MultiVersioning NewPM compatible --- src/aotcompile.cpp | 4 ++ src/llvm-multiversioning.cpp | 92 ++++++++++++++++++++++++------------ src/passes.h | 5 ++ 3 files changed, 70 insertions(+), 31 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index a99cc36e01ee2..77e768442ddee 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -934,6 +934,10 @@ static void registerCallbacks(PassBuilder &PB) { PM.addPass(RemoveJuliaAddrspacesPass()); return true; } + if (Name == "MultiVersioning") { + PM.addPass(MultiVersioning()); + return true; + } return false; }); diff --git a/src/llvm-multiversioning.cpp b/src/llvm-multiversioning.cpp index 57e90a9aa8056..81fbac23dbc52 100644 --- a/src/llvm-multiversioning.cpp +++ b/src/llvm-multiversioning.cpp @@ -7,6 +7,7 @@ // LLVM pass to clone function for different archs #include "llvm-version.h" +#include "passes.h" #include #include @@ -46,8 +47,6 @@ namespace { constexpr uint32_t clone_mask = JL_TARGET_CLONE_LOOP | JL_TARGET_CLONE_SIMD | JL_TARGET_CLONE_MATH | JL_TARGET_CLONE_CPU; -struct MultiVersioning; - // Treat identical mapping as missing and return `def` in that case. // We mainly need this to identify cloned function using value map after LLVM cloning // functions fills the map with identity entries. @@ -243,7 +242,7 @@ struct CloneCtx { return cast(vmap->lookup(orig_f)); } }; - CloneCtx(MultiVersioning *pass, Module &M); + CloneCtx(Module &M, function_ref GetLI, function_ref GetCG); void clone_bases(); void collect_func_infos(); void clone_all_partials(); @@ -277,12 +276,14 @@ struct CloneCtx { Type *T_void; PointerType *T_psize; MDNode *tbaa_const; - MultiVersioning *pass; std::vector specs; std::vector groups{}; std::vector fvars; std::vector gvars; Module &M; + function_ref GetLI; + function_ref GetCG; + // Map from original functiton to one based index in `fvars` std::map func_ids{}; std::vector orig_funcs{}; @@ -298,23 +299,6 @@ struct CloneCtx { bool has_cloneall{false}; }; -struct MultiVersioning: public ModulePass { - static char ID; - MultiVersioning() - : ModulePass(ID) - {} - -private: - bool runOnModule(Module &M) override; - void getAnalysisUsage(AnalysisUsage &AU) const override - { - AU.addRequired(); - AU.addRequired(); - AU.addPreserved(); - } - friend struct CloneCtx; -}; - template static inline std::vector consume_gv(Module &M, const char *name) { @@ -335,18 +319,19 @@ static inline std::vector consume_gv(Module &M, const char *name) } // Collect basic information about targets and functions. -CloneCtx::CloneCtx(MultiVersioning *pass, Module &M) +CloneCtx::CloneCtx(Module &M, function_ref GetLI, function_ref GetCG) : ctx(M.getContext()), T_size(M.getDataLayout().getIntPtrType(ctx, 0)), T_int32(Type::getInt32Ty(ctx)), T_void(Type::getVoidTy(ctx)), T_psize(PointerType::get(T_size, 0)), tbaa_const(tbaa_make_child_with_context(ctx, "jtbaa_const", nullptr, true).first), - pass(pass), specs(jl_get_llvm_clone_targets()), fvars(consume_gv(M, "jl_sysimg_fvars")), gvars(consume_gv(M, "jl_sysimg_gvars")), - M(M) + M(M), + GetLI(GetLI), + GetCG(GetCG) { groups.emplace_back(0, specs[0]); uint32_t ntargets = specs.size(); @@ -449,7 +434,7 @@ bool CloneCtx::is_vector(FunctionType *ty) const uint32_t CloneCtx::collect_func_info(Function &F) { uint32_t flag = 0; - if (!pass->getAnalysis(F).getLoopInfo().empty()) + if (!GetLI(F).empty()) flag |= JL_TARGET_CLONE_LOOP; if (is_vector(F.getFunctionType())) { flag |= JL_TARGET_CLONE_SIMD; @@ -563,7 +548,7 @@ void CloneCtx::check_partial(Group &grp, Target &tgt) auto *next_set = &sets[1]; // Reduce dispatch by expand the cloning set to functions that are directly called by // and calling cloned functions. - auto &graph = pass->getAnalysis().getCallGraph(); + auto &graph = GetCG(); while (!cur_set->empty()) { for (auto orig_f: *cur_set) { // Use the uncloned function since it's already in the call graph @@ -1079,7 +1064,7 @@ void CloneCtx::emit_metadata() } } -bool MultiVersioning::runOnModule(Module &M) +static bool runMultiVersioning(Module &M, function_ref GetLI, function_ref GetCG) { // Group targets and identify cloning bases. // Also initialize function info maps (we'll update these maps as we go) @@ -1092,7 +1077,7 @@ bool MultiVersioning::runOnModule(Module &M) if (M.getName() == "sysimage") return false; - CloneCtx clone(this, M); + CloneCtx clone(M, GetLI, GetCG); // Collect a list of original functions and clone base functions clone.clone_bases(); @@ -1130,16 +1115,61 @@ bool MultiVersioning::runOnModule(Module &M) return true; } -char MultiVersioning::ID = 0; -static RegisterPass X("JuliaMultiVersioning", "JuliaMultiVersioning Pass", +struct MultiVersioningLegacy: public ModulePass { + static char ID; + MultiVersioningLegacy() + : ModulePass(ID) + {} + +private: + bool runOnModule(Module &M) override; + void getAnalysisUsage(AnalysisUsage &AU) const override + { + AU.addRequired(); + AU.addRequired(); + AU.addPreserved(); + } +}; + +bool MultiVersioningLegacy::runOnModule(Module &M) +{ + auto GetLI = [this](Function &F) -> LoopInfo & { + return getAnalysis(F).getLoopInfo(); + }; + auto GetCG = [this]() -> CallGraph & { + return getAnalysis().getCallGraph(); + }; + return runMultiVersioning(M, GetLI, GetCG); +} + + +char MultiVersioningLegacy::ID = 0; +static RegisterPass X("JuliaMultiVersioning", "JuliaMultiVersioning Pass", false /* Only looks at CFG */, false /* Analysis Pass */); +} // anonymous namespace + +PreservedAnalyses MultiVersioning::run(Module &M, ModuleAnalysisManager &AM) +{ + auto &FAM = AM.getResult(M).getManager(); + auto GetLI = [&](Function &F) -> LoopInfo & { + return FAM.getResult(F); + }; + auto GetCG = [&]() -> CallGraph & { + return AM.getResult(M); + }; + if (runMultiVersioning(M, GetLI, GetCG)) { + auto preserved = PreservedAnalyses::allInSet(); + preserved.preserve(); + return preserved; + } + return PreservedAnalyses::all(); } Pass *createMultiVersioningPass() { - return new MultiVersioning(); + return new MultiVersioningLegacy(); } extern "C" JL_DLLEXPORT void LLVMExtraAddMultiVersioningPass_impl(LLVMPassManagerRef PM) diff --git a/src/passes.h b/src/passes.h index 75c0f04d22b00..afe58cd4740dc 100644 --- a/src/passes.h +++ b/src/passes.h @@ -64,6 +64,11 @@ struct FinalLowerGCPass : PassInfoMixin { static bool isRequired() { return true; } }; +struct MultiVersioning : PassInfoMixin { + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + static bool isRequired() { return true; } +}; + struct RemoveJuliaAddrspacesPass : PassInfoMixin { PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); static bool isRequired() { return true; } From b0a3130f55160f89692cca4c0f85b2e36014cced Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sat, 19 Feb 2022 21:07:08 -0500 Subject: [PATCH 4/4] [LLVM][PM] Make LowerPTLS NewPM compatible --- src/aotcompile.cpp | 4 ++++ src/llvm-ptls.cpp | 35 +++++++++++++++++++++++++++-------- src/passes.h | 9 ++++++++- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 77e768442ddee..fffaab1d27ea0 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -938,6 +938,10 @@ static void registerCallbacks(PassBuilder &PB) { PM.addPass(MultiVersioning()); return true; } + if (Name == "LowerPTLS") { + PM.addPass(LowerPTLSPass()); + return true; + } return false; }); diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index 297cbda53d747..5d4bcaa425565 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -7,6 +7,7 @@ #include "llvm-version.h" #include "support/dtypes.h" +#include "passes.h" #include #include @@ -34,13 +35,12 @@ typedef Instruction TerminatorInst; namespace { -struct LowerPTLS: public ModulePass { - static char ID; +struct LowerPTLS { LowerPTLS(bool imaging_mode=false) - : ModulePass(ID), - imaging_mode(imaging_mode) + : imaging_mode(imaging_mode) {} + bool runOnModule(Module &M); private: const bool imaging_mode; Module *M; @@ -62,7 +62,6 @@ struct LowerPTLS: public ModulePass { template T *add_comdat(T *G) const; GlobalVariable *create_aliased_global(Type *T, StringRef name) const; void fix_pgcstack_use(CallInst *pgcstack); - bool runOnModule(Module &M) override; }; void LowerPTLS::set_pgcstack_attrs(CallInst *pgcstack) const @@ -291,17 +290,37 @@ bool LowerPTLS::runOnModule(Module &_M) return true; } -char LowerPTLS::ID = 0; +struct LowerPTLSLegacy: public ModulePass { + static char ID; + LowerPTLSLegacy(bool imaging_mode=false) + : ModulePass(ID), + imaging_mode(imaging_mode) + {} -static RegisterPass X("LowerPTLS", "LowerPTLS Pass", + bool imaging_mode; + bool runOnModule(Module &M) override { + LowerPTLS lower(imaging_mode); + return lower.runOnModule(M); + } +}; + +char LowerPTLSLegacy::ID = 0; + +static RegisterPass X("LowerPTLS", "LowerPTLS Pass", false /* Only looks at CFG */, false /* Analysis Pass */); } // anonymous namespace +PreservedAnalyses LowerPTLSPass::run(Module &M, ModuleAnalysisManager &AM) { + LowerPTLS lower(imaging_mode); + lower.runOnModule(M); + return PreservedAnalyses::all(); +} + Pass *createLowerPTLSPass(bool imaging_mode) { - return new LowerPTLS(imaging_mode); + return new LowerPTLSLegacy(imaging_mode); } extern "C" JL_DLLEXPORT void LLVMExtraAddLowerPTLSPass_impl(LLVMPassManagerRef PM, LLVMBool imaging_mode) diff --git a/src/passes.h b/src/passes.h index afe58cd4740dc..3b229377f7cdc 100644 --- a/src/passes.h +++ b/src/passes.h @@ -75,7 +75,6 @@ struct RemoveJuliaAddrspacesPass : PassInfoMixin { }; struct RemoveAddrspacesPass : PassInfoMixin { - std::function ASRemapper; RemoveAddrspacesPass(); RemoveAddrspacesPass(std::function ASRemapper) : ASRemapper(std::move(ASRemapper)) {} @@ -84,6 +83,14 @@ struct RemoveAddrspacesPass : PassInfoMixin { static bool isRequired() { return true; } }; +struct LowerPTLSPass : PassInfoMixin { + bool imaging_mode; + LowerPTLSPass(bool imaging_mode=false) : imaging_mode(imaging_mode) {} + + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + static bool isRequired() { return true; } +}; + // Loop Passes struct JuliaLICMPass : PassInfoMixin { PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM,