From a727e1679af16f5ba72f32f274f7147719d378c3 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 17 Mar 2016 02:42:59 -0400 Subject: [PATCH] simplify interaction between new JITs and codegen this treats the new JITs (MCJIT, ORCJIT) much like the old JIT, but using Module as the atomic unit instead of Function this gets codegen closer to being separable (cachable) at the module level, with the remerging happening in the JIT after object file emission fix #15533 --- contrib/add_license_to_files.jl | 1 - src/Makefile | 4 +- src/ccall.cpp | 147 ++--- src/cgutils.cpp | 748 ++----------------------- src/codegen.cpp | 963 ++++++++++++-------------------- src/codegen_internal.h | 14 +- src/debuginfo.cpp | 4 + src/dump.c | 2 +- src/gf.c | 14 +- src/init.c | 5 + src/intrinsics.cpp | 81 +-- src/jitlayers.cpp | 637 ++++++++++++++++++++- src/julia_internal.h | 6 +- src/toplevel.c | 2 +- test/llvmcall.jl | 19 +- 15 files changed, 1144 insertions(+), 1503 deletions(-) diff --git a/contrib/add_license_to_files.jl b/contrib/add_license_to_files.jl index 64ced59076297..53bb003f5267f 100644 --- a/contrib/add_license_to_files.jl +++ b/contrib/add_license_to_files.jl @@ -43,7 +43,6 @@ const skipfiles = [ "../src/abi_x86.cpp", "../src/abi_x86_64.cpp", "../src/disasm.cpp", - "../src/jitlayers.cpp", "../src/support/END.h", "../src/support/ENTRY.amd64.h", "../src/support/ENTRY.i387.h", diff --git a/src/Makefile b/src/Makefile index ae46d015acc6e..5f7e50c34b190 100644 --- a/src/Makefile +++ b/src/Makefile @@ -117,8 +117,10 @@ $(BUILDDIR)/julia_flisp.boot: $(addprefix $(SRCDIR)/,jlfrontend.scm \ # additional dependency links $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SRCDIR)/flisp/*.h -$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp intrinsics.h cgutils.cpp ccall.cpp jitlayers.cpp abi_*.cpp) +$(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,intrinsics.cpp jitlayers.cpp intrinsics.h codegen_internal.h cgutils.cpp ccall.cpp abi_*.cpp) $(BUILDDIR)/anticodegen.o $(BUILDDIR)/anticodegen.dbg.obj: $(SRCDIR)/intrinsics.h +$(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: $(SRCDIR)/codegen_internal.h +$(BUILDDIR)/disasm.o $(BUILDDIR)/disasm.dbg.obj: $(SRCDIR)/codegen_internal.h $(BUILDDIR)/builtins.o $(BUILDDIR)/builtins.dbg.obj: $(SRCDIR)/table.c $(BUILDDIR)/gc.o $(BUILDDIR)/gc.dbg.obj: $(SRCDIR)/gc-debug.c $(BUILDDIR)/signal-handling.o $(BUILDDIR)/signal-handling.dbg.obj: $(addprefix $(SRCDIR)/,signals-*.c) diff --git a/src/ccall.cpp b/src/ccall.cpp index 0bf253f03d839..90d5635dca699 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -4,26 +4,8 @@ // --- the ccall, cglobal, and llvm intrinsics --- -// keep track of llvmcall declarations -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static std::map llvmcallDecls; -#else -static std::set llvmcallDecls; -#endif - -static std::map libMapGV; -static std::map symMapGV; - -static Value *GetStringPtr(llvm::IRBuilder <> *builder, GlobalValue *GV, const Twine &Name) -{ - Value *zero = ConstantInt::get(Type::getInt32Ty(jl_LLVMContext), 0); - Value *Args[] = { zero, zero }; -#ifdef LLVM37 - return builder->CreateInBoundsGEP(GV->getValueType(), GV, Args, Name); -#else - return builder->CreateInBoundsGEP(GV, Args, Name); -#endif -} +static StringMap libMapGV; +static StringMap symMapGV; static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, const char *f_name, jl_codectx_t *ctx) { @@ -53,52 +35,47 @@ static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, cons libsym = jl_RTLD_DEFAULT_handle; } else { + std::string name = "ccalllib_"; + name += f_lib; runtime_lib = true; libptrgv = libMapGV[f_lib]; if (libptrgv == NULL) { - libptrgv = new GlobalVariable(imaging_mode ? *shadow_module : *active_module, T_pint8, - false, GlobalVariable::PrivateLinkage, - ConstantPointerNull::get((PointerType*)T_pint8), f_lib); - libMapGV[f_lib] = libptrgv; + libptrgv = new GlobalVariable(*jl_Module, T_pint8, + false, GlobalVariable::ExternalLinkage, + NULL, name); + libMapGV[f_lib] = global_proto(libptrgv); libsym = jl_get_library(f_lib); assert(libsym != NULL); -#ifdef USE_MCJIT - jl_llvm_to_jl_value[libptrgv] = libsym; -#else - *((void**)jl_ExecutionEngine->getPointerToGlobal(libptrgv)) = libsym; -#endif + *(void**)jl_emit_and_add_to_shadow(libptrgv) = libsym; + } + else { + libptrgv = prepare_global(libptrgv); } } if (libsym == NULL) { -#ifdef USE_MCJIT - libsym = (void*)jl_llvm_to_jl_value[libptrgv]; -#else - libsym = *((void**)jl_ExecutionEngine->getPointerToGlobal(libptrgv)); -#endif + libsym = *(void**)jl_get_global(libptrgv); } - assert(libsym != NULL); GlobalVariable *llvmgv = symMapGV[f_name]; - Constant *initnul = ConstantPointerNull::get((PointerType*)T_pvoidfunc); if (llvmgv == NULL) { // MCJIT forces this to have external linkage eventually, so we would clobber // the symbol of the actual function. - std::string name = f_name; - name = "ccall_" + name; - llvmgv = new GlobalVariable(imaging_mode ? *shadow_module : *active_module, T_pvoidfunc, - false, GlobalVariable::PrivateLinkage, - initnul, name); - symMapGV[f_name] = llvmgv; -#ifdef USE_MCJIT - jl_llvm_to_jl_value[llvmgv] = jl_dlsym_e(libsym, f_name); -#else - *((void**)jl_ExecutionEngine->getPointerToGlobal(llvmgv)) = jl_dlsym_e(libsym, f_name); -#endif + std::string name = "ccall_"; + name += f_name; + llvmgv = new GlobalVariable(*jl_Module, T_pvoidfunc, + false, GlobalVariable::ExternalLinkage, + NULL, name); + symMapGV[f_name] = global_proto(llvmgv); + *(void**)jl_emit_and_add_to_shadow(llvmgv) = jl_dlsym_e(libsym, f_name); + } + else { + llvmgv = prepare_global(llvmgv); } BasicBlock *dlsym_lookup = BasicBlock::Create(jl_LLVMContext, "dlsym"), *ccall_bb = BasicBlock::Create(jl_LLVMContext, "ccall"); + Constant *initnul = ConstantPointerNull::get((PointerType*)T_pvoidfunc); builder.CreateCondBr(builder.CreateICmpNE(builder.CreateLoad(llvmgv), initnul), ccall_bb, dlsym_lookup); assert(ctx->f->getParent() != NULL); @@ -106,15 +83,15 @@ static Value *runtime_sym_lookup(PointerType *funcptype, const char *f_lib, cons builder.SetInsertPoint(dlsym_lookup); Value *libname; if (runtime_lib) { - libname = GetStringPtr(&builder, stringConst(f_lib), "f_lib"); + libname = stringConstPtr(f_lib); } else { libname = literal_static_pointer_val(f_lib, T_pint8); } #ifdef LLVM37 - Value *llvmf = builder.CreateCall(prepare_call(jldlsym_func), { libname, GetStringPtr(&builder, stringConst(f_name), "f_name"), libptrgv }); + Value *llvmf = builder.CreateCall(prepare_call(jldlsym_func), { libname, stringConstPtr(f_name), libptrgv }); #else - Value *llvmf = builder.CreateCall3(prepare_call(jldlsym_func), libname, GetStringPtr(&builder, stringConst(f_name), "f_name"), libptrgv); + Value *llvmf = builder.CreateCall3(prepare_call(jldlsym_func), libname, stringConstPtr(f_name), libptrgv); #endif builder.CreateStore(llvmf, llvmgv); builder.CreateBr(ccall_bb); @@ -598,7 +575,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c std::stringstream name; name << (ctx->f->getName().str()) << "u" << i++; ir_name = name.str(); - if (builtins_module->getFunction(ir_name) == NULL) + if (jl_Module->getFunction(ir_name) == NULL) break; } @@ -615,9 +592,7 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c std::string rstring; llvm::raw_string_ostream rtypename(rstring); rettype->print(rtypename); -#if defined(USE_MCJIT) || defined(USE_ORCJIT) std::map localDecls; -#endif if (decl != NULL) { std::stringstream declarations(jl_string_data(decl)); @@ -625,33 +600,16 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c // parse string line by line std::string declstr; while (std::getline(declarations, declstr, '\n')) { - uint64_t declhash = memhash(declstr.c_str(), declstr.length()); -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - auto it = llvmcallDecls.find(declhash); - if (it != llvmcallDecls.end()) { - prepare_call(it->second); - } - else { -#else - if (llvmcallDecls.count(declhash) == 0) { -#endif - // Find name of declaration by searching for '@' - std::string::size_type atpos = declstr.find('@') + 1; - // Find end of declaration by searching for '(' - std::string::size_type bracepos = declstr.find('(', atpos); - // Declaration name is the string between @ and ( - std::string declname = declstr.substr(atpos, bracepos - atpos); - - // Check if declaration already present in module - if(jl_Module->getNamedValue(declname) == NULL) { - ir_stream << "; Declarations\n" << declstr << "\n"; - } - -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - localDecls[declhash] = declname; -#else - llvmcallDecls.insert(declhash); -#endif + // Find name of declaration by searching for '@' + std::string::size_type atpos = declstr.find('@') + 1; + // Find end of declaration by searching for '(' + std::string::size_type bracepos = declstr.find('(', atpos); + // Declaration name is the string between @ and ( + std::string declname = declstr.substr(atpos, bracepos - atpos); + + // Check if declaration already present in module + if(jl_Module->getNamedValue(declname) == NULL) { + ir_stream << "; Declarations\n" << declstr << "\n"; } } } @@ -662,9 +620,9 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c std::string ir_string = ir_stream.str(); #ifdef LLVM36 Module *m = NULL; - bool failed = parseAssemblyInto(llvm::MemoryBufferRef(ir_string,"llvmcall"),*builtins_module,Err); + bool failed = parseAssemblyInto(llvm::MemoryBufferRef(ir_string,"llvmcall"),*jl_Module,Err); if (!failed) - m = builtins_module; + m = jl_Module; #else Module *m = ParseAssemblyString(ir_string.c_str(),jl_Module,Err,jl_LLVMContext); #endif @@ -675,12 +633,8 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c jl_error(stream.str().c_str()); } f = m->getFunction(ir_name); + f->removeFromParent(); -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - for (auto it : localDecls) { - llvmcallDecls[it.first] = cast(prepare_call(m->getNamedValue(it.second))); - } -#endif } else { assert(isPtr); @@ -692,13 +646,6 @@ static jl_cgval_t emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *c it != argtypes.end(); ++it, ++i) assert(*it == f->getFunctionType()->getParamType(i)); -#ifdef USE_MCJIT - if (f->getParent() != active_module) { - FunctionMover mover(active_module,f->getParent()); - f = mover.CloneFunction(f); - } -#endif - //f->dump(); #ifndef LLVM35 if (verifyFunction(*f,PrintMessageAction)) { @@ -1174,8 +1121,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) if (jl_is_tuple_type(fargt) && jl_is_leaf_type(fargt)) { frt = jl_tparam0(frt); JL_TRY { - Value *llvmf = prepare_call( - jl_cfunction_object((jl_function_t*)f, frt, (jl_tupletype_t*)fargt)); + Value *llvmf = prepare_call(jl_cfunction_object((jl_function_t*)f, frt, (jl_tupletype_t*)fargt)); // make sure to emit any side-effects that may have been part of the original expression emit_expr(args[4], ctx); emit_expr(args[6], ctx); @@ -1353,8 +1299,8 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) } if (needStackRestore) { - stacksave = CallInst::Create(prepare_call(Intrinsic::getDeclaration(builtins_module, - Intrinsic::stacksave))); + stacksave = CallInst::Create(Intrinsic::getDeclaration(jl_Module, + Intrinsic::stacksave)); if (savespot) { #ifdef LLVM38 instList.insertAfter(savespot->getIterator(), (Instruction*)stacksave); @@ -1381,10 +1327,7 @@ static jl_cgval_t emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx) result = ret; if (needStackRestore) { assert(stacksave != NULL); - builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, - Intrinsic::stackrestore)), - stacksave); + builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::stackrestore), stacksave); } if (0) { // Enable this to turn on SSPREQ (-fstack-protector) on the function containing this ccall ctx->f->addFnAttr(Attribute::StackProtectReq); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 8d54ffc2a074c..0845b94d0e8dc 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -4,482 +4,89 @@ // utility procedures used in code generation -template // for GlobalObject's -static T *addComdat(T *G) -{ -#if defined(_OS_WINDOWS_) - if (imaging_mode && !G->isDeclaration()) { -#ifdef LLVM35 - // Add comdat information to make MSVC link.exe happy - Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); - jl_Comdat->setSelectionKind(Comdat::NoDuplicates); - G->setComdat(jl_Comdat); - // add __declspec(dllexport) to everything marked for export - if (G->getLinkage() == GlobalValue::ExternalLinkage) - G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); -#endif - } -#endif - return G; -} - static Instruction *tbaa_decorate(MDNode *md, Instruction *load_or_store) { load_or_store->setMetadata( llvm::LLVMContext::MD_tbaa, md ); return load_or_store; } -// Fixing up references to other modules for MCJIT -std::set pending_globals; static GlobalVariable *prepare_global(GlobalVariable *G) { - pending_globals.insert(G); - return G; + Module *M = jl_builderModule; + GlobalValue *local = M->getNamedValue(G->getName()); + if (!local) { + local = global_proto(G, M); + } + return cast(local); } static llvm::Value *prepare_call(llvm::Value *Callee) { - llvm::Function *F = dyn_cast(Callee); - if (!F) - return Callee; - pending_globals.insert(F); - return Callee; -} - -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static GlobalValue *realize_pending_global(Module *M, GlobalValue *G, std::map &FixedGlobals) -{ - if (M == G->getParent() && M != builtins_module) { // can happen during bootstrap - //std::cout << "Skipping " << std::string(G->getName()) << " due to parentage" << std::endl; - return nullptr; - } - // If we come across a function that is still being constructed, - // this use needs to remain pending - if (!M || M == builtins_module) { - pending_globals.insert(G); - //std::cout << "Skipping " << std::string(G->getName()) << " due to construction" << std::endl; - return nullptr; - } - if (!FixedGlobals.count(M)) { - if (GlobalVariable *GV = dyn_cast(G)) { - GlobalVariable *NewGV = M->getGlobalVariable(GV->getName()); - if (!NewGV) { - NewGV = new GlobalVariable(*M, GV->getType()->getElementType(), - GV->isConstant(), GlobalVariable::ExternalLinkage, - NULL, GV->getName(), NULL, GV->getThreadLocalMode(), - GV->getType()->getAddressSpace()); - NewGV->setUnnamedAddr(GV->hasUnnamedAddr()); - // Move over initializer - if (GV->hasInitializer()) { - NewGV->setInitializer(GV->getInitializer()); - GV->setInitializer(nullptr); - } - } - FixedGlobals[M] = NewGV; + if (Function *F = dyn_cast(Callee)) { + Module *M = jl_builderModule; + GlobalValue *local = M->getNamedValue(Callee->getName()); + if (!local) { + local = function_proto(F, M); } - else { - Function *F = cast(G); - //std::cout << "Realizing " << std::string(F->getName()) << std::endl; - //if (!F->getParent()) { - //std::cout << "Skipping" << std::endl; - // return nullptr; - //} - Function *NewF = nullptr; - if (!F->isDeclaration() && F->getParent() == builtins_module) { - // It's a definition. Actually move the function and create a - // declaration in the original module - NewF = F; - F->removeFromParent(); - M->getFunctionList().push_back(F); - Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName(), - active_module); - } - else { - assert(F); - NewF = M->getFunction(F->getName()); - if (!NewF) { - NewF = Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName(), - M); - } - } - FixedGlobals[M] = NewF; - } - } - return FixedGlobals[M]; -} - -struct ExprChain { - ConstantExpr *Expr; - unsigned OpNo; - struct ExprChain *Next; -}; - -static void handleUse(Use &Use1, llvm::GlobalValue *G, - std::map &FixedGlobals, - struct ExprChain *Chain, struct ExprChain *ChainEnd) -{ - Instruction *User = dyn_cast(Use1.getUser()); - GlobalValue *GVUser = dyn_cast(Use1.getUser()); - if (!User && !GVUser) { - ConstantExpr *Expr = cast(Use1.getUser()); - Value::use_iterator UI2 = Expr->use_begin(), E2 = Expr->use_end(); - for (; UI2 != E2;) { - Use &Use2 = *UI2; - ++UI2; - struct ExprChain NextChain; - NextChain.Expr = Expr; - NextChain.OpNo = Use1.getOperandNo(); - NextChain.Next = nullptr; - if (ChainEnd) - ChainEnd->Next = &NextChain; - handleUse(Use2,G,FixedGlobals,Chain ? Chain : &NextChain,&NextChain); - } - return; - } - llvm::Module *M = nullptr; - if (User) { - Function *UsedInHere = User->getParent()->getParent(); - assert(UsedInHere); - M = UsedInHere->getParent(); - } else { - assert(GVUser); - M = GVUser->getParent(); - } - llvm::Constant *Replacement = realize_pending_global(M,G,FixedGlobals); - if (!Replacement) - return; - while (Chain) { - Replacement = Chain->Expr->getWithOperandReplaced(Chain->OpNo,Replacement); - Chain->Expr = cast(Replacement); - Chain = Chain->Next; - } - Use1.set(Replacement); -} - -// RAUW, but only for those users which live in a module, and create a module -// specific copy -static void realize_pending_globals() -{ - std::set local_pending_globals; - std::swap(local_pending_globals,pending_globals); - std::map FixedGlobals; - for (auto *G : local_pending_globals) { - Value::use_iterator UI = G->use_begin(), E = G->use_end(); - for (; UI != E;) - handleUse(*(UI++),G,FixedGlobals,nullptr,nullptr); - FixedGlobals.clear(); + return local; } + return Callee; } -static void realize_cycle(jl_cyclectx_t *cyclectx) +// --- string constants --- +static StringMap stringConstants; +static Value *stringConstPtr(const std::string &txt) { - // These need to be resolved together - for (auto *F : cyclectx->functions) { - F->removeFromParent(); - active_module->getFunctionList().push_back(F); - } - for (auto *CU : cyclectx->CUs) { - NamedMDNode *NMD = active_module->getOrInsertNamedMetadata("llvm.dbg.cu"); - NMD->addOperand(CU); - } -} -#endif - -template -#ifdef LLVM35 -static inline void add_named_global(GlobalObject *gv, T *_addr, bool dllimport = true) + StringRef ctxt(txt.c_str(), strlen(txt.c_str()) + 1); +#ifdef LLVM36 + StringMap::iterator pooledval = + stringConstants.insert(std::pair(ctxt, NULL)).first; #else -static inline void add_named_global(GlobalValue *gv, T *_addr, bool dllimport = true) -#endif -{ - // cast through integer to avoid c++ pedantic warning about casting between - // data and code pointers - void *addr = (void*)(uintptr_t)_addr; -#ifdef LLVM34 - StringRef name = gv->getName(); -#ifdef _OS_WINDOWS_ - std::string imp_name; -#endif + StringMap::MapEntryTy *pooledval = + &stringConstants.GetOrCreateValue(ctxt, (GlobalVariable*)NULL); #endif - -#ifdef _OS_WINDOWS_ - // setting JL_DLLEXPORT correctly only matters when building a binary - if (dllimport && imaging_mode) { - assert(gv->getLinkage() == GlobalValue::ExternalLinkage); -#ifdef LLVM35 - // add the __declspec(dllimport) attribute - gv->setDLLStorageClass(GlobalValue::DLLImportStorageClass); - // this will cause llvm to rename it, so we do the same - imp_name = Twine("__imp_", name).str(); - name = StringRef(imp_name); + StringRef pooledtxt = pooledval->getKey(); + if (imaging_mode) { + if (pooledval->second == NULL) { + static int strno = 0; + std::stringstream ssno; + ssno << "_j_str" << strno++; + GlobalVariable *gv = new GlobalVariable(*shadow_output, + ArrayType::get(T_int8, pooledtxt.size()), + true, + GlobalVariable::PrivateLinkage, + ConstantDataArray::get(getGlobalContext(), + ArrayRef( + (const unsigned char*)pooledtxt.data(), + pooledtxt.size())), + ssno.str()); + gv->setUnnamedAddr(true); + pooledval->second = gv; +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + jl_ExecutionEngine->addGlobalMapping(gv->getName(), (uintptr_t)pooledtxt.data()); #else - gv->setLinkage(GlobalValue::DLLImportLinkage); -#endif -#if defined(_P64) || defined(LLVM35) - // __imp_ variables are indirection pointers, so use malloc to simulate that - void **imp_addr = (void**)malloc(sizeof(void**)); - *imp_addr = addr; - addr = (void*)imp_addr; + jl_ExecutionEngine->addGlobalMapping(gv, (void*)pooledtxt.data()); #endif - } -#endif // _OS_WINDOWS_ - -#ifdef USE_ORCJIT - addComdat(gv); - jl_ExecutionEngine->addGlobalMapping(name, addr); -#elif defined(USE_MCJIT) - addComdat(gv); - sys::DynamicLibrary::AddSymbol(name, addr); -#else // USE_MCJIT - jl_ExecutionEngine->addGlobalMapping(gv, addr); -#endif // USE_MCJIT -} - -// --- string constants --- -static std::map stringConstants; + } -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static GlobalVariable *global_proto(GlobalVariable *G) -{ - GlobalVariable *proto = new GlobalVariable(G->getType()->getElementType(), - G->isConstant(), GlobalVariable::ExternalLinkage, - NULL, G->getName(), G->getThreadLocalMode()); - return proto; -} + GlobalVariable *v = prepare_global(pooledval->second); + Value *zero = ConstantInt::get(Type::getInt32Ty(jl_LLVMContext), 0); + Value *Args[] = { zero, zero }; +#ifdef LLVM37 + return builder.CreateInBoundsGEP(v->getValueType(), v, Args); #else -static GlobalVariable *global_proto(GlobalVariable *G) -{ - return G; -} + return builder.CreateInBoundsGEP(v, Args); #endif - -static GlobalVariable *stringConst(const std::string &txt) -{ - GlobalVariable *gv = stringConstants[txt]; - static int strno = 0; - // in inference, we can not share string constants between - // modules as there might be multiple compiles on the stack - // with calls in between them. - if (gv == NULL) { - std::stringstream ssno; - std::string vname; - ssno << strno; - vname += "_j_str"; - vname += ssno.str(); - gv = new GlobalVariable(*active_module, - ArrayType::get(T_int8, txt.length()+1), - true, - imaging_mode ? GlobalVariable::PrivateLinkage : GlobalVariable::ExternalLinkage, - ConstantDataArray::get(getGlobalContext(), - ArrayRef( - (const unsigned char*)txt.c_str(), - txt.length()+1)), - vname); - gv->setUnnamedAddr(true); - gv = imaging_mode ? gv : prepare_global(global_proto(gv)); - stringConstants[txt] = gv; - strno++; } else { - prepare_global(gv); + Value *v = ConstantExpr::getIntToPtr( + ConstantInt::get(T_size, (uintptr_t)pooledtxt.data()), + T_pint8); + return v; } - return gv; - -} - -// --- Shadow module handling --- - -typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing -static std::map jl_value_to_llvm; -JL_DLLEXPORT std::map jl_llvm_to_jl_value; - -// In imaging mode, cache a fast mapping of Function * to code address -// because this is queried in the hot path -static std::map emitted_function_symtab; - -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static Function *function_proto(Function *F) -{ - Function *NewF = Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName()); - NewF->setAttributes(AttributeSet()); - - // FunctionType does not include any attributes. Copy them over manually - // as codegen may make decisions based on the presence of certain attributes - NewF->copyAttributesFrom(F); - -#ifdef LLVM37 - // Declarations are not allowed to have personality routines, but - // copyAttributesFrom sets them anyway, so clear them again manually - NewF->setPersonalityFn(nullptr); -#endif - - return NewF; } -class FunctionMover : public ValueMaterializer -{ -public: - FunctionMover(llvm::Module *dest,llvm::Module *src) : - ValueMaterializer(), VMap(), destModule(dest), srcModule(src), - LazyFunctions(0) - { - } - ValueToValueMapTy VMap; - llvm::Module *destModule; - llvm::Module *srcModule; - std::vector LazyFunctions; - - Function *CloneFunctionProto(Function *F) - { - assert(!F->isDeclaration()); - Function *NewF = Function::Create(F->getFunctionType(), - Function::ExternalLinkage, - F->getName(), - destModule); - LazyFunctions.push_back(F); - VMap[F] = NewF; - return NewF; - } - - void CloneFunctionBody(Function *F) - { - Function *NewF = (Function*)(Value*)VMap[F]; - assert(NewF != NULL); - - Function::arg_iterator DestI = NewF->arg_begin(); - for (Function::const_arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E; ++I) { - DestI->setName(I->getName()); // Copy the name over... - VMap[&*I] = &*(DestI++); // Add mapping to VMap - } - - #ifdef LLVM36 - // Clone debug info - Not yet public API - // llvm::CloneDebugInfoMetadata(NewF,F,VMap); - #endif - - SmallVector Returns; - llvm::CloneFunctionInto(NewF,F,VMap,true,Returns,"",NULL,NULL,this); - } - - Function *CloneFunction(Function *F) - { - Function *NewF = (llvm::Function*)MapValue(F,VMap,RF_None,NULL,this); - ResolveLazyFunctions(); - return NewF; - } - - void ResolveLazyFunctions() - { - while (!LazyFunctions.empty()) { - Function *F = LazyFunctions.back(); - LazyFunctions.pop_back(); - - CloneFunctionBody(F); - } - } - - Value *InjectFunctionProto(Function *F) - { - Function *NewF = destModule->getFunction(F->getName()); - if (!NewF) { - NewF = function_proto(F); - destModule->getFunctionList().push_back(NewF); - } - return NewF; - } - -#ifdef LLVM38 - virtual Value *materializeDeclFor(Value *V) -#else - virtual Value *materializeValueFor (Value *V) -#endif - { - Function *F = dyn_cast(V); - if (F) { - if (F->isIntrinsic()) { - return destModule->getOrInsertFunction(F->getName(),F->getFunctionType()); - } - if (F->isDeclaration() || F->getParent() != destModule) { - if (F->getName().empty()) - return CloneFunctionProto(F); - Function *shadow = srcModule->getFunction(F->getName()); - if (shadow != NULL && !shadow->isDeclaration()) { - // Not truly external - // Check whether we already emitted it once - if (emitted_function_symtab.find(shadow) != emitted_function_symtab.end()) - return InjectFunctionProto(F); - - Function *oldF = destModule->getFunction(F->getName()); - if (oldF) - return oldF; - -#ifndef USE_ORCJIT - // Also check if this function is pending in any other module - if (jl_ExecutionEngine->FindFunctionNamed(F->getName().data())) - return InjectFunctionProto(F); -#endif - - return CloneFunctionProto(shadow); - } - else if (!F->isDeclaration()) { - return CloneFunctionProto(F); - } - } - // Still a declaration and still in a different module - if (F->isDeclaration() && F->getParent() != destModule) { - // Create forward declaration in current module - return InjectFunctionProto(F); - } - } - else if (isa(V)) { - GlobalVariable *GV = cast(V); - assert(GV != NULL); - GlobalVariable *oldGV = destModule->getGlobalVariable(GV->getName()); - if (oldGV != NULL) - return oldGV; - GlobalVariable *newGV = new GlobalVariable(*destModule, - GV->getType()->getElementType(), - GV->isConstant(), - GlobalVariable::ExternalLinkage, - NULL, - GV->getName()); - newGV->copyAttributesFrom(GV); - if (GV->isDeclaration()) - return newGV; - if (!GV->getName().empty()) { - uint64_t addr = jl_ExecutionEngine->getGlobalValueAddress(GV->getName()); - if (addr != 0) { - newGV->setExternallyInitialized(true); - return newGV; - } - } - std::map::iterator it; - it = jl_llvm_to_jl_value.find(GV); - if (it != jl_llvm_to_jl_value.end()) { - newGV->setInitializer(Constant::getIntegerValue(GV->getType()->getElementType(),APInt(sizeof(void*)*8,(intptr_t)it->second))); - newGV->setConstant(true); - } - else if (GV->hasInitializer()) { - Value *C = MapValue(GV->getInitializer(),VMap,RF_None,NULL,this); - newGV->setInitializer(cast(C)); - } - return newGV; - } - return NULL; - }; -}; -#else -static Function *function_proto(Function *F) -{ - return F; -} -#endif +// --- Debug info --- #ifdef LLVM37 static DIType *julia_type_to_di(jl_value_t *jt, DIBuilder *dbuilder, bool isboxed = false) @@ -569,218 +176,6 @@ static Value *literal_static_pointer_val(const void *p, Type *t) #endif } -static std::vector jl_sysimg_gvars; -static std::vector jl_sysimg_fvars; - -extern "C" int32_t jl_get_llvm_gv(jl_value_t *p) -{ - // map a jl_value_t memory location to a GlobalVariable - std::map::iterator it; - it = jl_value_to_llvm.find(p); - if (it == jl_value_to_llvm.end()) - return 0; - return it->second.index; -} - -#ifdef HAVE_CPUID -extern "C" { - extern void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType); -} -#endif - -static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap, - const char *sysimg_data, size_t sysimg_len) -{ - ArrayType *gvars_type = ArrayType::get(T_psize, jl_sysimg_gvars.size()); - addComdat(new GlobalVariable(*mod, - gvars_type, - true, - GlobalVariable::ExternalLinkage, - MapValue(ConstantArray::get(gvars_type, ArrayRef(jl_sysimg_gvars)), VMap), - "jl_sysimg_gvars")); - ArrayType *fvars_type = ArrayType::get(T_pvoidfunc, jl_sysimg_fvars.size()); - addComdat(new GlobalVariable(*mod, - fvars_type, - true, - GlobalVariable::ExternalLinkage, - MapValue(ConstantArray::get(fvars_type, ArrayRef(jl_sysimg_fvars)), VMap), - "jl_sysimg_fvars")); - addComdat(new GlobalVariable(*mod, - T_size, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_size,globalUnique+1), - "jl_globalUnique")); -#ifdef JULIA_ENABLE_THREADING - addComdat(new GlobalVariable(*mod, - T_size, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_size, jltls_states_func_idx), - "jl_ptls_states_getter_idx")); -#endif - - Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_options.cpu_target); - addComdat(new GlobalVariable(*mod, - feature_string->getType(), - true, - GlobalVariable::ExternalLinkage, - feature_string, - "jl_sysimg_cpu_target")); - -#ifdef HAVE_CPUID - // For native also store the cpuid - if (strcmp(jl_options.cpu_target,"native") == 0) { - uint32_t info[4]; - - jl_cpuid((int32_t*)info, 1); - addComdat(new GlobalVariable(*mod, - T_int64, - true, - GlobalVariable::ExternalLinkage, - ConstantInt::get(T_int64,((uint64_t)info[2])|(((uint64_t)info[3])<<32)), - "jl_sysimg_cpu_cpuid")); - } -#endif - - if (sysimg_data) { - Constant *data = ConstantDataArray::get(jl_LLVMContext, ArrayRef((const unsigned char*)sysimg_data, sysimg_len)); - addComdat(new GlobalVariable(*mod, data->getType(), true, - GlobalVariable::ExternalLinkage, - data, "jl_system_image_data")); - Constant *len = ConstantInt::get(T_size, sysimg_len); - addComdat(new GlobalVariable(*mod, len->getType(), true, - GlobalVariable::ExternalLinkage, - len, "jl_system_image_size")); - } -} - -static void jl_dump_shadow(char *fname, int jit_model, const char *sysimg_data, size_t sysimg_len, - bool dump_as_bc) -{ -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - realize_pending_globals(); -#endif -#ifdef JL_DEBUG_BUILD - verifyModule(*shadow_module); -#endif - -#ifdef LLVM36 - std::error_code err; - StringRef fname_ref = StringRef(fname); - raw_fd_ostream OS(fname_ref, err, sys::fs::F_None); -#elif defined(LLVM35) - std::string err; - raw_fd_ostream OS(fname, err, sys::fs::F_None); -#else - std::string err; - raw_fd_ostream OS(fname, err); -#endif -#ifdef LLVM37 // 3.7 simplified formatted output; just use the raw stream alone - raw_fd_ostream& FOS(OS); -#else - formatted_raw_ostream FOS(OS); -#endif - - // We don't want to use MCJIT's target machine because - // it uses the large code model and we may potentially - // want less optimizations there. - Triple TheTriple = Triple(jl_TargetMachine->getTargetTriple()); -#if defined(_OS_WINDOWS_) && defined(FORCE_ELF) -#ifdef LLVM35 - TheTriple.setObjectFormat(Triple::COFF); -#else - TheTriple.setEnvironment(Triple::UnknownEnvironment); -#endif -#elif defined(_OS_DARWIN_) && defined(FORCE_ELF) -#ifdef LLVM35 - TheTriple.setObjectFormat(Triple::MachO); -#else - TheTriple.setEnvironment(Triple::MachO); -#endif -#endif -#ifdef LLVM35 - std::unique_ptr -#else - OwningPtr -#endif - TM(jl_TargetMachine->getTarget().createTargetMachine( - TheTriple.getTriple(), - jl_TargetMachine->getTargetCPU(), - jl_TargetMachine->getTargetFeatureString(), - jl_TargetMachine->Options, -#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) - Reloc::PIC_, -#else - jit_model ? Reloc::PIC_ : Reloc::Default, -#endif - jit_model ? CodeModel::JITDefault : CodeModel::Default, - CodeGenOpt::Aggressive // -O3 - )); - -#ifdef LLVM38 - legacy::PassManager PM; -#else - PassManager PM; -#endif -#ifndef LLVM37 - PM.add(new TargetLibraryInfo(Triple(TM->getTargetTriple()))); -#else - PM.add(new TargetLibraryInfoWrapperPass(Triple(TM->getTargetTriple()))); -#endif -#ifdef LLVM37 -// No DataLayout pass needed anymore. -#elif defined(LLVM36) - PM.add(new DataLayoutPass()); -#elif defined(LLVM35) - PM.add(new DataLayoutPass(*jl_ExecutionEngine->getDataLayout())); -#else - PM.add(new DataLayout(*jl_ExecutionEngine->getDataLayout())); -#endif - - addOptimizationPasses(&PM); - if (!dump_as_bc) { - if (TM->addPassesToEmitFile(PM, FOS, TargetMachine::CGFT_ObjectFile, false)) { - jl_error("Could not generate obj file for this target"); - } - } - - // now copy the module, since PM.run may modify it - ValueToValueMapTy VMap; -#ifdef LLVM38 - Module *clone = CloneModule(shadow_module, VMap).release(); -#else - Module *clone = CloneModule(shadow_module, VMap); -#endif -#ifdef LLVM37 - // Reset the target triple to make sure it matches the new target machine - clone->setTargetTriple(TM->getTargetTriple().str()); -#ifdef LLVM38 - clone->setDataLayout(TM->createDataLayout()); -#else - clone->setDataLayout(TM->getDataLayout()->getStringRepresentation()); -#endif -#endif - - // add metadata information - jl_gen_llvm_globaldata(clone, VMap, sysimg_data, sysimg_len); - - // do the actual work - finalize_gc_frame(clone); - PM.run(*clone); - if (dump_as_bc) - WriteBitcodeToFile(clone, FOS); - delete clone; -} - -static int32_t jl_assign_functionID(Function *functionObject, int specsig) -{ - // give the function an index in the constant lookup table - if (!imaging_mode) - return 0; - jl_sysimg_fvars.push_back(ConstantExpr::getBitCast(functionObject, T_pvoidfunc)); - return jl_sysimg_fvars.size(); -} static Value *julia_gv(const char *cname, void *addr) { @@ -794,24 +189,11 @@ static Value *julia_gv(const char *cname, void *addr) std::stringstream gvname; gvname << cname << globalUnique++; // no existing GlobalVariable, create one and store it - GlobalVariable *gv = new GlobalVariable(imaging_mode ? *shadow_module : *builtins_module, T_pjlvalue, - false, imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - ConstantPointerNull::get((PointerType*)T_pjlvalue), gvname.str()); + GlobalVariable *gv = new GlobalVariable(*jl_builderModule, T_pjlvalue, + false, GlobalVariable::ExternalLinkage, + NULL, gvname.str()); addComdat(gv); - - // make the pointer valid for this session -#ifdef USE_MCJIT - jl_llvm_to_jl_value[gv] = addr; -#else - void **p = (void**)jl_ExecutionEngine->getPointerToGlobal(gv); - *p = addr; -#endif - // make the pointer valid for future sessions - jl_sysimg_gvars.push_back(ConstantExpr::getBitCast(gv, T_psize)); - jl_value_llvm gv_struct; - gv_struct.gv = prepare_global(gv); - gv_struct.index = jl_sysimg_gvars.size(); - jl_value_to_llvm[addr] = gv_struct; + *(void**)jl_emit_and_add_to_shadow(gv, addr) = addr; return builder.CreateLoad(gv); } @@ -1183,11 +565,7 @@ static Value *emit_datatype_isbitstype(Value *dt) static void just_emit_error(const std::string &txt, jl_codectx_t *ctx) { - Value *zeros[2] = { ConstantInt::get(T_int32, 0), - ConstantInt::get(T_int32, 0) }; - builder.CreateCall(prepare_call(jlerror_func), - builder.CreateGEP(stringConst(txt), - ArrayRef(zeros))); + builder.CreateCall(prepare_call(jlerror_func), stringConstPtr(txt)); } static void emit_error(const std::string &txt, jl_codectx_t *ctx) @@ -1252,12 +630,8 @@ static void null_pointer_check(Value *v, jl_codectx_t *ctx) static void emit_type_error(const jl_cgval_t &x, jl_value_t *type, const std::string &msg, jl_codectx_t *ctx) { - Value *zeros[2] = { ConstantInt::get(T_int32, 0), - ConstantInt::get(T_int32, 0) }; - Value *fname_val = builder.CreateGEP(stringConst(ctx->funcName), - ArrayRef(zeros)); - Value *msg_val = builder.CreateGEP(stringConst(msg), - ArrayRef(zeros)); + Value *fname_val = stringConstPtr(ctx->funcName); + Value *msg_val = stringConstPtr(msg); #ifdef LLVM37 builder.CreateCall(prepare_call(jltypeerror_func), { fname_val, msg_val, diff --git a/src/codegen.cpp b/src/codegen.cpp index d9338d669c192..687bd69f4a500 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -12,6 +12,9 @@ // also MCJIT debugging support on windows needs ELF (currently) #define FORCE_ELF #endif +#if defined(_CPU_X86_) +#define JL_NEED_FLOATTEMP_VAR 1 +#endif #ifndef __STDC_LIMIT_MACROS #define __STDC_LIMIT_MACROS @@ -198,31 +201,17 @@ namespace llvm { // for image reloading static bool imaging_mode = false; -#include "jitlayers.cpp" - -#ifdef USE_ORCJIT -JuliaOJIT *jl_ExecutionEngine; -#else -ExecutionEngine *jl_ExecutionEngine; -#endif - -#ifdef USE_MCJIT -static Module *shadow_module; -static Module *builtins_module; -static Module *active_module; -#define jl_Module (builder.GetInsertBlock()->getParent()->getParent()) -#else -static Module *jl_Module; -#define shadow_module jl_Module -#define active_module jl_Module -#define builtins_module jl_Module -#endif +static Module *shadow_output; +#define jl_Module ctx->f->getParent() +#define jl_builderModule builder.GetInsertBlock()->getParent()->getParent() static MDBuilder *mbuilder; static std::map argNumberStrings; +#ifndef USE_ORCJIT #ifdef LLVM38 -static legacy::FunctionPassManager *FPM; +static legacy::PassManager *PM; #else -static FunctionPassManager *FPM; +static PassManager *PM; +#endif #endif #ifdef LLVM37 @@ -322,12 +311,6 @@ static GlobalVariable *jltrue_var; static GlobalVariable *jlfalse_var; static GlobalVariable *jlemptysvec_var; static GlobalVariable *jlemptytuple_var; -#if defined(_CPU_X86_) -#define JL_NEED_FLOATTEMP_VAR 1 -#endif -#ifdef JL_NEED_FLOATTEMP_VAR -static GlobalVariable *jlfloattemp_var; -#endif static GlobalVariable *jldiverr_var; static GlobalVariable *jlundeferr_var; static GlobalVariable *jldomerr_var; @@ -438,6 +421,11 @@ static std::vector three_pvalue_llvmt; static std::map builtin_func_map; // --- code generation --- +extern "C" { + int globalUnique = 0; +} + +#include "jitlayers.cpp" // metadata tracking for a llvm Value* during codegen struct jl_cgval_t { @@ -556,13 +544,6 @@ typedef struct { jl_value_t *ty; } jl_arrayvar_t; -// Keeps tracks of all functions and compile units created during this cycle -// to be able to atomically add them to a module. -typedef struct { - std::vector functions; - std::vector CUs; -} jl_cyclectx_t; - // information about the context of a piece of code: its enclosing // function and module, and visible local variables and labels. typedef struct { @@ -595,8 +576,6 @@ typedef struct { llvm::DIBuilder *dbuilder; bool debug_enabled; std::vector to_inline; - - jl_cyclectx_t *cyclectx; } jl_codectx_t; typedef struct { @@ -624,8 +603,7 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, static jl_cgval_t emit_checked_var(Value *bp, jl_sym_t *name, jl_codectx_t *ctx, bool isvol=false); static Value *emit_condition(jl_value_t *cond, const std::string &msg, jl_codectx_t *ctx); static void allocate_gc_frame(BasicBlock *b0, jl_codectx_t *ctx); -static void finalize_gc_frame(Function *F); -static void finalize_gc_frame(Module *m); +static void jl_finalize_module(std::unique_ptr m); static GlobalVariable *prepare_global(GlobalVariable *G); // --- convenience functions for tagging llvm values with julia types --- @@ -717,10 +695,6 @@ static inline jl_cgval_t mark_julia_const(jl_value_t *jv) // --- utilities --- -extern "C" { - int globalUnique = 0; -} - static void emit_write_barrier(jl_codectx_t*, Value*, Value*); #include "cgutils.cpp" @@ -822,17 +796,18 @@ void jl_dump_compiles(void *s) // --- entry point --- //static int n_emit=0; -static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations, - jl_llvm_functions_t *definitions, jl_cyclectx_t *cyclectx); -static void jl_finalize_module(Module *m); +static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations); void jl_add_linfo_in_flight(StringRef name, jl_lambda_info_t *linfo, const DataLayout &DL); -//static int n_compile=0; -static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) +// this is the implementation component of jl_compile_linfo +// which compiles li and adds the result to the jitlayers +static void to_function(jl_lambda_info_t *li) { + // setup global state JL_LOCK(codegen); JL_SIGATOMIC_BEGIN(); assert(!li->inInference); + li->inCompile = 1; BasicBlock *old = nested_compile ? builder.GetInsertBlock() : NULL; DebugLoc olddl = builder.getCurrentDebugLocation(); bool last_n_c = nested_compile; @@ -840,33 +815,18 @@ static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) last_time = jl_hrtime(); nested_compile = true; jl_gc_inhibit_finalizers(nested_compile); + std::unique_ptr m; Function *f = NULL, *specf = NULL; + // actually do the work of emitting the function JL_TRY { - jl_llvm_functions_t definitions; - #if defined(USE_MCJIT) || defined(USE_ORCJIT) - jl_cyclectx_t *newcyclectx = cyclectx; - if (!newcyclectx) - newcyclectx = new jl_cyclectx_t; - emit_function(li, &li->functionObjects, &definitions, newcyclectx); - // If we're the root of the cycle, realize all functions - if (!cyclectx) { - realize_cycle(newcyclectx); - delete newcyclectx; - } - #else - emit_function(li, &li->functionObjects, &definitions, NULL); - #endif - f = (llvm::Function*)definitions.functionObject; - specf = (llvm::Function*)definitions.specFunctionObject; - li->functionID = jl_assign_functionID(f, 0); - if (specf) - li->specFunctionID = jl_assign_functionID(specf, 1); - if (f->getFunctionType() != jl_func_sig) - // mark the pointer as jl_fptr_sparam_t calling convention - li->jlcall_api = 1; + m = emit_function(li, &li->functionObjects); + f = (Function*)li->functionObjects.functionObject; + specf = (Function*)li->functionObjects.specFunctionObject; //n_emit++; } JL_CATCH { + // something failed! this is very bad, since other WIP may be pointing to this function + // but there's not much we can do now. try to clear much of the WIP anyways. li->functionObjects.functionObject = NULL; li->functionObjects.specFunctionObject = NULL; li->functionObjects.cFunctionList = NULL; @@ -875,45 +835,41 @@ static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) builder.SetInsertPoint(old); builder.SetCurrentDebugLocation(olddl); } + li->inCompile = 0; JL_SIGATOMIC_END(); JL_UNLOCK(codegen); jl_rethrow_with_add("error compiling %s", jl_symbol_name(li->name)); } - assert(f != NULL); - const DataLayout &DL = -#ifdef LLVM35 - f->getParent()->getDataLayout(); -#else - *jl_data_layout; -#endif // record that this function name came from this linfo, // so we can build a reverse mapping for debug-info. if (li->name != anonymous_sym) { + const DataLayout &DL = +#ifdef LLVM35 + m->getDataLayout(); +#else + *jl_data_layout; +#endif // but don't remember anonymous symbols because // they may not be rooted in the gc for the life of the program, // and the runtime doesn't notify us when the code becomes unreachable :( jl_add_linfo_in_flight((specf ? specf : f)->getName(), li, DL); } -#if !defined(USE_MCJIT) && !defined(USE_ORCJIT) - finalize_gc_frame(f); - if (specf) finalize_gc_frame(specf); -#ifdef JL_DEBUG_BUILD - if (verifyFunction(*f, PrintMessageAction) || - (specf && verifyFunction(*specf, PrintMessageAction))) { - f->dump(); - if (specf) specf->dump(); - gc_debug_critical_error(); - abort(); - } -#endif - FPM->run(*f); - if (specf) FPM->run(*specf); -#endif + // success. add the result to the execution engine now + jl_finalize_module(std::move(m)); + li->functionID = jl_assign_functionID(f, 0); + if (specf) + li->specFunctionID = jl_assign_functionID(specf, 1); + if (f->getFunctionType() != jl_func_sig) + // mark the pointer as jl_fptr_sparam_t calling convention + li->jlcall_api = 1; + + // done compiling: restore global state if (old != NULL) { builder.SetInsertPoint(old); builder.SetCurrentDebugLocation(olddl); } + li->inCompile = 0; nested_compile = last_n_c; jl_gc_inhibit_finalizers(nested_compile); JL_UNLOCK(codegen); @@ -925,7 +881,6 @@ static Function *to_function(jl_lambda_info_t *li, jl_cyclectx_t *cyclectx) jl_printf(dump_compiles_stream, "\"\n"); last_time = this_time; } - return specf ? specf : f; } #ifndef LLVM37 @@ -976,185 +931,78 @@ static void jl_setup_module(Module *m) #endif } -#if defined(USE_MCJIT) || defined(USE_ORCJIT) -static void jl_finalize_module(Module *m) +// this takes ownership of a module after code emission is complete +// and will add it to the execution engine when required (by jl_finalize_function) +static void finalize_gc_frame(Module *m); +static void jl_finalize_module(std::unique_ptr uniquem) { -#if defined(_CPU_X86_64_) && defined(_OS_WINDOWS_) && defined(USE_MCJIT) - ArrayType *atype = ArrayType::get(T_uint32,3); // want 4-byte alignment of 12-bytes of data - (new GlobalVariable(*m, atype, - false, GlobalVariable::InternalLinkage, - ConstantAggregateZero::get(atype), "__UnwindData"))->setSection(".text"); - (new GlobalVariable(*m, atype, - false, GlobalVariable::InternalLinkage, - ConstantAggregateZero::get(atype), "__catchjmp"))->setSection(".text"); -#endif + Module *m = uniquem.release(); // unique_ptr won't be able track what we do with this (the invariant is recovered by jl_finalize_function) finalize_gc_frame(m); - assert(jl_ExecutionEngine); -#if defined(LLVM36) && !defined(USE_ORCJIT) - jl_ExecutionEngine->addModule(std::unique_ptr(m)); -#else - jl_ExecutionEngine->addModule(m); +#if !defined(USE_ORCJIT) +#ifdef LLVM33 +#ifdef JL_DEBUG_BUILD + if (verifyModule(*m, PrintMessageAction)) { + m->dump(); + gc_debug_critical_error(); + abort(); + } #endif -} - -#if !defined(USE_ORCJIT) && defined(JL_DEBUG_BUILD) -static void writeRecoveryFile(llvm::Module *mod) -{ - std::error_code err; - mod->dump(); - std::cout << "Julia emitted a broken LLVM module (about to be __jl_dump.bc)." - << "Please file a bug report.\n" - << "If the module writing below fails," - << "please include the textual representation printed above this error."; - StringRef fname_ref = StringRef("__jl_dump.bc"); - raw_fd_ostream OS(fname_ref, err, sys::fs::F_None); - WriteBitcodeToFile(mod,OS); - OS.flush(); - gc_debug_critical_error(); - abort(); -} #endif + PM->run(*m); +#endif +#ifdef USE_MCJIT + // record the function names that are part of this Module + // so it can be added to the JIT when needed + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *F = &*I; + if (!F->isDeclaration()) + module_for_fname[F->getName()] = m; + } +#endif + jl_add_to_shadow(m); +} -static uint64_t getAddressForOrCompileFunction(llvm::Function *llvmf) +// this ensures that llvmf has been emitted to the execution engine, +// returning the function pointer to it +static uint64_t getAddressForFunction(llvm::Function *llvmf) { - #ifdef JL_DEBUG_BUILD +#ifdef JL_DEBUG_BUILD llvm::raw_fd_ostream out(1,false); - #endif - Function *ActiveF = active_module->getFunction(llvmf->getName()); - // Must have been in a prior module. Safe to ask the execution engine - // to emit it. - if (!ActiveF || ActiveF->isDeclaration()) - return jl_ExecutionEngine->getFunctionAddress(llvmf->getName()); - if (!imaging_mode) { - realize_pending_globals(); - #ifndef USE_ORCJIT - #ifdef JL_DEBUG_BUILD -#ifdef LLVM38 - Module *backup = llvm::CloneModule(active_module).release(); +#endif +#ifdef USE_MCJIT + jl_finalize_function(llvmf, NULL); + return jl_ExecutionEngine->getFunctionAddress(llvmf->getName()); #else - Module *backup = llvm::CloneModule(active_module); + return (uint64_t)jl_ExecutionEngine->getPointerToFunction( + cast(shadow_output->getNamedValue(llvmf->getName()))); #endif - if(verifyModule(*active_module)) - writeRecoveryFile(backup); - #endif - for (auto &F : active_module->functions()) { - if (F.isDeclaration()) - continue; - #ifdef JL_DEBUG_BUILD - if(verifyFunction(F)) - writeRecoveryFile(backup); - #endif - finalize_gc_frame(&F); - FPM->run(F); - #ifdef JL_DEBUG_BUILD - if(verifyFunction(F)) - writeRecoveryFile(backup); - #endif - } - #ifdef JL_DEBUG_BUILD - if(verifyModule(*active_module)) - writeRecoveryFile(backup); - delete backup; - #endif - #endif - jl_finalize_module(active_module); - } - uint64_t addr = jl_ExecutionEngine->getFunctionAddress(llvmf->getName()); - assert(addr != 0); - if (!imaging_mode) { - active_module = new Module("julia", jl_LLVMContext); - jl_setup_module(active_module); - } - return addr; } -#endif +// this assumes that jl_compile_linfo has already been called +// and forces compilation of the lambda info extern "C" void jl_generate_fptr(jl_lambda_info_t *li) { JL_LOCK(codegen); // objective: assign li->fptr assert(li->functionObjects.functionObject); + assert(!li->inCompile); if (li->fptr == NULL) { JL_SIGATOMIC_BEGIN(); - #ifdef USE_MCJIT - if (imaging_mode) { - // see if it has been emitted already (as part of compiling something else) - li->fptr = (jl_fptr_t)jl_ExecutionEngine->getFunctionAddress(((Function*)li->functionObjects.functionObject)->getName()); - if (li->fptr == NULL) { - // Copy the function out of the shadow module - Module *m = new Module("julia", jl_LLVMContext); - jl_setup_module(m); - FunctionMover mover(m, shadow_module); - mover.CloneFunction((Function*)li->functionObjects.functionObject); - if (li->functionObjects.specFunctionObject != NULL) - mover.CloneFunction((Function*)li->functionObjects.specFunctionObject); - if (li->functionObjects.cFunctionList != NULL) { - size_t i; - cFunctionList_t *list = (cFunctionList_t*)li->functionObjects.cFunctionList; - for (i = 0; i < list->len; i++) { - list->data()[i].f = mover.CloneFunction(list->data()[i].f); - } - } - jl_finalize_module(m); - li->fptr = (jl_fptr_t)jl_ExecutionEngine->getFunctionAddress(((Function*)li->functionObjects.functionObject)->getName()); - } - } - else { - li->fptr = (jl_fptr_t)getAddressForOrCompileFunction((Function*)li->functionObjects.functionObject); - } - #else - li->fptr = (jl_fptr_t)jl_ExecutionEngine->getPointerToFunction((Function*)li->functionObjects.functionObject); - #endif - + li->fptr = (jl_fptr_t)getAddressForFunction((Function*)li->functionObjects.functionObject); assert(li->fptr != NULL); -#ifndef KEEP_BODIES - if (!imaging_mode) - ((Function*)li->functionObjects.functionObject)->deleteBody(); -#endif - - if (li->functionObjects.cFunctionList != NULL) { - size_t i; - cFunctionList_t *list = (cFunctionList_t*)li->functionObjects.cFunctionList; - for (i = 0; i < list->len; i++) { -#ifdef USE_MCJIT - (void)getAddressForOrCompileFunction(list->data()[i].f); -#else - (void)jl_ExecutionEngine->getPointerToFunction(list->data()[i].f); -#endif -#ifndef KEEP_BODIES - if (!imaging_mode) { - list->data()[i].f->deleteBody(); - } -#endif - } - } - - if (li->functionObjects.specFunctionObject != NULL) { -#ifdef USE_MCJIT - if (imaging_mode) - (void)jl_ExecutionEngine->getFunctionAddress(((Function*)li->functionObjects.specFunctionObject)->getName()); - else - (void)getAddressForOrCompileFunction((Function*)li->functionObjects.specFunctionObject); -#else - (void)jl_ExecutionEngine->getPointerToFunction((Function*)li->functionObjects.specFunctionObject); -#endif -#ifndef KEEP_BODIES - if (!imaging_mode) - ((Function*)li->functionObjects.specFunctionObject)->deleteBody(); -#endif - } JL_SIGATOMIC_END(); } JL_UNLOCK(codegen); } -extern "C" void jl_compile_linfo(jl_lambda_info_t *li, void *cyclectx) +// this generates llvm code for the lambda info +// (and adds it to the shadow module), but doesn't yet compile +// or generate object code for it +extern "C" void jl_compile_linfo(jl_lambda_info_t *li) { if (li->functionObjects.functionObject == NULL) { // objective: assign li->functionObject - li->inCompile = 1; - (void)to_function(li, (jl_cyclectx_t *)cyclectx); - li->inCompile = 0; + to_function(li); } } @@ -1212,7 +1060,7 @@ static Function *jl_cfunction_object(jl_function_t *f, jl_value_t *rt, jl_tuplet } } - jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)sigt, NULL); + jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)sigt); if (li != NULL) { for(i=1; i < nargs+1; i++) { jl_value_t *speci = jl_nth_slot_type(li->specTypes, i); @@ -1256,29 +1104,13 @@ void *jl_function_ptr(jl_function_t *f, jl_value_t *rt, jl_value_t *argt) } assert(jl_is_tuple_type(argt)); Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt); - assert(llvmf); JL_GC_POP(); - -#ifdef USE_MCJIT - if (uint64_t addr = getAddressForOrCompileFunction(llvmf)) - return (void*)(intptr_t)addr; - if (llvmf->getParent() == shadow_module) { - // Copy the function out of the shadow module - Module *m = new Module("julia", jl_LLVMContext); - jl_setup_module(m); - FunctionMover mover(m, shadow_module); - (void)mover.CloneFunction(llvmf); - jl_finalize_module(m); - } -#endif -#ifdef USE_MCJIT - return (void*)getAddressForOrCompileFunction(llvmf); -#else - return jl_ExecutionEngine->getPointerToFunction(llvmf); -#endif + return (void*)getAddressForFunction(llvmf); } +// convenience function for debugging from gdb (pre-OrcJIT) +// it generally helps to have define KEEP_BODIES if you plan on using this extern "C" JL_DLLEXPORT void *jl_function_ptr_by_llvm_name(char *name) { #ifdef __has_feature @@ -1289,38 +1121,33 @@ void *jl_function_ptr_by_llvm_name(char *name) { return (void*)(intptr_t)jl_ExecutionEngine->FindFunctionNamed(name); } -// export a C-callable entry point for a function, with a given name +// export a C-callable entry point for a function (dllexport'ed dlsym), with a given name extern "C" JL_DLLEXPORT void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) { assert(jl_is_tuple_type(argt)); Function *llvmf = jl_cfunction_object(f, rt, (jl_tupletype_t*)argt); if (llvmf) { - Function *active_llvmf = active_module->getFunction(llvmf->getName()); - // In imaging mode, and in most cases in JIT mode (where the wrapper is specifically - // compiled for this function), we can simply use a global alias. - if (active_llvmf) { - #ifndef LLVM35 - new GlobalAlias(llvmf->getType(), GlobalValue::ExternalLinkage, name, llvmf, llvmf->getParent()); - #elif defined(LLVM37) && !defined(LLVM38) - GlobalAlias::create(cast(llvmf->getType()), - GlobalValue::ExternalLinkage, name, active_llvmf, active_module); - #else + // force eager emission of the function (llvm 3.3 gets confused otherwise and tries to do recursive compilation) + uint64_t Addr = getAddressForFunction(llvmf); (void)Addr; + // emit the function pointer and set up an alias in the execution engine +#if defined(USE_ORCJIT) || defined(USE_MCJIT) + jl_ExecutionEngine->addGlobalMapping(name, Addr); + if (imaging_mode) + // in the old JIT, the shadow_module aliases the engine_module, + // otherwise, adding it as a global mapping is needed unconditionally +#endif + { + llvmf = cast(shadow_output->getNamedValue(llvmf->getName())); + // in imaging_mode, also need to add the alias to the shadow_module +#if defined(LLVM38) GlobalAlias::create(llvmf->getType()->getElementType(), llvmf->getType()->getAddressSpace(), - GlobalValue::ExternalLinkage, name, active_llvmf, active_module); - #endif - } else { - // Otherwise we use a global mapping - assert(!imaging_mode); -#if defined(USE_ORCJIT) - jl_ExecutionEngine->addGlobalMapping(llvmf->getName(), - (void*)getAddressForOrCompileFunction(llvmf)); -#elif defined(USE_MCJIT) - jl_ExecutionEngine->addGlobalMapping(llvmf->getName(), - getAddressForOrCompileFunction(llvmf)); + GlobalValue::ExternalLinkage, name, llvmf, shadow_output); +#elif defined(LLVM37) + GlobalAlias::create(cast(llvmf->getType()), + GlobalValue::ExternalLinkage, name, llvmf, shadow_output); #else - jl_ExecutionEngine->addGlobalMapping(llvmf, - jl_ExecutionEngine->getPointerToFunction(llvmf)); + new GlobalAlias(llvmf->getType(), GlobalValue::ExternalLinkage, name, llvmf, shadow_output); #endif } } @@ -1328,13 +1155,16 @@ void jl_extern_c(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) // --- native code info, and dump function to IR and ASM --- // Get pointer to llvm::Function instance, compiling if necessary +// for use in reflection from Julia. +// this is paired with jl_dump_function_ir and jl_dump_function_asm in particular ways: +// misuse will leak memory or cause read-after-free extern "C" JL_DLLEXPORT void *jl_get_llvmf(jl_function_t *f, jl_tupletype_t *tt, bool getwrapper, bool getdeclarations) { jl_lambda_info_t *linfo = NULL; JL_GC_PUSH2(&linfo, &tt); if (tt != NULL) { - linfo = jl_get_specialization1(tt, NULL); + linfo = jl_get_specialization1(tt); if (linfo == NULL) { linfo = jl_method_lookup_by_type(jl_gf_mtable(f), tt, 0, 0); if (linfo == NULL) { @@ -1353,62 +1183,41 @@ void *jl_get_llvmf(jl_function_t *f, jl_tupletype_t *tt, bool getwrapper, bool g } if (!getdeclarations) { - Function *llvmDecl = NULL; - if (!getwrapper && linfo->functionObjects.specFunctionObject != NULL) - llvmDecl = (Function*)linfo->functionObjects.specFunctionObject; - else - llvmDecl = (Function*)linfo->functionObjects.functionObject; -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - Function *llvmf = llvmDecl ? active_module->getFunction(llvmDecl->getName()) : NULL; - // Note that in either case, we need to run the FPM manually, - // since this is now usually done as part of object emission -#else - Function *llvmf = llvmDecl && !llvmDecl->isDeclaration() ? llvmDecl : NULL; -#endif - if (!llvmf) { - Function *other; - jl_llvm_functions_t declarations; - emit_function(linfo, NULL, &declarations, NULL); - if (getwrapper || !declarations.specFunctionObject) { - llvmf = (llvm::Function*)declarations.functionObject; - other = (llvm::Function*)declarations.specFunctionObject; - } - else { - llvmf = (llvm::Function*)declarations.specFunctionObject; - other = (llvm::Function*)declarations.functionObject; - } - if (other) - other->eraseFromParent(); -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - realize_pending_globals(); - finalize_gc_frame(llvmf); - FPM->run(*llvmf); -#endif - llvmf->removeFromParent(); - if (llvmDecl) - llvmf->setName(llvmDecl->getName()); + // emit this function into a new module + Function *f, *specf; + jl_llvm_functions_t declarations; + std::unique_ptr m = emit_function(linfo, &declarations); + f = (llvm::Function*)declarations.functionObject; + specf = (llvm::Function*)declarations.specFunctionObject; + // swap declarations for definitions and destroy declarations + if (specf) { + Function *temp = cast(m->getNamedValue(specf->getName())); + delete specf; + specf = temp; + } + if (f) { + Function *temp = cast(m->getNamedValue(f->getName())); + delete f; + f = temp; + } + Function *specf_decl = (Function*)linfo->functionObjects.specFunctionObject; + if (specf_decl) { + specf->setName(specf_decl->getName()); + } + Function *f_decl = (Function*)linfo->functionObjects.functionObject; + if (f_decl) { + f->setName(f_decl->getName()); + } + finalize_gc_frame(m.release()); // the return object `llvmf` will be the owning pointer + JL_GC_POP(); + if (getwrapper || !specf) { + return f; } else { - ValueToValueMapTy VMap; - Function *oldllvmf = llvmf; - llvmf = CloneFunction(llvmf, VMap, false); -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - active_module->getFunctionList().push_back(llvmf); - realize_pending_globals(); - finalize_gc_frame(llvmf); - FPM->run(*llvmf); - llvmf->removeFromParent(); - llvmf->setName(oldllvmf->getName()); -#else - (void)oldllvmf; -#endif + return specf; } - JL_GC_POP(); - return llvmf; - } - if (linfo->functionObjects.functionObject == NULL) { - jl_compile_linfo(linfo, NULL); } + jl_compile_linfo(linfo); Function *llvmf; if (!getwrapper && linfo->functionObjects.specFunctionObject != NULL) { llvmf = (Function*)linfo->functionObjects.specFunctionObject; @@ -1420,16 +1229,8 @@ void *jl_get_llvmf(jl_function_t *f, jl_tupletype_t *tt, bool getwrapper, bool g return llvmf; } -extern "C" JL_DLLEXPORT -uint64_t jl_get_llvm_fptr(llvm::Function *llvmf) -{ -#if defined(USE_ORCJIT) || defined(USE_MCJIT) - return getAddressForOrCompileFunction(llvmf); -#else - return (uint64_t)jl_ExecutionEngine->getPointerToFunction(llvmf); -#endif -} - +// print an llvm IR acquired from jl_get_llvmf +// warning: this takes ownership of, and destroys, f->getParent() extern "C" JL_DLLEXPORT const jl_value_t *jl_dump_function_ir(void *f, bool strip_ir_metadata, bool dump_module) { @@ -1437,57 +1238,50 @@ const jl_value_t *jl_dump_function_ir(void *f, bool strip_ir_metadata, bool dump llvm::raw_string_ostream stream(code); Function *llvmf = dyn_cast((Function*)f); - if (!llvmf) - jl_error("jl_dump_function_ir: Expected Function*"); + if (!llvmf || (!llvmf->isDeclaration() && !llvmf->getParent())) + jl_error("jl_dump_function_ir: Expected Function* in a temporary Module"); - if (llvmf->isDeclaration()) { - // print the function declaration plain + if (!llvmf->getParent()) { + // print the function declaration as-is llvmf->print(stream); } else { - if (llvmf->getParent()) - jl_error("jl_dump_function_ir requires a parentless clone"); - // Put the function in a module - Module *m = new Module(llvmf->getName(), jl_LLVMContext); - jl_setup_module(m); - m->getFunctionList().push_back(llvmf); - Function *f2 = llvmf; + Module *m = llvmf->getParent(); if (strip_ir_metadata) { - // strip metadata from the copy - Function::BasicBlockListType::iterator f2_bb = f2->getBasicBlockList().begin(); - // iterate over all basic blocks in the function - for (; f2_bb != f2->getBasicBlockList().end(); ++f2_bb) { - BasicBlock::InstListType::iterator f2_il = (*f2_bb).getInstList().begin(); - // iterate over instructions in basic block - for (; f2_il != (*f2_bb).getInstList().end(); ) { - Instruction *inst = &*f2_il++; - // remove dbg.declare and dbg.value calls - if (isa(inst) || isa(inst)) { - inst->eraseFromParent(); - continue; - } + // strip metadata from all instructions in the module + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *f2 = &*I; + Function::BasicBlockListType::iterator f2_bb = f2->getBasicBlockList().begin(); + // iterate over all basic blocks in the function + for (; f2_bb != f2->getBasicBlockList().end(); ++f2_bb) { + BasicBlock::InstListType::iterator f2_il = (*f2_bb).getInstList().begin(); + // iterate over instructions in basic block + for (; f2_il != (*f2_bb).getInstList().end(); ) { + Instruction *inst = &*f2_il++; + // remove dbg.declare and dbg.value calls + if (isa(inst) || isa(inst)) { + inst->eraseFromParent(); + continue; + } - SmallVector, 4> MDForInst; - inst->getAllMetadata(MDForInst); - SmallVector, 4>::iterator md_iter = MDForInst.begin(); + SmallVector, 4> MDForInst; + inst->getAllMetadata(MDForInst); + SmallVector, 4>::iterator md_iter = MDForInst.begin(); - // iterate over all metadata kinds and set to NULL to remove - for (; md_iter != MDForInst.end(); ++md_iter) { - inst->setMetadata((*md_iter).first, NULL); + // iterate over all metadata kinds and set to NULL to remove + for (; md_iter != MDForInst.end(); ++md_iter) { + inst->setMetadata((*md_iter).first, NULL); + } } } } } if (dump_module) { -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - realize_pending_globals(); -#endif m->print(stream, NULL); } - else - f2->print(stream); - f2->eraseFromParent(); - m->dropAllReferences(); + else { + llvmf->print(stream); + } delete m; } @@ -1562,6 +1356,7 @@ static uint64_t compute_obj_symsize(const object::ObjectFile *obj, uint64_t offs return 0; } +// print a native disassembly for f (an LLVM function) extern "C" JL_DLLEXPORT const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) { @@ -1578,18 +1373,15 @@ const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) // Dump assembly code uint64_t symsize = 0; int64_t slide = 0, section_slide = 0; + uint64_t fptr = getAddressForFunction(llvmf); #ifdef USE_MCJIT - uint64_t fptr = getAddressForOrCompileFunction(llvmf); -#ifdef USE_ORCJIT // Look in the system image as well if (fptr == 0) - fptr = jl_ExecutionEngine->findSymbol( - jl_ExecutionEngine->mangle(llvmf->getName()), true).getAddress(); -#endif + fptr = (uintptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable( + jl_ExecutionEngine->getMangledName(llvmf)); llvm::DIContext *context = NULL; llvm::DIContext *&objcontext = context; #else - uint64_t fptr = (uintptr_t)jl_ExecutionEngine->getPointerToFunction(llvmf); std::vector context; llvm::DIContext *objcontext = NULL; #endif @@ -1642,29 +1434,91 @@ const jl_value_t *jl_dump_function_asm(void *f, int raw_mc) // Code coverage -typedef StringMap< std::vector > logdata_t; +const int logdata_blocksize = 32; // target getting nearby lines in the same general cache area and reducing calls to malloc by chunking +typedef uint64_t logdata_block[logdata_blocksize]; +typedef StringMap< std::vector > logdata_t; static logdata_t coverageData; static void coverageVisitLine(StringRef filename, int line) { - if (filename == "" || filename == "none" || filename == "no file") + assert(!imaging_mode); + if (filename == "" || filename == "none" || filename == "no file" || line < 0) return; - logdata_t::iterator it = coverageData.find(filename); - if (it == coverageData.end()) { - coverageData[filename] = std::vector(0); - } - std::vector &vec = coverageData[filename]; - if (vec.size() <= (size_t)line) - vec.resize(line+1, NULL); - if (vec[line] == NULL) { - vec[line] = addComdat(new GlobalVariable(*jl_Module, T_int64, false, - GlobalVariable::InternalLinkage, - ConstantInt::get(T_int64,0), "lcnt")); - } - GlobalVariable *v = prepare_global(vec[line]); - builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true), - ConstantInt::get(T_int64,1)), - v, true); + std::vector &vec = coverageData[filename]; + int block = line / logdata_blocksize; + line = line % logdata_blocksize; + if (vec.size() <= block) + vec.resize(block + 1); + if (vec[block] == NULL) { + vec[block] = (logdata_block*)calloc(1, sizeof(logdata_block)); + } + logdata_block &data = *vec[block]; + if (data[line] == 0) + data[line] = 1; + Value *v = ConstantExpr::getIntToPtr( + ConstantInt::get(T_size, (uintptr_t)&data[line]), + T_pint64); + builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true, "lcnt"), + ConstantInt::get(T_int64, 1)), + v, true); // not atomic, so this might be an underestimate, + // but it's faster this way +} + + +// Memory allocation log (malloc_log) + +static logdata_t mallocData; + +static void mallocVisitLine(StringRef filename, int line) +{ + assert(!imaging_mode); + if (filename == "" || filename == "none" || filename == "no file" || line < 0) { + jl_gc_sync_total_bytes(); + return; + } + std::vector &vec = mallocData[filename]; + int block = line / logdata_blocksize; + line = line % logdata_blocksize; + if (vec.size() <= block) + vec.resize(block + 1); + if (vec[block] == NULL) { + vec[block] = (logdata_block*)calloc(1, sizeof(logdata_block)); + } + logdata_block &data = *vec[block]; + if (data[line] == 0) + data[line] = 1; + Value *v = ConstantExpr::getIntToPtr( + ConstantInt::get(T_size, (uintptr_t)&data[line]), + T_pint64); + builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true, "bytecnt"), + builder.CreateCall(prepare_call(diff_gc_total_bytes_func) +#ifdef LLVM37 + , {} +#endif + )), + v, true); // not atomic, so this might be an underestimate, + // but it's faster this way +} + +// Resets the malloc counts. Needed to avoid including memory usage +// from JITting. +extern "C" JL_DLLEXPORT void jl_clear_malloc_data(void) +{ + logdata_t::iterator it = mallocData.begin(); + for (; it != mallocData.end(); it++) { + std::vector &bytes = (*it).second; + std::vector::iterator itb; + for (itb = bytes.begin(); itb != bytes.end(); itb++) { + if (*itb) { + logdata_block &data = **itb; + for (size_t i = 0; i < logdata_blocksize; i++) { + if (data[i] > 0) + data[i] = 1; + } + } + } + } + jl_gc_sync_total_bytes(); } extern "C" int isabspath(const char *in); @@ -1676,8 +1530,8 @@ static void write_log_data(logdata_t &logData, const char *extension) logdata_t::iterator it = logData.begin(); for (; it != logData.end(); it++) { std::string filename = it->first(); - std::vector &values = it->second; - if (values.size() > 1) { + std::vector &values = it->second; + if (!values.empty()) { if (!isabspath(filename.c_str())) filename = base + filename; std::ifstream inf(filename.c_str()); @@ -1686,6 +1540,7 @@ static void write_log_data(logdata_t &logData, const char *extension) std::ofstream outf(outfile.c_str(), std::ofstream::trunc | std::ofstream::out); char line[1024]; int l = 1; + int block = 0; while (!inf.eof()) { inf.getline(line, sizeof(line)); if (inf.fail() && !inf.bad()) { @@ -1693,26 +1548,22 @@ static void write_log_data(logdata_t &logData, const char *extension) inf.clear(); inf.ignore(std::numeric_limits::max(), '\n'); } - int value = -1; - if ((size_t)l < values.size()) { - GlobalVariable *gv = values[l]; - if (gv) { -#ifdef USE_MCJIT - int *p = (int*)(intptr_t)jl_ExecutionEngine->getGlobalValueAddress(gv->getName()); -#else - int *p = (int*)jl_ExecutionEngine->getPointerToGlobal(gv); -#endif - value = *p; - } + logdata_block *data = NULL; + if (block < values.size()) { + data = values[block]; + } + uint64_t value = data ? (*data)[l] : 0; + if (++l >= logdata_blocksize) { + l = 0; + block++; } outf.width(9); - if (value == -1) - outf<<'-'; + if (value == 0) + outf << '-'; else - outf<(0); - } - std::vector &vec = mallocData[filename]; - if (vec.size() <= (size_t)line) - vec.resize(line+1, NULL); - if (vec[line] == NULL) { - vec[line] = addComdat(new GlobalVariable(*jl_Module, T_int64, false, - GlobalVariable::InternalLinkage, - ConstantInt::get(T_int64,0), "bytecnt")); - } - GlobalVariable *v = prepare_global(vec[line]); - builder.CreateStore(builder.CreateAdd(builder.CreateLoad(v, true), - builder.CreateCall(prepare_call(diff_gc_total_bytes_func) -#ifdef LLVM37 - , {} -#endif - )), - v, true); -} - -// Resets the malloc counts. Needed to avoid including memory usage -// from JITting. -extern "C" JL_DLLEXPORT void jl_clear_malloc_data(void) -{ - logdata_t::iterator it = mallocData.begin(); - for (; it != mallocData.end(); it++) { - std::vector &bytes = (*it).second; - std::vector::iterator itb; - for (itb = bytes.begin(); itb != bytes.end(); itb++) { - if (*itb) { -#ifdef USE_MCJIT - int *p = (int*)(intptr_t)jl_ExecutionEngine->getGlobalValueAddress((*itb)->getName()); -#else - int *p = (int*)jl_ExecutionEngine->getPointerToGlobal(*itb); -#endif - *p = 0; - } - } - } - jl_gc_sync_total_bytes(); -} - extern "C" void jl_write_malloc_log(void) { write_log_data(mallocData, ".mem"); } + // --- code gen for intrinsic functions --- #include "intrinsics.cpp" @@ -2940,11 +2738,7 @@ static jl_cgval_t emit_call_function_object(jl_lambda_info_t *li, const jl_cgval jl_value_t *jlretty = li->rettype; bool retboxed; (void)julia_type_to_llvm(jlretty, &retboxed); - Function *cf = (Function*)li->functionObjects.specFunctionObject; - if (!cf->getParent() || (cf->getParent() == builtins_module && - builtins_module != active_module)) { // Call cycle - prepare_call(cf); - } + Function *cf = cast(prepare_call((Function*)li->functionObjects.specFunctionObject)); FunctionType *cft = cf->getFunctionType(); size_t nfargs = cft->getNumParams(); Value **argvals = (Value**) alloca(nfargs*sizeof(Value*)); @@ -3053,7 +2847,7 @@ static jl_cgval_t emit_call(jl_value_t **args, size_t arglen, jl_codectx_t *ctx, jl_sprint(args[0]), jl_sprint((jl_value_t*)aty)); }*/ - jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)aty, ctx->cyclectx); + jl_lambda_info_t *li = jl_get_specialization1((jl_tupletype_t*)aty); if (li != NULL) { assert(li->functionObjects.functionObject != NULL); theFptr = (Value*)li->functionObjects.functionObject; @@ -3137,11 +2931,12 @@ static Value *global_binding_pointer(jl_module_t *m, jl_sym_t *s, b = jl_get_binding(m, s); if (b == NULL) { // var not found. switch to delayed lookup. + std::stringstream name; + name << "delayedvar" << globalUnique++; Constant *initnul = ConstantPointerNull::get((PointerType*)T_pjlvalue); - GlobalVariable *bindinggv = - prepare_global(new GlobalVariable(imaging_mode ? *shadow_module : *builtins_module, T_pjlvalue, - false, GlobalVariable::PrivateLinkage, - initnul, "delayedvar")); + GlobalVariable *bindinggv = new GlobalVariable(*ctx->f->getParent(), T_pjlvalue, + false, GlobalVariable::PrivateLinkage, + initnul, name.str()); Value *cachedval = builder.CreateLoad(bindinggv); BasicBlock *have_val = BasicBlock::Create(jl_LLVMContext, "found"), *not_found = BasicBlock::Create(jl_LLVMContext, "notfound"); @@ -3814,11 +3609,11 @@ static void finalize_gc_frame(Function *F) static void finalize_gc_frame(Module *m) { -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - for (auto &F : m->functions()) { - if (F.isDeclaration()) + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *F = &*I; + if (F->isDeclaration()) continue; - finalize_gc_frame(&F); + finalize_gc_frame(F); } #ifndef JULIA_ENABLE_THREADING m->getFunction("jl_get_ptls_states")->eraseFromParent(); @@ -3827,7 +3622,6 @@ static void finalize_gc_frame(Module *m) m->getFunction("julia.gc_root_kill")->eraseFromParent(); m->getFunction("julia.gc_store")->eraseFromParent(); m->getFunction("julia.jlcall_frame_decl")->eraseFromParent(); -#endif } // here argt does not include the leading function type argument @@ -3872,7 +3666,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v if (fargt.size() + sret != fargt_sig.size()) jl_error("va_arg syntax not allowed for cfunction argument list"); - jl_compile_linfo(lam, NULL); + jl_compile_linfo(lam); if (!lam->functionObjects.functionObject) { jl_errorf("error compiling %s while creating cfunction", jl_symbol_name(lam->name)); @@ -3889,9 +3683,11 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v nested_compile = true; jl_gc_inhibit_finalizers(nested_compile); // no allocations expected between the top of this function (when last scanned lam->cFunctionList) and here, which might have triggered running julia code + Module *M = new Module(jl_symbol_name(lam->name), jl_LLVMContext); + jl_setup_module(M); Function *cw = Function::Create(FunctionType::get(sret ? T_void : prt, fargt_sig, false), - imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + GlobalVariable::ExternalLinkage, + funcName.str(), M); addComdat(cw); cw->setAttributes(attrs); #ifdef LLVM37 @@ -3918,7 +3714,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v jl_throw(jl_memory_exception); list2->len = len; list2->data()[len-1].isref = isref; - list2->data()[len-1].f = imaging_mode ? cw : cw_proto; + list2->data()[len-1].f = cw_proto; lam->functionObjects.cFunctionList = list2; // See whether this function is specsig or jlcall @@ -4094,13 +3890,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v else builder.CreateRet(r); -#if !defined(USE_MCJIT) && !defined(USE_ORCJIT) - finalize_gc_frame(cw); - FPM->run(*cw); -#endif - - cw->removeFromParent(); - active_module->getFunctionList().push_back(cw); + jl_finalize_module(std::unique_ptr(M)); // Restore the previous compile context if (old != NULL) { @@ -4115,7 +3905,7 @@ static Function *gen_cfun_wrapper(jl_lambda_info_t *lam, jl_function_t *ff, jl_v } // generate a julia-callable function that calls f (AKA lam) -static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Function *f, bool sret) +static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Function *f, bool sret, Module *M) { std::stringstream funcName; const std::string &fname = f->getName().str(); @@ -4125,8 +3915,8 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct else funcName << fname; - Function *w = Function::Create(jl_func_sig, imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + Function *w = Function::Create(jl_func_sig, GlobalVariable::ExternalLinkage, + funcName.str(), M); addComdat(w); #ifdef LLVM37 w->addFnAttr("no-frame-pointer-elim", "true"); @@ -4195,10 +3985,9 @@ static Function *gen_jlcall_wrapper(jl_lambda_info_t *lam, jl_expr_t *ast, Funct } // Compile to LLVM IR, using a specialized signature if applicable. -static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations, - jl_llvm_functions_t *definitions, jl_cyclectx_t *cyclectx) +static std::unique_ptr emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declarations) { - assert(definitions && "Capturing definitions is always required"); + assert(declarations && "Capturing declarations is always required"); // step 1. unpack AST and allocate codegen context for this function jl_expr_t *ast = (jl_expr_t*)lam->ast; @@ -4224,7 +4013,6 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio ctx.vaStack = false; ctx.inbounds.push_back(false); ctx.boundsCheck.push_back(false); - ctx.cyclectx = cyclectx; ctx.spvals_ptr = NULL; // step 2. process var-info lists to see what vars need boxing @@ -4327,6 +4115,8 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio funcName << "_" << globalUnique++; ctx.sret = false; + Module *M = new Module(jl_symbol_name(lam->name), jl_LLVMContext); + jl_setup_module(M); if (specsig) { // assumes !va and !needsparams std::vector fsig(0); Type *rt; @@ -4352,37 +4142,28 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio fsig.push_back(ty); } f = Function::Create(FunctionType::get(rt, fsig, false), - imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + GlobalVariable::ExternalLinkage, + funcName.str(), M); if (ctx.sret) f->addAttribute(1, Attribute::StructRet); addComdat(f); #ifdef LLVM37 f->addFnAttr("no-frame-pointer-elim", "true"); #endif - definitions->specFunctionObject = f; - if (declarations) - declarations->specFunctionObject = function_proto(f); - fwrap = gen_jlcall_wrapper(lam, ast, - (llvm::Function*)lam->functionObjects.specFunctionObject, ctx.sret); - definitions->functionObject = fwrap; - if (declarations) - declarations->functionObject = function_proto(fwrap); + fwrap = gen_jlcall_wrapper(lam, ast, f, ctx.sret, M); + declarations->functionObject = function_proto(fwrap); + declarations->specFunctionObject = function_proto(f); } else { f = Function::Create(needsparams ? jl_func_sig_sparams : jl_func_sig, - imaging_mode ? GlobalVariable::InternalLinkage : GlobalVariable::ExternalLinkage, - funcName.str(), builtins_module); + GlobalVariable::ExternalLinkage, + funcName.str(), M); addComdat(f); #ifdef LLVM37 f->addFnAttr("no-frame-pointer-elim", "true"); #endif - definitions->functionObject = f; - definitions->specFunctionObject = NULL; - if (declarations) { - declarations->functionObject = function_proto(f); - declarations->specFunctionObject = NULL; - } + declarations->functionObject = function_proto(f); + declarations->specFunctionObject = NULL; } if (jlrettype == (jl_value_t*)jl_bottom_type) f->setDoesNotReturn(); @@ -4440,7 +4221,7 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio filename = ""; int toplineno = lno; - DIBuilder dbuilder(*builtins_module); + DIBuilder dbuilder(*M); ctx.dbuilder = &dbuilder; #ifdef LLVM37 DIFile *topfile = NULL; @@ -4646,7 +4427,7 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio /* // step 6. (optional) check for stack overflow (the slower way) Value *cur_sp = - builder.CreateCall(Intrinsic::getDeclaration(jl_Module, + builder.CreateCall(Intrinsic::getDeclaration(M, Intrinsic::frameaddress), ConstantInt::get(T_int32, 0)); Value *sp_ok = @@ -4678,12 +4459,6 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio // must be in the first basic block for the llvm mem2reg pass to work // get pointers for locals stored in the gc frame array (argTemp) -#ifdef LLVM36 - if (ctx.debug_enabled) { - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::dbg_declare)); - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::dbg_value)); - } -#endif for (std::map::iterator I = ctx.vars.begin(), E = ctx.vars.end(); I != E; ++I) { jl_sym_t *s = I->first; jl_varinfo_t &varinfo = I->second; @@ -5065,9 +4840,6 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio for(std::vector::iterator it = ctx.to_inline.begin(); it != ctx.to_inline.end(); ++it) { Function *inlinef = (*it)->getCalledFunction(); InlineFunctionInfo info; - // Intrinsics that InlineFunction might create - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::lifetime_start)); - prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::lifetime_end)); if (!InlineFunction(*it,info)) jl_error("Inlining Pass failed"); if (inlinef->getParent()) @@ -5078,26 +4850,14 @@ static void emit_function(jl_lambda_info_t *lam, jl_llvm_functions_t *declaratio } } -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - if (cyclectx) { - cyclectx->functions.push_back(f); - if (fwrap) - cyclectx->functions.push_back(fwrap); - } -#endif - // step 15. Perform any delayed instantiations if (ctx.debug_enabled) { -#if defined(USE_MCJIT) || defined(USE_ORCJIT) - if(cyclectx) - cyclectx->CUs.push_back(CU); -#endif ctx.dbuilder->finalize(); } JL_GC_POP(); - return; + return std::unique_ptr(M); } // --- initialization --- @@ -5140,8 +4900,8 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs } else { // this assigns a function pointer (from loading the system image), to the function object - std::string funcName = jl_symbol_name(lam->name); - funcName = "julia_" + funcName; + std::stringstream funcName; + funcName << "jlsys_" << jl_symbol_name(lam->name) << "_" << globalUnique++; if (specsig) { // assumes !va std::vector fsig(0); jl_value_t *jlrettype = lam->rettype; @@ -5168,8 +4928,7 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs ty = PointerType::get(ty,0); fsig.push_back(ty); } - Function *f = Function::Create(FunctionType::get(rt, fsig, false), Function::ExternalLinkage, funcName, - shadow_module); + Function *f = Function::Create(FunctionType::get(rt, fsig, false), Function::ExternalLinkage, funcName.str(), shadow_output); if (sret) f->addAttribute(1, Attribute::StructRet); @@ -5184,7 +4943,7 @@ extern "C" void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specs lam->fptr = fptr; } else { - Function *f = jlcall_func_to_llvm(funcName, fptr, shadow_module); + Function *f = jlcall_func_to_llvm(funcName.str(), fptr, shadow_output); if (lam->functionObjects.functionObject == NULL) { lam->functionObjects.functionObject = (void*)f; assert(lam->fptr == NULL); @@ -5388,14 +5147,6 @@ static void init_julia_llvm_env(Module *m) NULL, "jl_dl_handle"); add_named_global(jldll_var, &jl_dl_handle); #endif -#ifdef JL_NEED_FLOATTEMP_VAR - // Has to be big enough for the biggest LLVM-supported float type - jlfloattemp_var = - addComdat(new GlobalVariable(*m, IntegerType::get(jl_LLVMContext,128), - false, GlobalVariable::ExternalLinkage, - ConstantInt::get(IntegerType::get(jl_LLVMContext,128),0), - "jl_float_temp")); -#endif #ifndef JULIA_ENABLE_THREADING // For non-threading, we use the address of the global variable directly @@ -5432,17 +5183,11 @@ static void init_julia_llvm_env(Module *m) PointerType *pfunctype = jltls_states_func->getFunctionType()->getPointerTo(); jltls_states_func_ptr = new GlobalVariable(*m, pfunctype, - false, GlobalVariable::InternalLinkage, - ConstantPointerNull::get(pfunctype), - "jl_get_ptls_states.ptr"); + false, GlobalVariable::ExternalLinkage, + NULL, "jl_get_ptls_states.ptr"); addComdat(jltls_states_func_ptr); -#ifdef USE_MCJIT - jl_llvm_to_jl_value[jltls_states_func_ptr] = - (void*)jl_get_ptls_states_getter(); -#else - void **p = (void**)jl_ExecutionEngine->getPointerToGlobal(jltls_states_func_ptr); + void **p = (void**)jl_emit_and_add_to_shadow(jltls_states_func_ptr); *p = (void*)jl_get_ptls_states_getter(); -#endif jl_sysimg_gvars.push_back(ConstantExpr::getBitCast(jltls_states_func_ptr, T_psize)); jltls_states_func_idx = jl_sysimg_gvars.size(); @@ -5869,12 +5614,6 @@ static void init_julia_llvm_env(Module *m) add_named_global(jlcall_frame_func, (void*)NULL, /*dllimport*/false); // set up optimization passes -#ifdef LLVM38 - FPM = new legacy::FunctionPassManager(m); -#else - FPM = new FunctionPassManager(m); -#endif - #ifdef LLVM37 // No DataLayout pass needed anymore. #elif defined(LLVM36) @@ -5885,11 +5624,22 @@ static void init_julia_llvm_env(Module *m) jl_data_layout = new DataLayout(*jl_ExecutionEngine->getDataLayout()); #endif +#ifndef USE_ORCJIT +#ifdef LLVM38 + PM = new legacy::PassManager(); +#else + PM = new PassManager(); +#endif +#ifndef LLVM37 + PM->add(new TargetLibraryInfo(Triple(jl_TargetMachine->getTargetTriple()))); +#else + PM->add(new TargetLibraryInfoWrapperPass(Triple(jl_TargetMachine->getTargetTriple()))); +#endif #ifndef LLVM37 - FPM->add(jl_data_layout); + PM->add(jl_data_layout); +#endif + addOptimizationPasses(PM); #endif - addOptimizationPasses(FPM); - FPM->doInitialization(); } // Helper to figure out what features to set for the LLVM target @@ -5976,24 +5726,13 @@ extern "C" void jl_init_codegen(void) InitializeNativeTargetAsmParser(); Module *m, *engine_module; - + engine_module = new Module("julia", jl_LLVMContext); #ifdef USE_MCJIT - m = shadow_module = new Module("shadow", jl_LLVMContext); - builtins_module = new Module("julia_builtins", jl_LLVMContext); - if (imaging_mode) { - engine_module = new Module("engine_module", jl_LLVMContext); - active_module = shadow_module; - } - else { - active_module = new Module("julia", jl_LLVMContext); - engine_module = m; -#ifdef USE_ORCJIT - engine_module = new Module("engine_module", jl_LLVMContext); -#endif - } + m = new Module("julia", jl_LLVMContext); #else - engine_module = m = jl_Module = new Module("julia", jl_LLVMContext); + m = engine_module; #endif + shadow_output = m; TargetOptions options = TargetOptions(); //options.PrintMachineCode = true; //Print machine code produced during JIT compiling @@ -6095,19 +5834,9 @@ extern "C" void jl_init_codegen(void) mbuilder = new MDBuilder(getGlobalContext()); // Now that the execution engine exists, initialize all modules -#ifdef USE_MCJIT - jl_setup_module(engine_module); - jl_setup_module(shadow_module); - jl_setup_module(active_module); - jl_setup_module(builtins_module); -#else jl_setup_module(engine_module); -#endif - - if (imaging_mode) - init_julia_llvm_env(shadow_module); - else - init_julia_llvm_env(builtins_module); + jl_setup_module(m); + init_julia_llvm_env(m); #ifndef USE_ORCJIT jl_ExecutionEngine->RegisterJITEventListener(CreateJuliaJITEventListener()); diff --git a/src/codegen_internal.h b/src/codegen_internal.h index 02d08b8aa3034..860193849e59c 100644 --- a/src/codegen_internal.h +++ b/src/codegen_internal.h @@ -1,4 +1,6 @@ -// Pre-declaration. Definition in disasm.cpp +// This file is a part of Julia. License is MIT: http://julialang.org/license + +// Declarations for disasm.cpp extern "C" void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #ifndef USE_MCJIT @@ -12,6 +14,7 @@ void jl_dump_asm_internal(uintptr_t Fptr, size_t Fsize, int64_t slide, #endif ); +// Declarations for debuginfo.cpp extern int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, int64_t *section_slide, const object::ObjectFile **object, #ifdef USE_MCJIT @@ -20,8 +23,17 @@ extern int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, int6 std::vector *lines #endif ); + extern bool jl_dylib_DI_for_fptr(size_t pointer, const object::ObjectFile **object, llvm::DIContext **context, int64_t *slide, int64_t *section_slide, bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename); + #ifdef USE_MCJIT extern void jl_cleanup_DI(llvm::DIContext *context); #endif + +#ifdef USE_ORCJIT +extern JL_DLLEXPORT void ORCNotifyObjectEmitted(JITEventListener *Listener, + const object::ObjectFile &obj, + const object::ObjectFile &debugObj, + const RuntimeDyld::LoadedObjectInfo &L); +#endif diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 2584f59f61895..198c68202922a 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -238,6 +238,10 @@ class JuliaJITEventListener: public JITEventListener #endif FuncInfo tmp = {&F, Size, Details.LineStarts, linfo}; info[(size_t)(Code)] = tmp; +#ifndef KEEP_BODIES + if (!jl_generating_output()) + const_cast(&F)->deleteBody(); +#endif uv_rwlock_wrunlock(&threadsafe); } diff --git a/src/dump.c b/src/dump.c index a62366553cfc7..4b9c9cea88d8a 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1921,7 +1921,7 @@ JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname) } // Get handle to sys.so - jl_sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_DEFAULT | JL_RTLD_GLOBAL); + jl_sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_LOCAL | JL_RTLD_NOW); // set cpu target if unspecified by user and available from sysimg // otherwise default to native. diff --git a/src/gf.c b/src/gf.c index 5d2286911d5d0..880c807f26d63 100644 --- a/src/gf.c +++ b/src/gf.c @@ -870,7 +870,7 @@ static jl_value_t *jl_call_unspecialized(jl_svec_t *sparam_vals, jl_lambda_info_ jl_value_t **args, uint32_t nargs) { if (__unlikely(meth->fptr == NULL)) { - jl_compile_linfo(meth, NULL); + jl_compile_linfo(meth); jl_generate_fptr(meth); } assert(jl_svec_len(meth->sparam_syms) == jl_svec_len(sparam_vals)); @@ -1414,7 +1414,7 @@ jl_lambda_info_t *jl_method_lookup(jl_methtable_t *mt, jl_value_t **args, size_t JL_DLLEXPORT jl_value_t *jl_matching_methods(jl_value_t *types, int lim); // compile-time method lookup -jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types, void *cyclectx) +jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types) { assert(jl_nparams(types) > 0); if (!jl_is_leaf_type((jl_value_t*)types)) @@ -1451,7 +1451,7 @@ jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types, void *cyclectx) if (sf->functionObjects.functionObject == NULL) { if (sf->fptr != NULL) goto not_found; - jl_compile_linfo(sf, cyclectx); + jl_compile_linfo(sf); } JL_GC_POP(); return sf; @@ -1506,7 +1506,7 @@ static int _compile_all_tvar_union(jl_methlist_t *meth, jl_tupletype_t *methsig) if (jl_is_leaf_type((jl_value_t*)methsig)) { // usually can create a specialized version of the function, // if the signature is already a leaftype - jl_lambda_info_t *spec = jl_get_specialization1(methsig, NULL); + jl_lambda_info_t *spec = jl_get_specialization1(methsig); if (spec) { if (methsig == meth->sig) { // replace unspecialized func with newly specialized version @@ -1545,7 +1545,7 @@ static int _compile_all_tvar_union(jl_methlist_t *meth, jl_tupletype_t *methsig) goto getnext; // signature wouldn't be callable / is invalid -- skip it } if (jl_is_leaf_type(sig)) { - if (jl_get_specialization1((jl_tupletype_t*)sig, NULL)) { + if (jl_get_specialization1((jl_tupletype_t*)sig)) { if (!jl_has_typevars((jl_value_t*)sig)) goto getnext; // success } } @@ -1680,7 +1680,7 @@ static void _compile_all_deq(jl_array_t *found) linfo->functionID = -1; } else { - jl_compile_linfo(linfo, NULL); + jl_compile_linfo(linfo); assert(linfo->functionID > 0); } } @@ -1783,7 +1783,7 @@ void jl_compile_all(void) JL_DLLEXPORT void jl_compile_hint(jl_tupletype_t *types) { - (void)jl_get_specialization1(types, NULL); + (void)jl_get_specialization1(types); } #ifdef JL_TRACE diff --git a/src/init.c b/src/init.c index 3930323cf43cc..2865b5ba0693a 100644 --- a/src/init.c +++ b/src/init.c @@ -615,6 +615,11 @@ void _julia_init(JL_IMAGE_SEARCH rel) init_stdio(); // libuv stdio cleanup depends on jl_init_tasks() because JL_TRY is used in jl_atexit_hook() + if ((jl_options.outputo || jl_options.outputbc) && + (jl_options.code_coverage || jl_options.malloc_log)) { + jl_error("cannot generate code-coverage or track allocation information while generating a .o or .bc output file"); + } + jl_init_codegen(); jl_start_threads(); diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index cb68f82a3f6ad..573e127e3bc90 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -1012,8 +1012,9 @@ static jl_cgval_t emit_intrinsic(intrinsic f, jl_value_t **args, size_t nargs, // understood that everything is implicitly rounded to 23 bits, // but if we start looking at more bits we need to actually do the // rounding first instead of carrying around incorrect low bits. - builder.CreateStore(FP(x), builder.CreateBitCast(prepare_global(jlfloattemp_var),FT(x->getType())->getPointerTo()), true); - x = builder.CreateLoad(builder.CreateBitCast(prepare_global(jlfloattemp_var),FT(x->getType())->getPointerTo()), true); + Value *jlfloattemp_var = emit_static_alloca(FT(x->getType())); + builder.CreateStore(FP(x), jlfloattemp_var); + x = builder.CreateLoad(jlfloattemp_var, true); #endif return mark_julia_type(builder.CreateFPExt(x, FTnbits(nb)), false, bt, ctx); } @@ -1140,12 +1141,12 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, case fma_float: { assert(y->getType() == x->getType()); assert(z->getType() == y->getType()); - Value *fmaintr = Intrinsic::getDeclaration(builtins_module, Intrinsic::fma, + Value *fmaintr = Intrinsic::getDeclaration(jl_Module, Intrinsic::fma, ArrayRef(x->getType())); #ifdef LLVM37 - return builder.CreateCall(prepare_call(fmaintr),{ FP(x), FP(y), FP(z) }); + return builder.CreateCall(fmaintr,{ FP(x), FP(y), FP(z) }); #else - return builder.CreateCall3(prepare_call(fmaintr), FP(x), FP(y), FP(z)); + return builder.CreateCall3(fmaintr, FP(x), FP(y), FP(z)); #endif } case muladd_float: @@ -1158,8 +1159,8 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, #else return builder.CreateCall3 #endif - (prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::fmuladd, - ArrayRef(x->getType()))), + (Intrinsic::getDeclaration(jl_Module, Intrinsic::fmuladd, + ArrayRef(x->getType())), #ifdef LLVM37 {FP(x), FP(y), FP(z)} #else @@ -1181,7 +1182,7 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Value *ix = JL_INT(x); Value *iy = JL_INT(y); assert(ix->getType() == iy->getType()); Value *intr = - Intrinsic::getDeclaration(builtins_module, + Intrinsic::getDeclaration(jl_Module, f==checked_sadd_int ? Intrinsic::sadd_with_overflow : (f==checked_uadd_int ? @@ -1195,9 +1196,9 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Intrinsic::umul_with_overflow)))), ArrayRef(ix->getType())); #ifdef LLVM37 - Value *res = builder.CreateCall(prepare_call(intr),{ix, iy}); + Value *res = builder.CreateCall(intr,{ix, iy}); #else - Value *res = builder.CreateCall2(prepare_call(intr), ix, iy); + Value *res = builder.CreateCall2(intr, ix, iy); #endif Value *obit = builder.CreateExtractValue(res, ArrayRef(1)); raise_exception_if(obit, prepare_global(jlovferr_var), ctx); @@ -1335,33 +1336,33 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, builder.CreateAShr(x, uint_cnvt(t,y))); case bswap_int: x = JL_INT(x); - return builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, Intrinsic::bswap, - ArrayRef(x->getType()))), x); + return builder.CreateCall( + Intrinsic::getDeclaration(jl_Module, Intrinsic::bswap, + ArrayRef(x->getType())), x); case ctpop_int: x = JL_INT(x); - return builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, Intrinsic::ctpop, - ArrayRef(x->getType()))), x); + return builder.CreateCall( + Intrinsic::getDeclaration(jl_Module, Intrinsic::ctpop, + ArrayRef(x->getType())), x); case ctlz_int: { x = JL_INT(x); Type *types[1] = {x->getType()}; - Value *ctlz = Intrinsic::getDeclaration(builtins_module, Intrinsic::ctlz, + Value *ctlz = Intrinsic::getDeclaration(jl_Module, Intrinsic::ctlz, ArrayRef(types)); #ifdef LLVM37 - return builder.CreateCall(prepare_call(ctlz), {x, ConstantInt::get(T_int1,0)}); + return builder.CreateCall(ctlz, {x, ConstantInt::get(T_int1,0)}); #else - return builder.CreateCall2(prepare_call(ctlz), x, ConstantInt::get(T_int1,0)); + return builder.CreateCall2(ctlz, x, ConstantInt::get(T_int1,0)); #endif } case cttz_int: { x = JL_INT(x); Type *types[1] = {x->getType()}; - Value *cttz = Intrinsic::getDeclaration(builtins_module, Intrinsic::cttz, ArrayRef(types)); + Value *cttz = Intrinsic::getDeclaration(jl_Module, Intrinsic::cttz, ArrayRef(types)); #ifdef LLVM37 - return builder.CreateCall(prepare_call(cttz), {x, ConstantInt::get(T_int1, 0)}); + return builder.CreateCall(cttz, {x, ConstantInt::get(T_int1, 0)}); #else - return builder.CreateCall2(prepare_call(cttz), x, ConstantInt::get(T_int1, 0)); + return builder.CreateCall2(cttz, x, ConstantInt::get(T_int1, 0)); #endif } @@ -1378,9 +1379,9 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, { x = FP(x); #ifdef LLVM34 - return builder.CreateCall(prepare_call( - Intrinsic::getDeclaration(builtins_module, Intrinsic::fabs, - ArrayRef(x->getType()))), + return builder.CreateCall( + Intrinsic::getDeclaration(jl_Module, Intrinsic::fabs, + ArrayRef(x->getType())), x); #else Type *intt = JL_INTT(x->getType()); @@ -1431,34 +1432,34 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, } case ceil_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::ceil, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::ceil, + ArrayRef(x->getType())), x); } case floor_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::floor, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::floor, + ArrayRef(x->getType())), x); } case trunc_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::trunc, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::trunc, + ArrayRef(x->getType())), x); } case rint_llvm: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::rint, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::rint, + ArrayRef(x->getType())), x); } case sqrt_llvm: { x = FP(x); raise_exception_unless(builder.CreateFCmpUGE(x, ConstantFP::get(x->getType(),0.0)), prepare_global(jldomerr_var), ctx); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::sqrt, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::sqrt, + ArrayRef(x->getType())), x); } case powi_llvm: { @@ -1467,12 +1468,12 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, Type *tx = x->getType(); // TODO: LLVM expects this to be i32 #ifdef LLVM36 Type *ts[1] = { tx }; - Value *powi = Intrinsic::getDeclaration(builtins_module, Intrinsic::powi, + Value *powi = Intrinsic::getDeclaration(jl_Module, Intrinsic::powi, ArrayRef(ts)); #ifdef LLVM37 - return builder.CreateCall(prepare_call(powi), {x, y}); + return builder.CreateCall(powi, {x, y}); #else - return builder.CreateCall2(prepare_call(powi), x, y); + return builder.CreateCall2(powi, x, y); #endif #else // issue #6506 @@ -1482,8 +1483,8 @@ static Value *emit_untyped_intrinsic(intrinsic f, Value *x, Value *y, Value *z, } case sqrt_llvm_fast: { x = FP(x); - return builder.CreateCall(prepare_call(Intrinsic::getDeclaration(builtins_module, Intrinsic::sqrt, - ArrayRef(x->getType()))), + return builder.CreateCall(Intrinsic::getDeclaration(jl_Module, Intrinsic::sqrt, + ArrayRef(x->getType())), x); } diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 78a56b3bfe9a2..2f560cc3d95e2 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1,6 +1,8 @@ -// This file is part of Julia. -// Parts of this file are copied from LLVM, under the UIUC license. +// This file is a part of Julia. License is MIT: http://julialang.org/license +// Except for parts of this file which were copied from LLVM, under the UIUC license (marked below). + +// this defines the set of optimization passes defined for Julia at various optimization levels template static void addOptimizationPasses(T *PM) { @@ -158,11 +160,6 @@ extern "C" { LLVM_ATTRIBUTE_NOINLINE extern void __jit_debug_register_code(); } -extern JL_DLLEXPORT void ORCNotifyObjectEmitted(JITEventListener *Listener, - const object::ObjectFile &obj, - const object::ObjectFile &debugObj, - const RuntimeDyld::LoadedObjectInfo &L); - #if defined(_OS_DARWIN_) && defined(LLVM37) && defined(LLVM_SHLIB) #define CUSTOM_MEMORY_MANAGER createRTDyldMemoryManagerOSX extern RTDyldMemoryManager *createRTDyldMemoryManagerOSX(); @@ -193,9 +190,10 @@ void NotifyDebugger(jit_code_entry *JITCodeEntry) __jit_debug_descriptor.relevant_entry = JITCodeEntry; __jit_debug_register_code(); } +// ------------------------ END OF TEMPORARY COPY FROM LLVM ----------------- -// -------------------------------------------------------------------------- - +// Custom object emission notification handler for the JuliaOJIT +// TODO: hook up RegisterJITEventListener, instead of hard-coding the GDB and JuliaListener targets class DebugObjectRegistrar { private: void NotifyGDB(OwningBinary &DebugObj) @@ -253,7 +251,11 @@ class DebugObjectRegistrar { } SavedObjects.push_back(std::move(SavedObject)); - ORCNotifyObjectEmitted(JuliaListener.get(),*Object,*SavedObjects.back().getBinary(),*LO); + + ORCNotifyObjectEmitted(JuliaListener.get(), + *Object, + *SavedObjects.back().getBinary(), + *LO); ++oit; ++lit; @@ -261,6 +263,8 @@ class DebugObjectRegistrar { } }; +// A simplified model of the LLVM ExecutionEngine that implements only the methods that Julia needs +// but tries to roughly match the API anyways so that compatibility is easier class JuliaOJIT { public: typedef orc::ObjectLinkingLayer ObjLayerT; @@ -313,26 +317,49 @@ class JuliaOJIT { report_fatal_error("FATAL: unable to dlopen self\n" + *ErrorStr); } - std::string mangle(const std::string &Name) + std::string getMangledName(const std::string &Name) { - std::string MangledName; - { - raw_string_ostream MangledNameStream(MangledName); - Mangler::getNameWithPrefix(MangledNameStream, Name, DL); - } - return MangledName; + SmallString<128> FullName; + Mangler::getNameWithPrefix(FullName, Name, DL); + return FullName.str(); + } + + std::string getMangledName(const GlobalValue *GV) + { + return getMangledName(GV->getName()); } - void addGlobalMapping(StringRef Name, void *Addr) + void addGlobalMapping(StringRef Name, uint64_t Addr) { - GlobalSymbolTable[mangle(Name)] = Addr; + bool successful = GlobalSymbolTable.insert(make_pair(getMangledName(Name), (void*)Addr)).second; + assert(successful); + } + + void *getPointerToGlobalIfAvailable(StringRef S) { + GlobalSymbolTableT::const_iterator pos = GlobalSymbolTable.find(S); + if (pos != GlobalSymbolTable.end()) + return pos->second; + return nullptr; } - ModuleHandleT addModule(Module *M) + ModuleHandleT addModule(std::unique_ptr M) { +#ifndef NDEBUG + // validate the relocations for M + for (Module::iterator I = M->begin(), E = M->end(); I != E; ) { + Function *F = &*I; + ++I; + if (F->isDeclaration()) { + if (F->use_empty()) + F->eraseFromParent(); + else + assert(F->isIntrinsic() || findUnmangledSymbol(F->getName()) || + SectionMemoryManager::getSymbolAddressInProcess(F->getName())); + } + } +#endif // We need a memory manager to allocate memory and resolve symbols for this - // new module. Create one that resolves symbols by looking back into the - // JIT. + // new module. Create one that resolves symbols by looking back into the JIT. auto Resolver = orc::createLambdaResolver( [&](const std::string &Name) { // TODO: consider moving the FunctionMover resolver here @@ -350,7 +377,7 @@ class JuliaOJIT { [](const std::string &S) { return nullptr; } ); SmallVector,1> Ms; - Ms.push_back(std::unique_ptr{M}); + Ms.push_back(std::move(M)); return CompileLayer->addModuleSet(std::move(Ms), MemMgr, std::move(Resolver)); @@ -362,9 +389,9 @@ class JuliaOJIT { { if (ExportedSymbolsOnly) { // Step 1: Check against list of known external globals - GlobalSymbolTableT::const_iterator pos = GlobalSymbolTable.find(Name); - if (pos != GlobalSymbolTable.end()) - return orc::JITSymbol((uintptr_t)pos->second, JITSymbolFlags::Exported); + void *Addr = getPointerToGlobalIfAvailable(Name); + if (Addr != nullptr) + return orc::JITSymbol((uintptr_t)Addr, JITSymbolFlags::Exported); } // Step 2: Search all previously emitted symbols return CompileLayer->findSymbol(Name, ExportedSymbolsOnly); @@ -372,17 +399,17 @@ class JuliaOJIT { orc::JITSymbol findUnmangledSymbol(const std::string Name) { - return findSymbol(mangle(Name), true); + return findSymbol(getMangledName(Name), true); } uint64_t getGlobalValueAddress(const std::string &Name) { - return findSymbol(mangle(Name), false).getAddress(); + return findSymbol(getMangledName(Name), false).getAddress(); } uint64_t getFunctionAddress(const std::string &Name) { - return findSymbol(mangle(Name), false).getAddress(); + return findSymbol(getMangledName(Name), false).getAddress(); } Function *FindFunctionNamed(const std::string &Name) @@ -422,3 +449,555 @@ class JuliaOJIT { } #endif + +#ifdef USE_ORCJIT +JuliaOJIT *jl_ExecutionEngine; +#else +ExecutionEngine *jl_ExecutionEngine; +#endif + +// destructively move the contents of src into dest +// this assumes that the targets of the two modules are the same +// including the DataLayout and ModuleFlags (for example) +// and that there is no module-level assembly +static void jl_merge_module(Module *dest, std::unique_ptr src) +{ + assert(dest != src.get()); + for (Module::global_iterator I = src->global_begin(), E = src->global_end(); I != E;) { + GlobalVariable *sG = &*I; + GlobalValue *dG = dest->getNamedValue(sG->getName()); + ++I; + if (dG) { + if (sG->isDeclaration()) { + sG->replaceAllUsesWith(dG); + sG->eraseFromParent(); + continue; + } + else { + dG->replaceAllUsesWith(sG); + dG->eraseFromParent(); + } + } + sG->removeFromParent(); + dest->getGlobalList().push_back(sG); + } + + for (Module::iterator I = src->begin(), E = src->end(); I != E;) { + Function *sG = &*I; + GlobalValue *dG = dest->getNamedValue(sG->getName()); + ++I; + if (dG) { + if (sG->isDeclaration()) { + sG->replaceAllUsesWith(dG); + sG->eraseFromParent(); + continue; + } + else { + dG->replaceAllUsesWith(sG); + dG->eraseFromParent(); + } + } + sG->removeFromParent(); + dest->getFunctionList().push_back(sG); + } + + for (Module::alias_iterator I = src->alias_begin(), E = src->alias_end(); I != E;) { + GlobalAlias *sG = &*I; + GlobalValue *dG = dest->getNamedValue(sG->getName()); + ++I; + if (dG) { + if (!dG->isDeclaration()) { // aliases are always definitions, so this test is reversed from the above two + sG->replaceAllUsesWith(dG); + sG->eraseFromParent(); + continue; + } + else { + dG->replaceAllUsesWith(sG); + dG->eraseFromParent(); + } + } + sG->removeFromParent(); + dest->getAliasList().push_back(sG); + } + + // metadata nodes need to be explicitly merged not just copied + // so there are special passes here for each known type of metadata + NamedMDNode *sNMD = src->getNamedMetadata("llvm.dbg.cu"); + if (sNMD) { + NamedMDNode *dNMD = dest->getOrInsertNamedMetadata("llvm.dbg.cu"); +#ifdef LLVM35 + for (NamedMDNode::op_iterator I = sNMD->op_begin(), E = sNMD->op_end(); I != E; ++I) { + dNMD->addOperand(*I); + } +#else + for (unsigned i = 0, l = sNMD->getNumOperands(); i < l; i++) { + dNMD->addOperand(sNMD->getOperand(i)); + } +#endif + } +} + +// to finalizing a function, look up its name in the `module_for_fname` map of unfinalized functions +// and merge it, plus any other modules it depends upon, into `collector` +// then add `collector` to the execution engine +// +// in the old JIT, functions are finalized by adding them to the shadow module +// (which aliases the engine module), so this is unneeded +#ifdef USE_MCJIT +static StringMap module_for_fname; +static void jl_finalize_function(StringRef F, Module *collector = NULL) +{ + std::unique_ptr m(module_for_fname.lookup(F)); + if (m) { + // probably not many unresolved declarations, but be sure iterate over their Names, + // since the declarations may get destroyed by the jl_merge_module call + SmallVector to_finalize; + for (Module::iterator I = m->begin(), E = m->end(); I != E; ++I) { + Function *F = &*I; + if (!F->isDeclaration()) { + module_for_fname.erase(F->getName()); + } + else if (!F->isIntrinsic()) { + to_finalize.push_back(F->getName()); + } + } + + for (auto F : to_finalize) { + jl_finalize_function(F, collector ? collector : m.get()); + } + + if (collector) { + jl_merge_module(collector, std::move(m)); + } + else { +#if defined(_CPU_X86_64_) && defined(_OS_WINDOWS_) && defined(LLVM35) + // Add special values used by debuginfo to build the UnwindData table registration for Win64 + ArrayType *atype = ArrayType::get(T_uint32, 3); // want 4-byte alignment of 12-bytes of data + (new GlobalVariable(*m, atype, + false, GlobalVariable::InternalLinkage, + ConstantAggregateZero::get(atype), "__UnwindData"))->setSection(".text"); + (new GlobalVariable(*m, atype, + false, GlobalVariable::InternalLinkage, + ConstantAggregateZero::get(atype), "__catchjmp"))->setSection(".text"); +#endif + assert(jl_ExecutionEngine); +#if defined(LLVM36) + jl_ExecutionEngine->addModule(std::move(m)); +#else + jl_ExecutionEngine->addModule(m.release()); +#endif + } + } +} +static void jl_finalize_function(Function *F, Module *collector = NULL) +{ + jl_finalize_function(F->getName(), collector); +} +#endif + +// MSVC's link.exe requires each function declaration to have a Comdat section +// rather than litter the code with conditionals, +// all global values that get emitted call this function +// and it decides whether the definition needs a Comdat section and adds the appropriate declation +// TODO: consider moving this into jl_add_to_shadow or jl_dump_shadow? the JIT doesn't care, so most calls are now no-ops +template // for GlobalObject's +static T *addComdat(T *G) +{ +#if defined(_OS_WINDOWS_) + if (imaging_mode && !G->isDeclaration()) { +#ifdef LLVM35 + // Add comdat information to make MSVC link.exe happy + Comdat *jl_Comdat = G->getParent()->getOrInsertComdat(G->getName()); + jl_Comdat->setSelectionKind(Comdat::NoDuplicates); + G->setComdat(jl_Comdat); + // add __declspec(dllexport) to everything marked for export + if (G->getLinkage() == GlobalValue::ExternalLinkage) + G->setDLLStorageClass(GlobalValue::DLLExportStorageClass); +#endif + } +#endif + return G; +} + +// Connect Modules via prototypes, each owned by module `M` +static GlobalVariable *global_proto(GlobalVariable *G, Module *M = NULL) +{ + // Copy the GlobalVariable, but without the initializer, so it becomes a declaration + GlobalVariable *proto = new GlobalVariable(G->getType()->getElementType(), + G->isConstant(), GlobalVariable::ExternalLinkage, + NULL, G->getName(), G->getThreadLocalMode()); + if (M) + M->getGlobalList().push_back(proto); + return proto; +} + +static Function *function_proto(Function *F, Module *M = NULL) +{ + // Copy the declaration characteristics of the Function (not the body) + Function *NewF = Function::Create(F->getFunctionType(), + Function::ExternalLinkage, + F->getName(), M); + NewF->setAttributes(AttributeSet()); + + // FunctionType does not include any attributes. Copy them over manually + // as codegen may make decisions based on the presence of certain attributes + NewF->copyAttributesFrom(F); + +#ifdef LLVM37 + // Declarations are not allowed to have personality routines, but + // copyAttributesFrom sets them anyway, so clear them again manually + NewF->setPersonalityFn(nullptr); +#endif + + return NewF; +} + +// helper function for adding a DLLImport (dlsym) address to the execution engine +// (for values created locally or in the sysimage, jl_emit_and_add_to_shadow is generally preferable) +template +#ifdef LLVM35 +static inline void add_named_global(GlobalObject *gv, T *_addr, bool dllimport = true) +#else +static inline void add_named_global(GlobalValue *gv, T *_addr, bool dllimport = true) +#endif +{ + // cast through integer to avoid c++ pedantic warning about casting between + // data and code pointers + void *addr = (void*)(uintptr_t)_addr; +#ifdef LLVM34 + StringRef name = gv->getName(); +#ifdef _OS_WINDOWS_ + std::string imp_name; +#endif +#endif + +#ifdef _OS_WINDOWS_ + // setting JL_DLLEXPORT correctly only matters when building a binary + if (dllimport && imaging_mode) { + assert(gv->getLinkage() == GlobalValue::ExternalLinkage); +#ifdef LLVM35 + // add the __declspec(dllimport) attribute + gv->setDLLStorageClass(GlobalValue::DLLImportStorageClass); + // this will cause llvm to rename it, so we do the same + imp_name = Twine("__imp_", name).str(); + name = StringRef(imp_name); +#else + gv->setLinkage(GlobalValue::DLLImportLinkage); +#endif +#if defined(_P64) || defined(LLVM35) + // __imp_ variables are indirection pointers, so use malloc to simulate that + void **imp_addr = (void**)malloc(sizeof(void**)); + *imp_addr = addr; + addr = (void*)imp_addr; +#endif + } +#endif // _OS_WINDOWS_ + +#ifdef USE_ORCJIT + addComdat(gv); + jl_ExecutionEngine->addGlobalMapping(name, (uintptr_t)addr); +#elif defined(USE_MCJIT) + addComdat(gv); + sys::DynamicLibrary::AddSymbol(name, addr); +#else // USE_MCJIT + jl_ExecutionEngine->addGlobalMapping(gv, addr); +#endif // USE_MCJIT +} + +static std::vector jl_sysimg_gvars; +static std::vector jl_sysimg_fvars; +typedef struct {Value *gv; int32_t index;} jl_value_llvm; // uses 1-based indexing +static std::map jl_value_to_llvm; + +// global variables to pointers are pretty common, +// so this method is available as a convenience for emitting them. +// for other types, the formula for implementation is straightforward: +// (see stringConstPtr, for an alternative example to the code below) +// +// if in imaging_mode, emit a GlobalVariable with the same name and an initializer to the shadow_module +// making it valid for emission and reloading in the sysimage +// +// then add a global mapping to the current value (usually from calloc'd space) +// to the execution engine to make it valid for the current session (with the current value) +static void* jl_emit_and_add_to_shadow(GlobalVariable *gv, void *gvarinit = NULL) +{ + PointerType *T = cast(gv->getType()->getElementType()); // pointer is the only supported type here + + GlobalVariable *shadowvar = NULL; +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + if (imaging_mode) +#endif + shadowvar = global_proto(gv, shadow_output); + + if (shadowvar) { + shadowvar->setInitializer(ConstantPointerNull::get(T)); + shadowvar->setLinkage(GlobalVariable::InternalLinkage); + addComdat(shadowvar); + if (imaging_mode && gvarinit) { + // make the pointer valid for future sessions + jl_sysimg_gvars.push_back(ConstantExpr::getBitCast(shadowvar, T_psize)); + jl_value_llvm gv_struct; + gv_struct.gv = global_proto(gv); + gv_struct.index = jl_sysimg_gvars.size(); + jl_value_to_llvm[gvarinit] = gv_struct; + } + } + + // make the pointer valid for this session +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + void *slot = calloc(1, sizeof(void*)); + jl_ExecutionEngine->addGlobalMapping(gv->getName(), (uintptr_t)slot); + return slot; +#else + return jl_ExecutionEngine->getPointerToGlobal(shadowvar); +#endif +} + +static void* jl_get_global(GlobalVariable *gv) +{ +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + void *p = (void*)(intptr_t)jl_ExecutionEngine->getPointerToGlobalIfAvailable( + jl_ExecutionEngine->getMangledName(gv)); +#else + void *p = jl_ExecutionEngine->getPointerToGlobal( + shadow_output->getNamedValue(gv->getName())); +#endif + assert(p); + return p; +} + +// clones the contents of the module `m` to the shadow_output collector +// in the old JIT, this is equivalent to also adding it to the execution engine +static void jl_add_to_shadow(Module *m) +{ +#if defined(USE_MCJIT) || defined(USE_ORCJIT) + if (!imaging_mode) + return; + ValueToValueMapTy VMap; + std::unique_ptr clone(CloneModule(m, VMap)); + for (Module::iterator I = clone->begin(), E = clone->end(); I != E; ++I) { + Function *F = &*I; + if (!F->isDeclaration()) { + F->setLinkage(Function::InternalLinkage); + addComdat(F); + } + } +#else + // on the old jit, the shadow_module is the same as the execution engine_module + std::unique_ptr clone(m); +#endif + jl_merge_module(shadow_output, std::move(clone)); +} + +#ifdef HAVE_CPUID +extern "C" { + extern void jl_cpuid(int32_t CPUInfo[4], int32_t InfoType); +} +#endif + +static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap, + const char *sysimg_data, size_t sysimg_len) +{ + ArrayType *gvars_type = ArrayType::get(T_psize, jl_sysimg_gvars.size()); + addComdat(new GlobalVariable(*mod, + gvars_type, + true, + GlobalVariable::ExternalLinkage, + MapValue(ConstantArray::get(gvars_type, ArrayRef(jl_sysimg_gvars)), VMap), + "jl_sysimg_gvars")); + ArrayType *fvars_type = ArrayType::get(T_pvoidfunc, jl_sysimg_fvars.size()); + addComdat(new GlobalVariable(*mod, + fvars_type, + true, + GlobalVariable::ExternalLinkage, + MapValue(ConstantArray::get(fvars_type, ArrayRef(jl_sysimg_fvars)), VMap), + "jl_sysimg_fvars")); + addComdat(new GlobalVariable(*mod, + T_size, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_size, globalUnique+1), + "jl_globalUnique")); +#ifdef JULIA_ENABLE_THREADING + addComdat(new GlobalVariable(*mod, + T_size, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_size, jltls_states_func_idx), + "jl_ptls_states_getter_idx")); +#endif + + Constant *feature_string = ConstantDataArray::getString(jl_LLVMContext, jl_options.cpu_target); + addComdat(new GlobalVariable(*mod, + feature_string->getType(), + true, + GlobalVariable::ExternalLinkage, + feature_string, + "jl_sysimg_cpu_target")); + +#ifdef HAVE_CPUID + // For native also store the cpuid + if (strcmp(jl_options.cpu_target,"native") == 0) { + uint32_t info[4]; + + jl_cpuid((int32_t*)info, 1); + addComdat(new GlobalVariable(*mod, + T_int64, + true, + GlobalVariable::ExternalLinkage, + ConstantInt::get(T_int64,((uint64_t)info[2])|(((uint64_t)info[3])<<32)), + "jl_sysimg_cpu_cpuid")); + } +#endif + + if (sysimg_data) { + Constant *data = ConstantDataArray::get(jl_LLVMContext, ArrayRef((const unsigned char*)sysimg_data, sysimg_len)); + addComdat(new GlobalVariable(*mod, data->getType(), true, + GlobalVariable::ExternalLinkage, + data, "jl_system_image_data")); + Constant *len = ConstantInt::get(T_size, sysimg_len); + addComdat(new GlobalVariable(*mod, len->getType(), true, + GlobalVariable::ExternalLinkage, + len, "jl_system_image_size")); + } +} + +// takes the running content that has collected in the shadow module and dump it to disk +// this builds the object file portion of the sysimage files for fast startup +static void jl_dump_shadow(char *fname, int jit_model, const char *sysimg_data, size_t sysimg_len, + bool dump_as_bc) +{ +#ifdef JL_DEBUG_BUILD + verifyModule(*shadow_output); +#endif + +#ifdef LLVM36 + std::error_code err; + StringRef fname_ref = StringRef(fname); + raw_fd_ostream OS(fname_ref, err, sys::fs::F_None); +#elif defined(LLVM35) + std::string err; + raw_fd_ostream OS(fname, err, sys::fs::F_None); +#else + std::string err; + raw_fd_ostream OS(fname, err); +#endif +#ifdef LLVM37 // 3.7 simplified formatted output; just use the raw stream alone + raw_fd_ostream& FOS(OS); +#else + formatted_raw_ostream FOS(OS); +#endif + + // We don't want to use MCJIT's target machine because + // it uses the large code model and we may potentially + // want less optimizations there. + Triple TheTriple = Triple(jl_TargetMachine->getTargetTriple()); +#if defined(_OS_WINDOWS_) && defined(FORCE_ELF) +#ifdef LLVM35 + TheTriple.setObjectFormat(Triple::COFF); +#else + TheTriple.setEnvironment(Triple::UnknownEnvironment); +#endif +#elif defined(_OS_DARWIN_) && defined(FORCE_ELF) +#ifdef LLVM35 + TheTriple.setObjectFormat(Triple::MachO); +#else + TheTriple.setEnvironment(Triple::MachO); +#endif +#endif +#ifdef LLVM35 + std::unique_ptr +#else + OwningPtr +#endif + TM(jl_TargetMachine->getTarget().createTargetMachine( + TheTriple.getTriple(), + jl_TargetMachine->getTargetCPU(), + jl_TargetMachine->getTargetFeatureString(), + jl_TargetMachine->Options, +#if defined(_OS_LINUX_) || defined(_OS_FREEBSD_) + Reloc::PIC_, +#else + jit_model ? Reloc::PIC_ : Reloc::Default, +#endif + jit_model ? CodeModel::JITDefault : CodeModel::Default, + CodeGenOpt::Aggressive // -O3 TODO: respect command -O0 flag? + )); + +#ifdef LLVM38 + legacy::PassManager PM; +#else + PassManager PM; +#endif +#ifndef LLVM37 + PM.add(new TargetLibraryInfo(Triple(TM->getTargetTriple()))); +#else + PM.add(new TargetLibraryInfoWrapperPass(Triple(TM->getTargetTriple()))); +#endif + + + // set up optimization passes +#ifdef LLVM37 + // No DataLayout pass needed anymore. +#elif defined(LLVM36) + PM.add(new DataLayoutPass()); +#elif defined(LLVM35) + PM.add(new DataLayoutPass(*jl_ExecutionEngine->getDataLayout())); +#else + PM.add(new DataLayout(*jl_ExecutionEngine->getDataLayout())); +#endif + + addOptimizationPasses(&PM); + if (!dump_as_bc) { + if (TM->addPassesToEmitFile(PM, FOS, TargetMachine::CGFT_ObjectFile, false)) { + jl_error("Could not generate obj file for this target"); + } + } + + // now copy the module, since PM.run may modify it + ValueToValueMapTy VMap; +#ifdef LLVM38 + Module *clone = CloneModule(shadow_output, VMap).release(); +#else + Module *clone = CloneModule(shadow_output, VMap); +#endif +#ifdef LLVM37 + // Reset the target triple to make sure it matches the new target machine + clone->setTargetTriple(TM->getTargetTriple().str()); +#ifdef LLVM38 + clone->setDataLayout(TM->createDataLayout()); +#else + clone->setDataLayout(TM->getDataLayout()->getStringRepresentation()); +#endif +#endif + + // add metadata information + jl_gen_llvm_globaldata(clone, VMap, sysimg_data, sysimg_len); + + // do the actual work + PM.run(*clone); + if (dump_as_bc) + WriteBitcodeToFile(clone, FOS); + delete clone; +} + +static int32_t jl_assign_functionID(Function *functionObject, int specsig) +{ + // give the function an index in the constant lookup table + if (!imaging_mode) + return 0; + jl_sysimg_fvars.push_back(ConstantExpr::getBitCast( + shadow_output->getNamedValue(functionObject->getName()), + T_pvoidfunc)); + return jl_sysimg_fvars.size(); +} + +extern "C" int32_t jl_get_llvm_gv(jl_value_t *p) +{ + // map a jl_value_t memory location to a GlobalVariable + std::map::iterator it; + it = jl_value_to_llvm.find(p); + if (it == jl_value_to_llvm.end()) + return 0; + return it->second.index; +} diff --git a/src/julia_internal.h b/src/julia_internal.h index 4a50cf2e682e6..72edff4c96b54 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -56,13 +56,13 @@ STATIC_INLINE jl_value_t *newstruct(jl_datatype_t *type) } void jl_generate_fptr(jl_lambda_info_t *li); -void jl_compile_linfo(jl_lambda_info_t *li, void *cyclectx); +void jl_compile_linfo(jl_lambda_info_t *li); // invoke (compiling if necessary) the jlcall function pointer for a method STATIC_INLINE jl_value_t *jl_call_method_internal(jl_lambda_info_t *meth, jl_value_t **args, uint32_t nargs) { if (__unlikely(meth->fptr == NULL)) { - jl_compile_linfo(meth, NULL); + jl_compile_linfo(meth); jl_generate_fptr(meth); } if (meth->jlcall_api == 0) @@ -263,7 +263,7 @@ int32_t jl_get_llvm_gv(jl_value_t *p); void jl_idtable_rehash(jl_array_t **pa, size_t newsz); JL_DLLEXPORT jl_methtable_t *jl_new_method_table(jl_sym_t *name, jl_module_t *module); -jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types, void *cyclectx); +jl_lambda_info_t *jl_get_specialization1(jl_tupletype_t *types); jl_function_t *jl_module_get_initializer(jl_module_t *m); uint32_t jl_module_next_counter(jl_module_t *m); void jl_fptr_to_llvm(jl_fptr_t fptr, jl_lambda_info_t *lam, int specsig); diff --git a/src/toplevel.c b/src/toplevel.c index 1eb3fe14742ac..44a010370fc5b 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -81,7 +81,7 @@ static void jl_module_load_time_initialize(jl_module_t *m) jl_value_t *tt = jl_is_type(f) ? (jl_value_t*)jl_wrap_Type(f) : jl_typeof(f); JL_GC_PUSH1(&tt); tt = (jl_value_t*)jl_apply_tuple_type_v(&tt, 1); - jl_get_specialization1((jl_tupletype_t*)tt, NULL); + jl_get_specialization1((jl_tupletype_t*)tt); JL_GC_POP(); } } diff --git a/test/llvmcall.jl b/test/llvmcall.jl index 90e5ccc6d3d29..2c58958d01237 100644 --- a/test/llvmcall.jl +++ b/test/llvmcall.jl @@ -70,19 +70,12 @@ end Int32(1), Int32(2))) == 3 # Test whether declarations work properly -# This test only work properly for llvm 3.5+ -# On llvm <3.5+ even though the the compilation fails on the first try, -# llvm still adds the intrinsice declaration to the module and subsequent calls -# are succesfull. -#if convert(VersionNumber, Base.libllvm_version) > v"3.5-" -# -#function undeclared_ceil(x::Float64) -# llvmcall("""%2 = call double @llvm.ceil.f64(double %0) -# ret double %2""", Float64, Tuple{Float64}, x) -#end -#@test_throws ErrorException undeclared_ceil(4.2) -# -#end +function undeclared_ceil(x::Float64) + llvmcall("""%2 = call double @llvm.ceil.f64(double %0) + ret double %2""", Float64, Tuple{Float64}, x) +end +@test_throws ErrorException undeclared_ceil(4.2) +@test_throws ErrorException undeclared_ceil(4.2) function declared_floor(x::Float64) llvmcall(