diff --git a/Compiler/src/stmtinfo.jl b/Compiler/src/stmtinfo.jl index 4cbd2ab39fd465..9e468d45aac911 100644 --- a/Compiler/src/stmtinfo.jl +++ b/Compiler/src/stmtinfo.jl @@ -70,7 +70,7 @@ function _add_edges_impl(edges::Vector{Any}, info::MethodMatchInfo, mi_edge::Boo mi = specialize_method(m) # don't allow `Method`-edge for this optimized format edge = mi else - mi = edge.def + mi = edge.def::MethodInstance end if mi.specTypes === m.spec_types add_one_edge!(edges, edge) @@ -100,7 +100,7 @@ end function add_one_edge!(edges::Vector{Any}, edge::MethodInstance) for i in 1:length(edges) edgeᵢ = edges[i] - edgeᵢ isa CodeInstance && (edgeᵢ = edgeᵢ.def) + edgeᵢ isa CodeInstance && (edgeᵢ = edgeᵢ.def::MethodInstance) edgeᵢ isa MethodInstance || continue if edgeᵢ === edge && !(i > 1 && edges[i-1] isa Type) return # found existing covered edge @@ -112,7 +112,7 @@ end function add_one_edge!(edges::Vector{Any}, edge::CodeInstance) for i in 1:length(edges) edgeᵢ_orig = edgeᵢ = edges[i] - edgeᵢ isa CodeInstance && (edgeᵢ = edgeᵢ.def) + edgeᵢ isa CodeInstance && (edgeᵢ = edgeᵢ.def::MethodInstance) edgeᵢ isa MethodInstance || continue if edgeᵢ === edge.def && !(i > 1 && edges[i-1] isa Type) if edgeᵢ_orig isa MethodInstance diff --git a/base/boot.jl b/base/boot.jl index 0df0cde64f8c03..929623270c4a16 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -446,6 +446,12 @@ struct InitError <: WrappedException error end +struct ABIOverwrite + abi::Type + def::MethodInstance + ABIOverwrite(@nospecialize(abi::Type), def::MethodInstance) = new(abi, def) +end + struct PrecompilableError <: Exception end String(s::String) = s # no constructor yet diff --git a/base/show.jl b/base/show.jl index e332cf521addbd..05df37ef09d97e 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1353,7 +1353,13 @@ end show(io::IO, mi::Core.MethodInstance) = show_mi(io, mi) function show(io::IO, codeinst::Core.CodeInstance) print(io, "CodeInstance for ") - show_mi(io, codeinst.def) + def = codeinst.def + if isa(def, Core.ABIOverwrite) + show_mi(io, def.def) + print(io, " (ABI Overriden)") + else + show_mi(io, def::MethodInstance) + end end function show_mi(io::IO, mi::Core.MethodInstance, from_stackframe::Bool=false) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index ff14901c2e47fa..b338198a76fd13 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -101,7 +101,7 @@ void jl_get_llvm_mis_impl(void *native_code, arraylist_t* MIs) jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; auto map = data->jl_fvar_map; for (auto &ci : map) { - jl_method_instance_t *mi = ci.first->def; + jl_method_instance_t *mi = jl_get_ci_mi(ci.first); arraylist_push(MIs, mi); } } @@ -344,11 +344,11 @@ static void compile_workqueue(jl_codegen_params_t ¶ms, CompilationPolicy pol if ((policy != CompilationPolicy::Default || params.params->trim) && jl_atomic_load_relaxed(&codeinst->inferred) == jl_nothing) { // XXX: SOURCE_MODE_FORCE_SOURCE is wrong here (neither sufficient nor necessary) - codeinst = jl_type_infer(codeinst->def, jl_atomic_load_relaxed(&codeinst->max_world), SOURCE_MODE_FORCE_SOURCE); + codeinst = jl_type_infer(jl_get_ci_mi(codeinst), jl_atomic_load_relaxed(&codeinst->max_world), SOURCE_MODE_FORCE_SOURCE); } if (codeinst) { orc::ThreadSafeModule result_m = - jl_create_ts_module(name_from_method_instance(codeinst->def), + jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), params.tsctx, params.DL, params.TargetTriple); auto decls = jl_emit_codeinst(result_m, codeinst, NULL, params); if (result_m) @@ -389,7 +389,7 @@ static void compile_workqueue(jl_codegen_params_t ¶ms, CompilationPolicy pol proto.decl->setLinkage(GlobalVariable::InternalLinkage); //protodecl->setAlwaysInline(); jl_init_function(proto.decl, params.TargetTriple); - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; // TODO: maybe this can be cached in codeinst->specfptr? @@ -527,7 +527,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm } else { JL_GC_PROMISE_ROOTED(codeinst->rettype); - orc::ThreadSafeModule result_m = jl_create_ts_module(name_from_method_instance(codeinst->def), + orc::ThreadSafeModule result_m = jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), params.tsctx, clone.getModuleUnlocked()->getDataLayout(), Triple(clone.getModuleUnlocked()->getTargetTriple())); jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, NULL, params); @@ -2158,7 +2158,7 @@ void jl_get_llvmf_defn_impl(jl_llvmf_dump_t* dump, jl_method_instance_t *mi, jl_ // This would also be nice, but it seems to cause OOMs on the windows32 builder // To get correct names in the IR this needs to be at least 2 output.debug_level = params.debug_info_level; - auto decls = jl_emit_code(m, mi, src, output); + auto decls = jl_emit_code(m, mi, src, NULL, output); Function *F = NULL; if (m) { diff --git a/src/codegen.cpp b/src/codegen.cpp index 8662016fd069f2..83817887346beb 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3046,9 +3046,8 @@ static bool uses_specsig(jl_value_t *sig, bool needsparams, jl_value_t *rettype, return false; // jlcall sig won't require any box allocations } -static std::pair uses_specsig(jl_method_instance_t *lam, jl_value_t *rettype, bool prefer_specsig) +static std::pair uses_specsig(jl_value_t *abi, jl_method_instance_t *lam, jl_value_t *rettype, bool prefer_specsig) { - jl_value_t *sig = lam->specTypes; bool needsparams = false; if (jl_is_method(lam->def.method)) { if ((size_t)jl_subtype_env_size(lam->def.method->sig) != jl_svec_len(lam->sparam_vals)) @@ -3058,7 +3057,7 @@ static std::pair uses_specsig(jl_method_instance_t *lam, jl_value_t needsparams = true; } } - return std::make_pair(uses_specsig(sig, needsparams, rettype, prefer_specsig), needsparams); + return std::make_pair(uses_specsig(abi, needsparams, rettype, prefer_specsig), needsparams); } @@ -4373,6 +4372,7 @@ static jl_llvm_functions_t orc::ThreadSafeModule &TSM, jl_method_instance_t *lam, jl_code_info_t *src, + jl_value_t *abi, jl_value_t *rettype, jl_codegen_params_t ¶ms); @@ -5490,6 +5490,13 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt) return emit_invoke(ctx, lival, argv, nargs, rt); } +static jl_value_t *get_ci_abi(jl_code_instance_t *ci) +{ + if (jl_typeof(ci->def) == (jl_value_t*)jl_abioverwrite_type) + return ((jl_abi_overwrite_t*)ci->def)->abi; + return jl_get_ci_mi(ci)->specTypes; +} + static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayRef argv, size_t nargs, jl_value_t *rt) { ++EmittedInvokes; @@ -5525,7 +5532,7 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR } else if (invoke != jl_fptr_sparam_addr) { bool specsig, needsparams; - std::tie(specsig, needsparams) = uses_specsig(mi, codeinst->rettype, ctx.params->prefer_specsig); + std::tie(specsig, needsparams) = uses_specsig(get_ci_abi(codeinst), mi, codeinst->rettype, ctx.params->prefer_specsig); std::string name; StringRef protoname; bool need_to_emit = true; @@ -7200,7 +7207,7 @@ Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, M jl_init_function(f, params.TargetTriple); if (trim_may_error(params.params->trim)) { // TODO: Debuginfo! - push_frames(ctx, ctx.linfo, codeinst->def, 1); + push_frames(ctx, ctx.linfo, jl_get_ci_mi(codeinst), 1); } jl_name_jlfunc_args(params, f); //f->setAlwaysInline(); @@ -7217,7 +7224,7 @@ Function *emit_tojlinvoke(jl_code_instance_t *codeinst, StringRef theFptrName, M } else { theFunc = prepare_call(jlinvoke_func); - theFarg = literal_pointer_val(ctx, (jl_value_t*)codeinst->def); + theFarg = literal_pointer_val(ctx, (jl_value_t*)jl_get_ci_mi(codeinst)); } theFarg = track_pjlvalue(ctx, theFarg); auto args = f->arg_begin(); @@ -8327,13 +8334,13 @@ get_specsig_di(jl_codectx_t &ctx, jl_debugcache_t &debuginfo, jl_value_t *rt, jl } /* aka Core.Compiler.tuple_tfunc */ -static jl_datatype_t *compute_va_type(jl_method_instance_t *lam, size_t nreq) +static jl_datatype_t *compute_va_type(jl_value_t *sig, size_t nreq) { - size_t nvargs = jl_nparams(lam->specTypes)-nreq; + size_t nvargs = jl_nparams(sig)-nreq; jl_svec_t *tupargs = jl_alloc_svec(nvargs); JL_GC_PUSH1(&tupargs); - for (size_t i = nreq; i < jl_nparams(lam->specTypes); ++i) { - jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); + for (size_t i = nreq; i < jl_nparams(sig); ++i) { + jl_value_t *argType = jl_nth_slot_type(sig, i); // n.b. specTypes is required to be a datatype by construction for specsig if (is_uniquerep_Type(argType)) argType = jl_typeof(jl_tparam0(argType)); @@ -8373,6 +8380,7 @@ static jl_llvm_functions_t orc::ThreadSafeModule &TSM, jl_method_instance_t *lam, jl_code_info_t *src, + jl_value_t *abi, jl_value_t *jlrettype, jl_codegen_params_t ¶ms) { @@ -8453,7 +8461,7 @@ static jl_llvm_functions_t int n_ssavalues = jl_is_long(src->ssavaluetypes) ? jl_unbox_long(src->ssavaluetypes) : jl_array_nrows(src->ssavaluetypes); size_t vinfoslen = jl_array_dim0(src->slotflags); ctx.slots.resize(vinfoslen, jl_varinfo_t(ctx.builder.getContext())); - assert(lam->specTypes); // the specTypes field should always be assigned + assert(abi); // the specTypes field should always be assigned // create SAvalue locations for SSAValue objects @@ -8462,7 +8470,7 @@ static jl_llvm_functions_t ctx.ssavalue_usecount.assign(n_ssavalues, 0); bool specsig, needsparams; - std::tie(specsig, needsparams) = uses_specsig(lam, jlrettype, params.params->prefer_specsig); + std::tie(specsig, needsparams) = uses_specsig(abi, lam, jlrettype, params.params->prefer_specsig); // step 3. some variable analysis size_t i; @@ -8472,7 +8480,7 @@ static jl_llvm_functions_t jl_sym_t *argname = slot_symbol(ctx, i); if (argname == jl_unused_sym) continue; - jl_value_t *ty = jl_nth_slot_type(lam->specTypes, i); + jl_value_t *ty = jl_nth_slot_type(abi, i); // TODO: jl_nth_slot_type should call jl_rewrap_unionall // specTypes is required to be a datatype by construction for specsig, but maybe not otherwise // OpaqueClosure implicitly loads the env @@ -8490,7 +8498,7 @@ static jl_llvm_functions_t if (va && ctx.vaSlot != -1) { jl_varinfo_t &varinfo = ctx.slots[ctx.vaSlot]; varinfo.isArgument = true; - vatyp = specsig ? compute_va_type(lam, nreq) : (jl_tuple_type); + vatyp = specsig ? compute_va_type(abi, nreq) : (jl_tuple_type); varinfo.value = mark_julia_type(ctx, (Value*)NULL, false, vatyp); } @@ -8542,7 +8550,7 @@ static jl_llvm_functions_t ArgNames[i] = name; } } - returninfo = get_specsig_function(ctx, M, NULL, declarations.specFunctionObject, lam->specTypes, + returninfo = get_specsig_function(ctx, M, NULL, declarations.specFunctionObject, abi, jlrettype, ctx.is_opaque_closure, JL_FEAT_TEST(ctx,gcstack_arg), ArgNames, nreq); f = cast(returninfo.decl.getCallee()); @@ -8576,7 +8584,7 @@ static jl_llvm_functions_t std::string wrapName; raw_string_ostream(wrapName) << "jfptr_" << ctx.name << "_" << jl_atomic_fetch_add_relaxed(&globalUniqueGeneratedNames, 1); declarations.functionObject = wrapName; - size_t nparams = jl_nparams(lam->specTypes); + size_t nparams = jl_nparams(abi); gen_invoke_wrapper(lam, jlrettype, returninfo, nparams, retarg, ctx.is_opaque_closure, declarations.functionObject, M, ctx.emission_context); // TODO: add attributes: maybe_mark_argument_dereferenceable(Arg, argType) // TODO: add attributes: dereferenceable @@ -8596,10 +8604,10 @@ static jl_llvm_functions_t declarations.functionObject = needsparams ? "jl_fptr_sparam" : "jl_fptr_args"; } - if (ctx.emission_context.debug_level >= 2 && lam->def.method && jl_is_method(lam->def.method) && lam->specTypes != (jl_value_t*)jl_emptytuple_type) { + if (ctx.emission_context.debug_level >= 2 && lam->def.method && jl_is_method(lam->def.method) && abi != (jl_value_t*)jl_emptytuple_type) { ios_t sigbuf; ios_mem(&sigbuf, 0); - jl_static_show_func_sig((JL_STREAM*) &sigbuf, (jl_value_t*)lam->specTypes); + jl_static_show_func_sig((JL_STREAM*) &sigbuf, (jl_value_t*)abi); f->addFnAttr("julia.fsig", StringRef(sigbuf.buf, sigbuf.size)); ios_close(&sigbuf); } @@ -8656,7 +8664,7 @@ static jl_llvm_functions_t else if (!specsig) subrty = debugcache.jl_di_func_sig; else - subrty = get_specsig_di(ctx, debugcache, jlrettype, lam->specTypes, dbuilder); + subrty = get_specsig_di(ctx, debugcache, jlrettype, abi, dbuilder); SP = dbuilder.createFunction(nullptr ,dbgFuncName // Name ,f->getName() // LinkageName @@ -8987,7 +8995,7 @@ static jl_llvm_functions_t nullptr, nullptr, /*isboxed*/true, AtomicOrdering::NotAtomic, false, sizeof(void*)); } else { - jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); + jl_value_t *argType = jl_nth_slot_type(abi, i); // TODO: jl_nth_slot_type should call jl_rewrap_unionall? // specTypes is required to be a datatype by construction for specsig, but maybe not otherwise bool isboxed = deserves_argbox(argType); @@ -9061,10 +9069,10 @@ static jl_llvm_functions_t assert(vi.boxroot == NULL); } else if (specsig) { - ctx.nvargs = jl_nparams(lam->specTypes) - nreq; + ctx.nvargs = jl_nparams(abi) - nreq; SmallVector vargs(ctx.nvargs); - for (size_t i = nreq; i < jl_nparams(lam->specTypes); ++i) { - jl_value_t *argType = jl_nth_slot_type(lam->specTypes, i); + for (size_t i = nreq; i < jl_nparams(abi); ++i) { + jl_value_t *argType = jl_nth_slot_type(abi, i); // n.b. specTypes is required to be a datatype by construction for specsig bool isboxed = deserves_argbox(argType); Type *llvmArgType = isboxed ? ctx.types().T_prjlvalue : julia_type_to_llvm(ctx, argType); @@ -10011,6 +10019,7 @@ jl_llvm_functions_t jl_emit_code( orc::ThreadSafeModule &m, jl_method_instance_t *li, jl_code_info_t *src, + jl_value_t *abi, jl_codegen_params_t ¶ms) { JL_TIMING(CODEGEN, CODEGEN_LLVM); @@ -10019,8 +10028,10 @@ jl_llvm_functions_t jl_emit_code( assert((params.params == &jl_default_cgparams /* fast path */ || !params.cache || compare_cgparams(params.params, &jl_default_cgparams)) && "functions compiled with custom codegen params must not be cached"); + if (!abi) + abi = li->specTypes; JL_TRY { - decls = emit_function(m, li, src, src->rettype, params); + decls = emit_function(m, li, src, abi, src->rettype, params); auto stream = *jl_ExecutionEngine->get_dump_emitted_mi_name_stream(); if (stream) { jl_printf(stream, "%s\t", decls.specFunctionObject.c_str()); @@ -10089,11 +10100,11 @@ jl_llvm_functions_t jl_emit_codeinst( jl_codegen_params_t ¶ms) { JL_TIMING(CODEGEN, CODEGEN_Codeinst); - jl_timing_show_method_instance(codeinst->def, JL_TIMING_DEFAULT_BLOCK); + jl_timing_show_method_instance(jl_get_ci_mi(codeinst), JL_TIMING_DEFAULT_BLOCK); JL_GC_PUSH1(&src); if (!src) { src = (jl_code_info_t*)jl_atomic_load_relaxed(&codeinst->inferred); - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); jl_method_t *def = mi->def.method; // Check if this is the generic method for opaque closure wrappers - // if so, this must compile specptr such that it holds the specptr -> invoke wrapper @@ -10111,7 +10122,7 @@ jl_llvm_functions_t jl_emit_codeinst( } } assert(jl_egal((jl_value_t*)jl_atomic_load_relaxed(&codeinst->debuginfo), (jl_value_t*)src->debuginfo) && "trying to generate code for a codeinst for an incompatible src"); - jl_llvm_functions_t decls = jl_emit_code(m, codeinst->def, src, params); + jl_llvm_functions_t decls = jl_emit_code(m, jl_get_ci_mi(codeinst), src, get_ci_abi(codeinst), params); const std::string &specf = decls.specFunctionObject; const std::string &f = decls.functionObject; @@ -10119,7 +10130,7 @@ jl_llvm_functions_t jl_emit_codeinst( // Prepare debug info to receive this function // record that this function name came from this linfo, // so we can build a reverse mapping for debug-info. - bool toplevel = !jl_is_method(codeinst->def->def.method); + bool toplevel = !jl_is_method(jl_get_ci_mi(codeinst)->def.method); if (!toplevel) { //Safe b/c params holds context lock const DataLayout &DL = m.getModuleUnlocked()->getDataLayout(); @@ -10135,7 +10146,7 @@ jl_llvm_functions_t jl_emit_codeinst( jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); // don't change inferred state if (inferred) { - jl_method_t *def = codeinst->def->def.method; + jl_method_t *def = jl_get_ci_mi(codeinst)->def.method; if (// keep code when keeping everything !(JL_DELETE_NON_INLINEABLE) || // aggressively keep code when debugging level >= 2 diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 31f1ba8281a89a..ce8406bf91490d 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -167,7 +167,7 @@ void jl_add_code_in_flight(StringRef name, jl_code_instance_t *codeinst, const D // Non-opaque-closure MethodInstances are considered globally rooted // through their methods, but for OC, we need to create a global root // here. - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); if (jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure) jl_as_global_root((jl_value_t*)mi, 1); getJITDebugRegistry().add_code_in_flight(name, codeinst, DL); @@ -374,7 +374,7 @@ void JITDebugInfoRegistry::registerJITObject(const object::ObjectFile &Object, jl_method_instance_t *mi = NULL; if (codeinst) { JL_GC_PROMISE_ROOTED(codeinst); - mi = codeinst->def; + mi = jl_get_ci_mi(codeinst); } jl_profile_atomic([&]() JL_NOTSAFEPOINT { if (mi) diff --git a/src/engine.cpp b/src/engine.cpp index 2b68de731c4dd1..858f37b55e85e1 100644 --- a/src/engine.cpp +++ b/src/engine.cpp @@ -139,7 +139,7 @@ void jl_engine_fulfill(jl_code_instance_t *ci, jl_code_info_t *src) { jl_task_t *ct = jl_current_task; std::unique_lock lock(engine_lock); - auto record = Reservations.find(InferKey{ci->def, ci->owner}); + auto record = Reservations.find(InferKey{jl_get_ci_mi(ci), ci->owner}); if (record == Reservations.end() || record->second.ci != ci) return; assert(jl_atomic_load_relaxed(&ct->tid) == record->second.tid); diff --git a/src/gf.c b/src/gf.c index 3ff87772a2320d..262d2f510defed 100644 --- a/src/gf.c +++ b/src/gf.c @@ -537,7 +537,7 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( jl_task_t *ct = jl_current_task; jl_code_instance_t *codeinst = (jl_code_instance_t*)jl_gc_alloc(ct->ptls, sizeof(jl_code_instance_t), jl_code_instance_type); - codeinst->def = mi; + codeinst->def = (jl_value_t*)mi; codeinst->owner = owner; jl_atomic_store_relaxed(&codeinst->edges, edges); jl_atomic_store_relaxed(&codeinst->min_world, min_world); @@ -1771,17 +1771,18 @@ static void invalidate_code_instance(jl_code_instance_t *replaced, size_t max_wo JL_GC_POP(); } //jl_static_show(JL_STDERR, (jl_value_t*)replaced->def); - if (!jl_is_method(replaced->def->def.method)) + jl_method_instance_t *replaced_mi = jl_get_ci_mi(replaced); + if (!jl_is_method(replaced_mi->def.method)) return; // shouldn't happen, but better to be safe - JL_LOCK(&replaced->def->def.method->writelock); + JL_LOCK(&replaced_mi->def.method->writelock); if (jl_atomic_load_relaxed(&replaced->max_world) == ~(size_t)0) { assert(jl_atomic_load_relaxed(&replaced->min_world) - 1 <= max_world && "attempting to set illogical world constraints (probable race condition)"); jl_atomic_store_release(&replaced->max_world, max_world); } assert(jl_atomic_load_relaxed(&replaced->max_world) <= max_world); // recurse to all backedges to update their valid range also - _invalidate_backedges(replaced->def, max_world, depth + 1); - JL_UNLOCK(&replaced->def->def.method->writelock); + _invalidate_backedges(replaced_mi, max_world, depth + 1); + JL_UNLOCK(&replaced_mi->def.method->writelock); } static void _invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_world, int depth) { @@ -2275,7 +2276,7 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) int replaced_edge; if (invokeTypes) { // n.b. normally we must have mi.specTypes <: invokeTypes <: m.sig (though it might not strictly hold), so we only need to check the other subtypes - if (jl_egal(invokeTypes, caller->def->def.method->sig)) + if (jl_egal(invokeTypes, jl_get_ci_mi(caller)->def.method->sig)) replaced_edge = 0; // if invokeTypes == m.sig, then the only way to change this invoke is to replace the method itself else replaced_edge = jl_subtype(invokeTypes, type) && is_replacing(ambig, type, m, d, n, invokeTypes, NULL, morespec); @@ -2851,7 +2852,7 @@ jl_code_instance_t *jl_compile_method_internal(jl_method_instance_t *mi, size_t jl_callptr_t ucache_invoke = jl_atomic_load_acquire(&ucache->invoke); if (ucache_invoke == NULL) { if ((!jl_is_method(def) || def->source == jl_nothing) && - !jl_cached_uninferred(jl_atomic_load_relaxed(&ucache->def->cache), world)) { + !jl_cached_uninferred(jl_atomic_load_relaxed(&jl_get_ci_mi(ucache)->cache), world)) { jl_throw(jl_new_struct(jl_missingcodeerror_type, (jl_value_t*)mi)); } jl_generate_fptr_for_unspecialized(ucache); @@ -2897,7 +2898,7 @@ jl_value_t *jl_fptr_args(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_co jl_value_t *jl_fptr_sparam(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *m) { - jl_svec_t *sparams = m->def->sparam_vals; + jl_svec_t *sparams = jl_get_ci_mi(m)->sparam_vals; assert(sparams != jl_emptysvec); jl_fptr_sparam_t invoke = jl_atomic_load_relaxed(&m->specptr.fptr3); assert(invoke && "Forgot to set specptr for jl_fptr_sparam!"); diff --git a/src/interpreter.c b/src/interpreter.c index 13dc45cf2ae6e5..efd93fd527d627 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -764,7 +764,7 @@ jl_code_info_t *jl_code_for_interpreter(jl_method_instance_t *mi, size_t world) jl_value_t *NOINLINE jl_fptr_interpret_call(jl_value_t *f, jl_value_t **args, uint32_t nargs, jl_code_instance_t *codeinst) { interpreter_state *s; - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); jl_task_t *ct = jl_current_task; size_t world = ct->world_age; jl_code_info_t *src = NULL; diff --git a/src/ircode.c b/src/ircode.c index bec8d46513eef8..598644983e59a8 100644 --- a/src/ircode.c +++ b/src/ircode.c @@ -1034,7 +1034,7 @@ JL_DLLEXPORT jl_code_info_t *jl_uncompress_ir(jl_method_t *m, jl_code_instance_t JL_UNLOCK(&m->writelock); // Might GC JL_GC_POP(); if (metadata) { - code->parent = metadata->def; + code->parent = jl_get_ci_mi(metadata->def); jl_gc_wb(code, code->parent); code->rettype = metadata->rettype; jl_gc_wb(code, code->rettype); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index c8d8356687dcfd..aee6fd61ecb2cb 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -335,7 +335,7 @@ static int jl_analyze_workqueue(jl_code_instance_t *callee, jl_codegen_params_t jl_init_function(proto.decl, params.TargetTriple); // TODO: maybe this can be cached in codeinst->specfptr? int8_t gc_state = jl_gc_unsafe_enter(ct->ptls); // codegen may contain safepoints (such as jl_subtype calls) - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); size_t nrealargs = jl_nparams(mi->specTypes); // number of actual arguments being passed bool is_opaque_closure = jl_is_method(mi->def.value) && mi->def.method->is_for_opaque_closure; emit_specsig_to_fptr1(proto.decl, proto.cc, proto.return_roots, mi->specTypes, codeinst->rettype, is_opaque_closure, nrealargs, params, pinvoke, 0, 0); @@ -525,7 +525,7 @@ static void jl_compile_codeinst_now(jl_code_instance_t *codeinst) jl_ExecutionEngine->addModule(std::move(TSM)); // may safepoint // If logging of the compilation stream is enabled, // then dump the method-instance specialization type to the stream - jl_method_instance_t *mi = codeinst->def; + jl_method_instance_t *mi = jl_get_ci_mi(codeinst); if (jl_is_method(mi->def.method)) { auto stream = *jl_ExecutionEngine->get_dump_compiles_stream(); if (stream) { @@ -646,7 +646,7 @@ static void jl_emit_codeinst_to_jit( params.imaging_mode = imaging_default(); params.debug_level = jl_options.debug_level; orc::ThreadSafeModule result_m = - jl_create_ts_module(name_from_method_instance(codeinst->def), params.tsctx, params.DL, params.TargetTriple); + jl_create_ts_module(name_from_method_instance(jl_get_ci_mi(codeinst)), params.tsctx, params.DL, params.TargetTriple); jl_llvm_functions_t decls = jl_emit_codeinst(result_m, codeinst, src, params); // contains safepoints if (!result_m) return; @@ -845,14 +845,14 @@ void jl_generate_fptr_for_unspecialized_impl(jl_code_instance_t *unspec) compiler_start_time = jl_hrtime(); jl_code_info_t *src = NULL; JL_GC_PUSH1(&src); - jl_method_t *def = unspec->def->def.method; + jl_method_t *def = jl_get_ci_mi(unspec)->def.method; if (jl_is_method(def)) { src = (jl_code_info_t*)def->source; if (src && (jl_value_t*)src != jl_nothing) src = jl_uncompress_ir(def, NULL, (jl_value_t*)src); } else { - jl_method_instance_t *mi = unspec->def; + jl_method_instance_t *mi = jl_get_ci_mi(unspec); jl_code_instance_t *uninferred = jl_cached_uninferred( jl_atomic_load_relaxed(&mi->cache), 1); assert(uninferred); @@ -2294,7 +2294,7 @@ StringRef JuliaOJIT::getFunctionAtAddress(uint64_t Addr, jl_callptr_t invoke, jl else { stream_fname << "jlsys_"; } - const char* unadorned_name = jl_symbol_name(codeinst->def->def.method->name); + const char* unadorned_name = jl_symbol_name(jl_get_ci_mi(codeinst)->def.method->name); stream_fname << unadorned_name << "_" << RLST_inc++; *fname = std::move(stream_fname.str()); // store to ReverseLocalSymbolTable addGlobalMapping(*fname, Addr); diff --git a/src/jitlayers.h b/src/jitlayers.h index ba4ac3081795e5..59c61aff3c9bfe 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -277,6 +277,7 @@ jl_llvm_functions_t jl_emit_code( orc::ThreadSafeModule &M, jl_method_instance_t *mi, jl_code_info_t *src, + jl_value_t *abi, jl_codegen_params_t ¶ms); jl_llvm_functions_t jl_emit_codeinst( diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 8711c14514145b..41e5a84773ca45 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -116,6 +116,7 @@ XX(jl_simplevector_type) \ XX(jl_slotnumber_type) \ XX(jl_ssavalue_type) \ + XX(jl_abioverwrite_type) \ XX(jl_stackovf_exception) \ XX(jl_string_type) \ XX(jl_symbol_type) \ diff --git a/src/jltypes.c b/src/jltypes.c index 6c6325d84a5ff0..3490c204dcb858 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3656,7 +3656,7 @@ void jl_init_types(void) JL_GC_DISABLED "specsigflags", "precompile", "relocatability", "invoke", "specptr"), // function object decls jl_svec(18, - jl_method_instance_type, + jl_any_type, jl_any_type, jl_any_type, jl_ulong_type, diff --git a/src/julia.h b/src/julia.h index 81e6cf42da5673..840117b36ede31 100644 --- a/src/julia.h +++ b/src/julia.h @@ -433,7 +433,7 @@ typedef struct _jl_opaque_closure_t { // This type represents an executable operation typedef struct _jl_code_instance_t { JL_DATA_TYPE - jl_method_instance_t *def; // method this is specialized from + jl_value_t *def; // MethodInstance or ABIOverwrite jl_value_t *owner; // Compiler token this belongs to, `jl_nothing` is reserved for native _Atomic(struct _jl_code_instance_t*) next; // pointer to the next cache entry @@ -486,6 +486,13 @@ typedef struct _jl_code_instance_t { } specptr; // private data for `jlcall entry point } jl_code_instance_t; +// May be used as the ->def field of a CodeInstance to override the ABI +typedef struct _jl_abi_overwrite_t { + JL_DATA_TYPE + jl_value_t *abi; + jl_method_instance_t *def; +} jl_abi_overwrite_t; + // all values are callable as Functions typedef jl_value_t jl_function_t; @@ -942,6 +949,7 @@ extern JL_DLLIMPORT jl_datatype_t *jl_fielderror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_atomicerror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_missingcodeerror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_lineinfonode_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_datatype_t *jl_abioverwrite_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_stackovf_exception JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_memory_exception JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_readonlymemory_exception JL_GLOBALLY_ROOTED; diff --git a/src/julia_internal.h b/src/julia_internal.h index a093cc5d21b14c..fe1ea62272dfb0 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -689,6 +689,13 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( uint32_t effects, jl_value_t *analysis_results, uint8_t relocatability, jl_debuginfo_t *di, jl_svec_t *edges /* , int absolute_max*/); +STATIC_INLINE jl_method_instance_t *jl_get_ci_mi(jl_code_instance_t *ci) +{ + if (jl_typeof(ci->def) == (jl_value_t*)jl_abioverwrite_type) + return ((jl_abi_overwrite_t*)ci->def)->def; + return (jl_method_instance_t*)ci->def; +} + JL_DLLEXPORT const char *jl_debuginfo_file(jl_debuginfo_t *debuginfo) JL_NOTSAFEPOINT; JL_DLLEXPORT const char *jl_debuginfo_file1(jl_debuginfo_t *debuginfo) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_module_t *jl_debuginfo_module1(jl_value_t *debuginfo_def) JL_NOTSAFEPOINT; diff --git a/src/precompile_utils.c b/src/precompile_utils.c index d008cd26a28e90..329189f227a84b 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -337,7 +337,7 @@ static void *jl_precompile_worklist(jl_array_t *worklist, jl_array_t *extext_met n = jl_array_nrows(new_ext_cis); for (i = 0; i < n; i++) { jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(new_ext_cis, i); - precompile_enq_specialization_(ci->def, m); + precompile_enq_specialization_(jl_get_ci_mi(ci), m); } } } diff --git a/src/staticdata.c b/src/staticdata.c index c2e8d7e3956eab..0ffd5383d91ec7 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -100,7 +100,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 193 +#define NUM_TAGS 194 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -215,6 +215,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_addrspace_typename); INSERT_TAG(jl_addrspacecore_type); INSERT_TAG(jl_debuginfo_type); + INSERT_TAG(jl_abioverwrite_type); // special typenames INSERT_TAG(jl_tuple_typename); @@ -903,7 +904,7 @@ static void jl_insert_into_serialization_queue(jl_serializer_state *s, jl_value_ // record_field_change((jl_value_t**)&ci->next, NULL); // Why are we checking that the method/module this originates from is in_image? // and then disconnect this CI? - if (jl_object_in_image((jl_value_t*)ci->def->def.value)) { + if (jl_object_in_image((jl_value_t*)jl_get_ci_mi(ci)->def.value)) { // TODO: if (ci in ci->defs->cache) record_field_change((jl_value_t**)&ci->next, NULL); } @@ -1781,7 +1782,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED fptr_id = JL_API_CONST; } else { - if (jl_is_method(ci->def->def.method)) { + if (jl_is_method(jl_get_ci_mi(ci)->def.method)) { builtin_id = jl_fptr_id(jl_atomic_load_relaxed(&ci->specptr.fptr)); if (builtin_id) { // found in the table of builtins assert(builtin_id >= 2); @@ -2326,7 +2327,7 @@ static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) jl_code_instance_t *codeinst = (jl_code_instance_t*)(base + offset); assert(jl_is_method(codeinst->def->def.method) && jl_atomic_load_relaxed(&codeinst->invoke) != jl_fptr_const_return); assert(specfunc ? jl_atomic_load_relaxed(&codeinst->invoke) != NULL : jl_atomic_load_relaxed(&codeinst->invoke) == NULL); - linfos[i] = codeinst->def; // now it's a MethodInstance + linfos[i] = jl_get_ci_mi(codeinst); // now it's a MethodInstance void *fptr = fvars.ptrs[i]; for (; clone_idx < fvars.nclones; clone_idx++) { uint32_t idx = fvars.clone_idxs[clone_idx] & jl_sysimg_val_mask; diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 77e66c74590863..7f2cd7110d958a 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -188,7 +188,7 @@ static int has_backedge_to_worklist(jl_method_instance_t *mi, htable_t *visited, jl_code_instance_t *be; i = get_next_edge(mi->backedges, i, NULL, &be); JL_GC_PROMISE_ROOTED(be); // get_next_edge propagates the edge for us here - int child_found = has_backedge_to_worklist(be->def, visited, stack); + int child_found = has_backedge_to_worklist(jl_get_ci_mi(be), visited, stack); if (child_found == 1 || child_found == 2) { // found what we were looking for, so terminate early found = 1; @@ -219,7 +219,7 @@ static int is_relocatable_ci(htable_t *relocatable_ext_cis, jl_code_instance_t * { if (!ci->relocatability) return 0; - jl_method_instance_t *mi = ci->def; + jl_method_instance_t *mi = jl_get_ci_mi(ci); jl_method_t *m = mi->def.method; if (!ptrhash_has(relocatable_ext_cis, ci) && jl_object_in_image((jl_value_t*)m) && (!jl_is_method(m) || jl_object_in_image((jl_value_t*)m->module))) return 0; @@ -249,7 +249,7 @@ static jl_array_t *queue_external_cis(jl_array_t *list) assert(jl_is_code_instance(ci)); if (!ci->relocatability) continue; - jl_method_instance_t *mi = ci->def; + jl_method_instance_t *mi = jl_get_ci_mi(ci); jl_method_t *m = mi->def.method; if (ci->owner == jl_nothing && jl_atomic_load_relaxed(&ci->inferred) && jl_is_method(m) && jl_object_in_image((jl_value_t*)m->module)) { int found = has_backedge_to_worklist(mi, &visited, &stack); @@ -283,7 +283,7 @@ static void jl_collect_new_roots(htable_t *relocatable_ext_cis, jl_array_t *root for (size_t i = 0; i < l; i++) { jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(new_ext_cis, i); assert(jl_is_code_instance(ci)); - jl_method_t *m = ci->def->def.method; + jl_method_t *m = jl_get_ci_mi(ci)->def.method; assert(jl_is_method(m)); ptrhash_put(&mset, (void*)m, (void*)m); ptrhash_put(relocatable_ext_cis, (void*)ci, (void*)ci); @@ -901,7 +901,7 @@ static int jl_verify_method(jl_code_instance_t *codeinst, size_t *minworld, size jl_method_instance_t *callee = (jl_method_instance_t*)jl_svecref(callees, j + 1); jl_method_t *meth; if (jl_is_code_instance(callee)) - callee = ((jl_code_instance_t*)callee)->def; + callee = jl_get_ci_mi((jl_code_instance_t*)callee); if (jl_is_method_instance(callee)) meth = callee->def.method; else { @@ -1029,7 +1029,7 @@ static void jl_insert_backedges(jl_array_t *edges, jl_array_t *ext_ci_list) for (size_t i = 0; i < nedges; i++) { jl_code_instance_t *codeinst = (jl_code_instance_t*)jl_array_ptr_ref(edges, i); jl_svec_t *callees = jl_atomic_load_relaxed(&codeinst->edges); - jl_method_instance_t *caller = codeinst->def; + jl_method_instance_t *caller = jl_get_ci_mi(codeinst); jl_verify_method_graph(codeinst, &stack, &visiting); size_t minvalid = jl_atomic_load_relaxed(&codeinst->min_world); size_t maxvalid = jl_atomic_load_relaxed(&codeinst->max_world); diff --git a/test/precompile.jl b/test/precompile.jl index 1607d4c6b502b5..90ed6a08c3e6e1 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -915,7 +915,7 @@ precompile_test_harness("code caching") do dir # external callers mods = Module[] for be in mi.backedges - push!(mods, (be.def.def::Method).module) # XXX + push!(mods, ((be.def::MethodInstance).def::Method).module) # XXX end @test MA ∈ mods @test MB ∈ mods @@ -1837,7 +1837,7 @@ precompile_test_harness("PkgCacheInspector") do load_path m = only(external_methods).func::Method @test m.name == :repl_cmd && m.nargs < 2 @test new_ext_cis === nothing || any(new_ext_cis) do ci - mi = ci.def + mi = ci.def::MethodInstance mi.specTypes == Tuple{typeof(Base.repl_cmd), Int, String} end end