From 0f0d425f51ca043503e35309b2436cc86c84d6f5 Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 12 Jan 2023 10:40:03 -0700 Subject: [PATCH 1/2] Add !noalias and !alias.scope metadata The main idea here is that the TBAA domain is ill-equipped for reasoning about regions (and, in particular, suffers total precision less when merging disparate types in a `memcpy`). Instead, `!noalias` should be used for region-based memory information and `!tbaa` should be used exclusively for layout. We use (5) regions corresponding to the top level of the TBAA tree: - gcframe - stack - data - constant - type_metadata For now, this leaves the TBAA hierarchy in tact and only adds additional `!noalias` metadata. `!tbaa` annotations should be the same as before. --- src/cgutils.cpp | 57 ++++++++++---- src/codegen.cpp | 198 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 238 insertions(+), 17 deletions(-) diff --git a/src/cgutils.cpp b/src/cgutils.cpp index e56c570c3afae..c0e9f88bd117f 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -886,8 +886,8 @@ static Value *data_pointer(jl_codectx_t &ctx, const jl_cgval_t &x) return data; } -static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, MDNode *tbaa_dst, Value *src, MDNode *tbaa_src, - uint64_t sz, unsigned align, bool is_volatile) +static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src, + jl_aliasinfo_t const &src_ai, uint64_t sz, unsigned align, bool is_volatile) { if (sz == 0) return; @@ -929,44 +929,73 @@ static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, MDNode *tbaa_dst, Va src = emit_bitcast(ctx, src, dstty); } if (directel) { - auto val = tbaa_decorate(tbaa_src, ctx.builder.CreateAlignedLoad(directel, src, Align(align), is_volatile)); - tbaa_decorate(tbaa_dst, ctx.builder.CreateAlignedStore(val, dst, Align(align), is_volatile)); + auto val = src_ai.decorateInst(ctx.builder.CreateAlignedLoad(directel, src, Align(align), is_volatile)); + dst_ai.decorateInst(ctx.builder.CreateAlignedStore(val, dst, Align(align), is_volatile)); ++SkippedMemcpys; return; } } #endif + ++EmittedMemcpys; + // the memcpy intrinsic does not allow to specify different alias tags // for the load part (x.tbaa) and the store part (ctx.tbaa().tbaa_stack). // since the tbaa lattice has to be a tree we have unfortunately // x.tbaa ∪ ctx.tbaa().tbaa_stack = tbaa_root if x.tbaa != ctx.tbaa().tbaa_stack - ++EmittedMemcpys; - ctx.builder.CreateMemCpy(dst, MaybeAlign(align), src, MaybeAlign(0), sz, is_volatile, MDNode::getMostGenericTBAA(tbaa_dst, tbaa_src)); + + // Now that we use scoped aliases to label disparate regions of memory, the TBAA + // metadata should be revisited so that it only represents memory layouts. Once + // that's done, we can expect that in most cases tbaa(src) == tbaa(dst) and the + // above problem won't be as serious. + + auto merged_ai = dst_ai.merge(src_ai); + ctx.builder.CreateMemCpy(dst, MaybeAlign(align), src, MaybeAlign(0), sz, is_volatile, + merged_ai.tbaa, merged_ai.tbaa_struct, merged_ai.scope, merged_ai.noalias); } -static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, MDNode *tbaa_dst, Value *src, MDNode *tbaa_src, - Value *sz, unsigned align, bool is_volatile) +static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src, + jl_aliasinfo_t const &src_ai, Value *sz, unsigned align, bool is_volatile) { if (auto const_sz = dyn_cast(sz)) { - emit_memcpy_llvm(ctx, dst, tbaa_dst, src, tbaa_src, const_sz->getZExtValue(), align, is_volatile); + emit_memcpy_llvm(ctx, dst, dst_ai, src, src_ai, const_sz->getZExtValue(), align, is_volatile); return; } ++EmittedMemcpys; - ctx.builder.CreateMemCpy(dst, MaybeAlign(align), src, MaybeAlign(0), sz, is_volatile, MDNode::getMostGenericTBAA(tbaa_dst, tbaa_src)); + + auto merged_ai = dst_ai.merge(src_ai); + ctx.builder.CreateMemCpy(dst, MaybeAlign(align), src, MaybeAlign(0), sz, is_volatile, + merged_ai.tbaa, merged_ai.tbaa_struct, merged_ai.scope, merged_ai.noalias); } template static void emit_memcpy(jl_codectx_t &ctx, Value *dst, MDNode *tbaa_dst, Value *src, MDNode *tbaa_src, T1 &&sz, unsigned align, bool is_volatile=false) { - emit_memcpy_llvm(ctx, dst, tbaa_dst, src, tbaa_src, sz, align, is_volatile); + emit_memcpy_llvm(ctx, dst, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src, + jl_aliasinfo_t::fromTBAA(ctx, tbaa_src), sz, align, is_volatile); } template static void emit_memcpy(jl_codectx_t &ctx, Value *dst, MDNode *tbaa_dst, const jl_cgval_t &src, T1 &&sz, unsigned align, bool is_volatile=false) { - emit_memcpy_llvm(ctx, dst, tbaa_dst, data_pointer(ctx, src), src.tbaa, sz, align, is_volatile); + emit_memcpy_llvm(ctx, dst, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), data_pointer(ctx, src), + jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), sz, align, is_volatile); +} + +template +static void emit_memcpy(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src, + jl_aliasinfo_t const &src_ai, T1 &&sz, unsigned align, bool is_volatile=false) +{ + emit_memcpy_llvm(ctx, dst, dst_ai, src, src_ai, sz, align, is_volatile); +} + +template +static void emit_memcpy(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, const jl_cgval_t &src, + T1 &&sz, unsigned align, bool is_volatile=false) +{ + auto src_ai = jl_aliasinfo_t::fromTBAA(ctx, src.tbaa); + emit_memcpy_llvm(ctx, dst, dst_ai, data_pointer(ctx, src), src_ai, sz, align, is_volatile); } static LoadInst *emit_nthptr_recast(jl_codectx_t &ctx, Value *v, Value *idx, MDNode *tbaa, Type *type) @@ -2695,7 +2724,9 @@ static Value *emit_arrayptr_internal(jl_codectx_t &ctx, const jl_cgval_t &tinfo, LoadInst *LI = ctx.builder.CreateAlignedLoad(LoadT, addr, Align(sizeof(char *))); LI->setOrdering(AtomicOrdering::NotAtomic); LI->setMetadata(LLVMContext::MD_nonnull, MDNode::get(ctx.builder.getContext(), None)); - tbaa_decorate(arraytype_constshape(tinfo.typ) ? ctx.tbaa().tbaa_const : ctx.tbaa().tbaa_arrayptr, LI); + jl_aliasinfo_t aliasinfo = jl_aliasinfo_t::fromTBAA(ctx, arraytype_constshape(tinfo.typ) ? ctx.tbaa().tbaa_const : ctx.tbaa().tbaa_arrayptr); + aliasinfo.decorateInst(LI); + return LI; } diff --git a/src/codegen.cpp b/src/codegen.cpp index 6a884c861a457..5885405179c4b 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -356,6 +356,57 @@ struct jl_tbaacache_t { } }; +struct jl_noaliascache_t { + // Each domain operates completely independently. + // "No aliasing" is inferred if it is implied by any domain. + + // memory regions domain + struct jl_regions_t { + MDNode *gcframe; // GC frame + MDNode *stack; // Stack slot + MDNode *data; // Any user data that `pointerset/ref` are allowed to alias + MDNode *type_metadata; // Non-user-accessible type metadata incl. size, union selectors, etc. + MDNode *constant; // Memory that is immutable by the time LLVM can see it + + jl_regions_t(): gcframe(nullptr), stack(nullptr), data(nullptr), type_metadata(nullptr), constant(nullptr) {} + + void initialize(llvm::LLVMContext &context) { + MDBuilder mbuilder(context); + MDNode *domain = mbuilder.createAliasScopeDomain("jnoalias"); + + this->gcframe = mbuilder.createAliasScope("jnoalias_gcframe", domain); + this->stack = mbuilder.createAliasScope("jnoalias_stack", domain); + this->data = mbuilder.createAliasScope("jnoalias_data", domain); + this->type_metadata = mbuilder.createAliasScope("jnoalias_typemd", domain); + this->constant = mbuilder.createAliasScope("jnoalias_const", domain); + } + } regions; + + // `@aliasscope` domain + struct jl_aliasscope_t { + MDNode *current; + + jl_aliasscope_t(): current(nullptr) {} + + // No init required, this->current is only used to store the currently active aliasscope + void initialize(llvm::LLVMContext &context) {} + } aliasscope; + + bool initialized; + + jl_noaliascache_t(): regions(), aliasscope(), initialized(false) {} + + void initialize(llvm::LLVMContext &context) { + if (initialized) { + assert(®ions.constant->getContext() == &context); + return; + } + initialized = true; + regions.initialize(context); + aliasscope.initialize(context); + } +}; + struct jl_debugcache_t { // Basic DITypes DIDerivedType *jl_pvalue_dillvmt; @@ -1266,6 +1317,69 @@ static bool deserves_sret(jl_value_t *dt, Type *T) return (size_t)jl_datatype_size(dt) > sizeof(void*) && !T->isFloatingPointTy() && !T->isVectorTy(); } +// Alias Analysis Info (analogous to llvm::AAMDNodes) +struct jl_aliasinfo_t { + MDNode *tbaa = nullptr; // '!tbaa': Struct-path TBAA. TBAA graph forms a tree (indexed by offset). + // Two pointers do not alias if they are not transitive parents + // (effectively, subfields) of each other or equal. + MDNode *tbaa_struct = nullptr; // '!tbaa.struct': Describes memory layout of struct. + MDNode *scope = nullptr; // '!alias.scope': Generic "noalias" memory access sets. + // If alias.scope(inst_a) ⊆ noalias(inst_b) (in any "domain") + // => inst_a, inst_b do not alias. + MDNode *noalias = nullptr; // '!noalias': See '!alias.scope' above. + + enum class Region { unknown, gcframe, stack, data, constant, type_metadata }; // See jl_regions_t + + explicit jl_aliasinfo_t() = default; + explicit jl_aliasinfo_t(jl_codectx_t &ctx, Region r, MDNode *tbaa); + explicit jl_aliasinfo_t(MDNode *tbaa, MDNode *tbaa_struct, MDNode *scope, MDNode *noalias) + : tbaa(tbaa), tbaa_struct(tbaa_struct), scope(scope), noalias(noalias) {} + jl_aliasinfo_t(const jl_aliasinfo_t &) = default; + + // Add !tbaa, !tbaa.struct, !alias.scope, !noalias annotations to an instruction. + // + // Also adds `invariant.load` to load instructions in the constant !noalias scope. + Instruction *decorateInst(Instruction *inst) const { + + if (this->tbaa) + inst->setMetadata(LLVMContext::MD_tbaa, this->tbaa); + if (this->tbaa_struct) + inst->setMetadata(LLVMContext::MD_tbaa_struct, this->tbaa_struct); + if (this->scope) + inst->setMetadata(LLVMContext::MD_alias_scope, this->scope); + if (this->noalias) + inst->setMetadata(LLVMContext::MD_noalias, this->noalias); + + if (this->scope && isa(inst)) { + // If this is in the read-only region, mark the load with "!invariant.load" + if (this->scope->getNumOperands() == 1) { + MDNode *operand = cast(this->scope->getOperand(0)); + auto scope_name = cast(operand->getOperand(0))->getString(); + if (scope_name == "jnoalias_const") + inst->setMetadata(LLVMContext::MD_invariant_load, MDNode::get(inst->getContext(), None)); + } + } + + return inst; + } + + // Merge two sets of alias information. + jl_aliasinfo_t merge(const jl_aliasinfo_t &other) const { + jl_aliasinfo_t result; + result.tbaa = MDNode::getMostGenericTBAA(this->tbaa, other.tbaa); + result.tbaa_struct = nullptr; + result.scope = MDNode::getMostGenericAliasScope(this->scope, other.scope); + result.noalias = MDNode::intersect(this->noalias, other.noalias); + return result; + } + + // Create alias information based on the provided TBAA metadata. + // + // This function only exists to help transition to using !noalias to encode + // memory region non-aliasing. It should be deleted once the TBAA metadata + // is improved to encode only memory layout and *not* memory regions. + static jl_aliasinfo_t fromTBAA(jl_codectx_t &ctx, MDNode *tbaa); +}; // metadata tracking for a llvm Value* during codegen struct jl_cgval_t { @@ -1431,6 +1545,7 @@ class jl_codectx_t { jl_module_t *module = NULL; jl_typecache_t type_cache; jl_tbaacache_t tbaa_cache; + jl_noaliascache_t aliasscope_cache; jl_method_instance_t *linfo = NULL; jl_value_t *rettype = NULL; jl_code_info_t *source = NULL; @@ -1442,7 +1557,6 @@ class jl_codectx_t { Value *spvals_ptr = NULL; Value *argArray = NULL; Value *argCount = NULL; - MDNode *aliasscope = NULL; std::string funcName; int vaSlot = -1; // name of vararg argument int nReqArgs = 0; @@ -1481,6 +1595,11 @@ class jl_codectx_t { return tbaa_cache; } + jl_noaliascache_t &noalias() { + aliasscope_cache.initialize(builder.getContext()); + return aliasscope_cache; + } + ~jl_codectx_t() { // Transfer local delayed calls to the global queue for (auto call_target : call_targets) @@ -1492,6 +1611,77 @@ GlobalVariable *JuliaVariable::realize(jl_codectx_t &ctx) { return realize(jl_Module); } +jl_aliasinfo_t::jl_aliasinfo_t(jl_codectx_t &ctx, Region r, MDNode *tbaa): tbaa(tbaa), tbaa_struct(nullptr) { + MDNode *alias_scope = nullptr; + jl_noaliascache_t::jl_regions_t regions = ctx.noalias().regions; + switch (r) { + case Region::unknown: + alias_scope = nullptr; + break; + case Region::gcframe: + alias_scope = regions.gcframe; + break; + case Region::stack: + alias_scope = regions.stack; + break; + case Region::data: + alias_scope = regions.data; + break; + case Region::constant: + alias_scope = regions.constant; + break; + case Region::type_metadata: + alias_scope = regions.type_metadata; + break; + } + + MDNode *all_scopes[5] = { regions.gcframe, regions.stack, regions.data, regions.type_metadata, regions.constant }; + if (alias_scope) { + // The matching region is added to !alias.scope + // All other regions are added to !noalias + + int i = 0; + Metadata *scopes[1] = { alias_scope }; + Metadata *noaliases[4]; + for (auto const &scope: all_scopes) { + if (scope == alias_scope) continue; + noaliases[i++] = scope; + } + + this->scope = MDNode::get(ctx.builder.getContext(), ArrayRef(scopes)); + this->noalias = MDNode::get(ctx.builder.getContext(), ArrayRef(noaliases)); + } +} + +jl_aliasinfo_t jl_aliasinfo_t::fromTBAA(jl_codectx_t &ctx, MDNode *tbaa) { + auto cache = ctx.tbaa(); + + // Each top-level TBAA node has a corresponding !alias.scope scope + MDNode *tbaa_srcs[5] = { cache.tbaa_gcframe, cache.tbaa_stack, cache.tbaa_data, cache.tbaa_array, cache.tbaa_const }; + Region regions[5] = { Region::gcframe, Region::stack, Region::data, Region::type_metadata, Region::constant }; + + if (tbaa != nullptr) { + MDNode *node = cast(tbaa->getOperand(1)); + if (cast(node->getOperand(0))->getString() != "jtbaa") { + + // Climb up to node just before root + MDNode *parent_node = cast(node->getOperand(1)); + while (cast(parent_node->getOperand(0))->getString() != "jtbaa") { + node = parent_node; + parent_node = cast(node->getOperand(1)); + } + + // Find the matching node's index + for (int i = 0; i < 5; i++) { + if (cast(tbaa_srcs[i]->getOperand(1)) == node) + return jl_aliasinfo_t(ctx, regions[i], tbaa); + } + } + } + + return jl_aliasinfo_t(ctx, Region::unknown, tbaa); +} + static Type *julia_type_to_llvm(jl_codectx_t &ctx, jl_value_t *jt, bool *isboxed = NULL); static jl_returninfo_t get_specsig_function(jl_codectx_t &ctx, Module *M, StringRef name, jl_value_t *sig, jl_value_t *jlrettype, bool is_opaque_closure); static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaval = -1); @@ -3238,7 +3428,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, *ret = emit_unionload(ctx, data, ptindex, ety, elsz, al, ctx.tbaa().tbaa_arraybuf, true, union_max, ctx.tbaa().tbaa_arrayselbyte); } else { - MDNode *aliasscope = (f == jl_builtin_const_arrayref) ? ctx.aliasscope : nullptr; + MDNode *aliasscope = (f == jl_builtin_const_arrayref) ? ctx.noalias().aliasscope.current : nullptr; *ret = typed_load(ctx, emit_arrayptr(ctx, ary, ary_ex), idx, ety, @@ -3353,7 +3543,7 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_arrayptr(ctx, ary, ary_ex, isboxed), idx, val, jl_cgval_t(), ety, isboxed ? ctx.tbaa().tbaa_ptrarraybuf : ctx.tbaa().tbaa_arraybuf, - ctx.aliasscope, + ctx.noalias().aliasscope.current, data_owner, isboxed, isboxed ? AtomicOrdering::Release : AtomicOrdering::NotAtomic, // TODO: we should do this for anything with CountTrackedPointers(elty).count > 0 @@ -7651,7 +7841,7 @@ static jl_llvm_functions_t ctx.builder.SetCurrentDebugLocation(linetable.at(debuginfoloc).loc); coverageVisitStmt(debuginfoloc); } - ctx.aliasscope = aliasscopes[cursor]; + ctx.noalias().aliasscope.current = aliasscopes[cursor]; jl_value_t *stmt = jl_array_ptr_ref(stmts, cursor); jl_expr_t *expr = jl_is_expr(stmt) ? (jl_expr_t*)stmt : nullptr; if (jl_is_returnnode(stmt)) { From d04d36ffd8ad3149739139584df9cd3a35af90de Mon Sep 17 00:00:00 2001 From: Cody Tapscott Date: Thu, 12 Jan 2023 11:00:20 -0700 Subject: [PATCH 2/2] Derive `!noalias` from `!tbaa` for most loads/stores This is an interim solution that derives the correct `!noalias` region from the existing TBAA information. Later we will want to: - Revise the TBAA hierarchy to remove region information - Delete `jl_aliasinfo_t::fromTBAA()` - Update `jl_cgval_t` to store a `jl_aliasinfo_t` --- src/ccall.cpp | 30 +++-- src/cgutils.cpp | 202 +++++++++++++++++---------------- src/codegen.cpp | 122 ++++++++++++-------- src/intrinsics.cpp | 35 +++--- test/llvmpasses/aliasscopes.jl | 27 +++-- 5 files changed, 234 insertions(+), 182 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index 498109f069cc6..1ff2ca7828b06 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -550,10 +550,12 @@ static Value *julia_to_native( // since those are immutable. Value *slot = emit_static_alloca(ctx, to); if (!jvinfo.ispointer()) { - tbaa_decorate(jvinfo.tbaa, ctx.builder.CreateStore(emit_unbox(ctx, to, jvinfo, jlto), slot)); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, jvinfo.tbaa); + ai.decorateInst(ctx.builder.CreateStore(emit_unbox(ctx, to, jvinfo, jlto), slot)); } else { - emit_memcpy(ctx, slot, jvinfo.tbaa, jvinfo, jl_datatype_size(jlto), julia_alignment(jlto)); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, jvinfo.tbaa); + emit_memcpy(ctx, slot, ai, jvinfo, jl_datatype_size(jlto), julia_alignment(jlto)); } return slot; } @@ -1571,7 +1573,8 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) const int tid_offset = offsetof(jl_task_t, tid); Value *ptid = ctx.builder.CreateInBoundsGEP(getInt16Ty(ctx.builder.getContext()), ptask_i16, ConstantInt::get(getSizeTy(ctx.builder.getContext()), tid_offset / sizeof(int16_t))); LoadInst *tid = ctx.builder.CreateAlignedLoad(getInt16Ty(ctx.builder.getContext()), ptid, Align(sizeof(int16_t))); - tbaa_decorate(ctx.tbaa().tbaa_gcframe, tid); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe); + ai.decorateInst(tid); return mark_or_box_ccall_result(ctx, tid, retboxed, rt, unionall, static_rt); } else if (is_libjulia_func(jl_gc_disable_finalizers_internal) @@ -1675,8 +1678,10 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) len = ctx.builder.CreateAlignedLoad(getSizeTy(ctx.builder.getContext()), ptr, Align(sizeof(size_t))); // Only mark with TBAA if we are sure about the type. // This could otherwise be in a dead branch - if (svecv.typ == (jl_value_t*)jl_simplevector_type) - tbaa_decorate(ctx.tbaa().tbaa_const, cast(len)); + if (svecv.typ == (jl_value_t*)jl_simplevector_type) { + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + ai.decorateInst(cast(len)); + } MDBuilder MDB(ctx.builder.getContext()); auto rng = MDB.createRange( Constant::getNullValue(getSizeTy(ctx.builder.getContext())), ConstantInt::get(getSizeTy(ctx.builder.getContext()), INTPTR_MAX / sizeof(void*) - 1)); @@ -1701,8 +1706,10 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) load->setAtomic(AtomicOrdering::Unordered); // Only mark with TBAA if we are sure about the type. // This could otherwise be in a dead branch - if (svecv.typ == (jl_value_t*)jl_simplevector_type) - tbaa_decorate(ctx.tbaa().tbaa_const, load); + if (svecv.typ == (jl_value_t*)jl_simplevector_type) { + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + ai.decorateInst(load); + } JL_GC_POP(); return mark_or_box_ccall_result(ctx, load, retboxed, rt, unionall, static_rt); } @@ -1736,7 +1743,8 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) Value *slot_addr = ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, arrayptr, idx); LoadInst *load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, slot_addr, Align(sizeof(void*))); load->setAtomic(AtomicOrdering::Unordered); - tbaa_decorate(ctx.tbaa().tbaa_ptrarraybuf, load); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_ptrarraybuf); + ai.decorateInst(load); Value *res = ctx.builder.CreateZExt(ctx.builder.CreateICmpNE(load, Constant::getNullValue(ctx.types().T_prjlvalue)), getInt32Ty(ctx.builder.getContext())); JL_GC_POP(); return mark_or_box_ccall_result(ctx, res, retboxed, rt, unionall, static_rt); @@ -1838,7 +1846,8 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) Value *ph1 = emit_bitcast(ctx, decay_derived(ctx, boxed(ctx, val)), getSizePtrTy(ctx.builder.getContext())); Value *ph2 = ctx.builder.CreateInBoundsGEP(getSizeTy(ctx.builder.getContext()), ph1, ConstantInt::get(getSizeTy(ctx.builder.getContext()), hash_offset / sizeof(size_t))); LoadInst *hashval = ctx.builder.CreateAlignedLoad(getSizeTy(ctx.builder.getContext()), ph2, Align(sizeof(size_t))); - tbaa_decorate(ctx.tbaa().tbaa_const, hashval); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + ai.decorateInst(hashval); return mark_or_box_ccall_result(ctx, hashval, retboxed, rt, unionall, static_rt); } else if (!val.isboxed) { @@ -2128,7 +2137,8 @@ jl_cgval_t function_sig_t::emit_a_ccall( auto slot = emit_static_alloca(ctx, resultTy); slot->setAlignment(Align(boxalign)); ctx.builder.CreateAlignedStore(result, slot, Align(boxalign)); - emit_memcpy(ctx, strct, tbaa, slot, tbaa, rtsz, boxalign); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + emit_memcpy(ctx, strct, ai, slot, ai, rtsz, boxalign); } else { init_bits_value(ctx, strct, result, tbaa, boxalign); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index c0e9f88bd117f..8188c741c1dc4 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -474,7 +474,8 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p) if (!ctx.emission_context.imaging) return literal_static_pointer_val(p, ctx.types().T_pjlvalue); Value *pgv = literal_pointer_val_slot(ctx, p); - return tbaa_decorate(ctx.tbaa().tbaa_const, maybe_mark_load_dereferenceable( + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + return ai.decorateInst(maybe_mark_load_dereferenceable( ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, pgv, Align(sizeof(void*))), false, jl_typeof(p))); } @@ -489,7 +490,8 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_binding_t *p) return literal_static_pointer_val(p, ctx.types().T_pjlvalue); // bindings are prefixed with jl_bnd# Value *pgv = julia_pgv(ctx, "jl_bnd#", p->name, p->owner, p); - return tbaa_decorate(ctx.tbaa().tbaa_const, maybe_mark_load_dereferenceable( + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + return ai.decorateInst(maybe_mark_load_dereferenceable( ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, pgv, Align(sizeof(void*))), false, sizeof(jl_binding_t), alignof(jl_binding_t))); } @@ -526,9 +528,11 @@ static Value *julia_binding_gv(jl_codectx_t &ctx, jl_binding_t *b) { // emit a literal_pointer_val to a jl_binding_t // binding->value are prefixed with * - if (ctx.emission_context.imaging) - return tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, + if (ctx.emission_context.imaging) { + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + return ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, julia_pgv(ctx, "*", b->name, b->owner, b), Align(sizeof(void*)))); + } else return literal_static_pointer_val(b, ctx.types().T_pjlvalue); } @@ -967,22 +971,6 @@ static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const merged_ai.tbaa, merged_ai.tbaa_struct, merged_ai.scope, merged_ai.noalias); } -template -static void emit_memcpy(jl_codectx_t &ctx, Value *dst, MDNode *tbaa_dst, Value *src, MDNode *tbaa_src, - T1 &&sz, unsigned align, bool is_volatile=false) -{ - emit_memcpy_llvm(ctx, dst, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src, - jl_aliasinfo_t::fromTBAA(ctx, tbaa_src), sz, align, is_volatile); -} - -template -static void emit_memcpy(jl_codectx_t &ctx, Value *dst, MDNode *tbaa_dst, const jl_cgval_t &src, - T1 &&sz, unsigned align, bool is_volatile=false) -{ - emit_memcpy_llvm(ctx, dst, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), data_pointer(ctx, src), - jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), sz, align, is_volatile); -} - template static void emit_memcpy(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src, jl_aliasinfo_t const &src_ai, T1 &&sz, unsigned align, bool is_volatile=false) @@ -1006,7 +994,8 @@ static LoadInst *emit_nthptr_recast(jl_codectx_t &ctx, Value *v, Value *idx, MDN emit_bitcast(ctx, maybe_decay_tracked(ctx, v), ctx.types().T_pprjlvalue), idx); LoadInst *load = ctx.builder.CreateLoad(type, emit_bitcast(ctx, vptr, PointerType::get(type, 0))); - tbaa_decorate(tbaa, load); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.decorateInst(load); return load; } @@ -1049,9 +1038,10 @@ static jl_cgval_t emit_typeof(jl_codectx_t &ctx, const jl_cgval_t &p, bool maybe p.typ, counter); auto emit_unboxty = [&] () -> Value* { + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); if (ctx.emission_context.imaging) return track_pjlvalue( - ctx, tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, datatype_or_p, Align(sizeof(void*))))); + ctx, ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, datatype_or_p, Align(sizeof(void*))))); return datatype_or_p; }; Value *res; @@ -1093,25 +1083,28 @@ static Value *emit_datatype_types(jl_codectx_t &ctx, Value *dt) { Value *Ptr = emit_bitcast(ctx, decay_derived(ctx, dt), ctx.types().T_ppjlvalue); Value *Idx = ConstantInt::get(getSizeTy(ctx.builder.getContext()), offsetof(jl_datatype_t, types) / sizeof(void*)); - return tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad( + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + return ai.decorateInst(ctx.builder.CreateAlignedLoad( ctx.types().T_pjlvalue, ctx.builder.CreateInBoundsGEP(ctx.types().T_pjlvalue, Ptr, Idx), Align(sizeof(void*)))); } static Value *emit_datatype_nfields(jl_codectx_t &ctx, Value *dt) { Value *type_svec = emit_bitcast(ctx, emit_datatype_types(ctx, dt), getSizePtrTy(ctx.builder.getContext())); - return tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(getSizeTy(ctx.builder.getContext()), type_svec, Align(sizeof(void*)))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + return ai.decorateInst(ctx.builder.CreateAlignedLoad(getSizeTy(ctx.builder.getContext()), type_svec, Align(sizeof(void*)))); } static Value *emit_datatype_size(jl_codectx_t &ctx, Value *dt) { + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); Value *Ptr = emit_bitcast(ctx, decay_derived(ctx, dt), getInt32PtrTy(ctx.builder.getContext())->getPointerTo()); Value *Idx = ConstantInt::get(getSizeTy(ctx.builder.getContext()), offsetof(jl_datatype_t, layout) / sizeof(int32_t*)); Ptr = ctx.builder.CreateInBoundsGEP(getInt32PtrTy(ctx.builder.getContext()), Ptr, Idx); - Ptr = tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(getInt32PtrTy(ctx.builder.getContext()), Ptr, Align(sizeof(int32_t*)))); + Ptr = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt32PtrTy(ctx.builder.getContext()), Ptr, Align(sizeof(int32_t*)))); Idx = ConstantInt::get(getSizeTy(ctx.builder.getContext()), offsetof(jl_datatype_layout_t, size) / sizeof(int32_t)); Ptr = ctx.builder.CreateInBoundsGEP(getInt32Ty(ctx.builder.getContext()), Ptr, Idx); - return tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(getInt32Ty(ctx.builder.getContext()), Ptr, Align(sizeof(int32_t)))); + return ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt32Ty(ctx.builder.getContext()), Ptr, Align(sizeof(int32_t)))); } /* this is valid code, it's simply unused @@ -1165,12 +1158,13 @@ static Value *emit_sizeof(jl_codectx_t &ctx, const jl_cgval_t &p) static Value *emit_datatype_mutabl(jl_codectx_t &ctx, Value *dt) { + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); Value *Ptr = emit_bitcast(ctx, decay_derived(ctx, dt), ctx.types().T_ppint8); Value *Idx = ConstantInt::get(getSizeTy(ctx.builder.getContext()), offsetof(jl_datatype_t, name)); - Value *Nam = tbaa_decorate(ctx.tbaa().tbaa_const, + Value *Nam = ai.decorateInst( ctx.builder.CreateAlignedLoad(getInt8PtrTy(ctx.builder.getContext()), ctx.builder.CreateInBoundsGEP(getInt8PtrTy(ctx.builder.getContext()), Ptr, Idx), Align(sizeof(int8_t*)))); Value *Idx2 = ConstantInt::get(getSizeTy(ctx.builder.getContext()), offsetof(jl_typename_t, n_uninitialized) + sizeof(((jl_typename_t*)nullptr)->n_uninitialized)); - Value *mutabl = tbaa_decorate(ctx.tbaa().tbaa_const, + Value *mutabl = ai.decorateInst( ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), Nam, Idx2), Align(1))); mutabl = ctx.builder.CreateLShr(mutabl, 1); return ctx.builder.CreateTrunc(mutabl, getInt1Ty(ctx.builder.getContext())); @@ -1181,7 +1175,8 @@ static Value *emit_datatype_isprimitivetype(jl_codectx_t &ctx, Value *typ) { Value *isprimitive; isprimitive = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), emit_bitcast(ctx, decay_derived(ctx, typ), getInt8PtrTy(ctx.builder.getContext())), offsetof(jl_datatype_t, hash) + sizeof(((jl_datatype_t*)nullptr)->hash)); - isprimitive = tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), isprimitive, Align(1))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + isprimitive = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), isprimitive, Align(1))); isprimitive = ctx.builder.CreateLShr(isprimitive, 7); isprimitive = ctx.builder.CreateTrunc(isprimitive, getInt1Ty(ctx.builder.getContext())); return isprimitive; @@ -1194,7 +1189,8 @@ static Value *emit_datatype_name(jl_codectx_t &ctx, Value *dt) ctx.types().T_pjlvalue, emit_bitcast(ctx, maybe_decay_tracked(ctx, dt), ctx.types().T_ppjlvalue), ConstantInt::get(getSizeTy(ctx.builder.getContext()), n)); - return tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, vptr, Align(sizeof(void*)))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + return ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, vptr, Align(sizeof(void*)))); } // --- generating various error checks --- @@ -1596,7 +1592,8 @@ static Value *emit_isconcrete(jl_codectx_t &ctx, Value *typ) { Value *isconcrete; isconcrete = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), emit_bitcast(ctx, decay_derived(ctx, typ), getInt8PtrTy(ctx.builder.getContext())), offsetof(jl_datatype_t, hash) + sizeof(((jl_datatype_t*)nullptr)->hash)); - isconcrete = tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), isconcrete, Align(1))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + isconcrete = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), isconcrete, Align(1))); isconcrete = ctx.builder.CreateLShr(isconcrete, 1); isconcrete = ctx.builder.CreateTrunc(isconcrete, getInt1Ty(ctx.builder.getContext())); return isconcrete; @@ -1774,17 +1771,16 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j else if (!alignment) alignment = julia_alignment(jltype); if (intcast && Order == AtomicOrdering::NotAtomic) { - emit_memcpy(ctx, intcast, ctx.tbaa().tbaa_stack, data, tbaa, nb, alignment); + emit_memcpy(ctx, intcast, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), data, jl_aliasinfo_t::fromTBAA(ctx, tbaa), nb, alignment); } else { LoadInst *load = ctx.builder.CreateAlignedLoad(elty, data, Align(alignment), false); load->setOrdering(Order); - if (aliasscope) - load->setMetadata("alias.scope", aliasscope); if (isboxed) maybe_mark_load_dereferenceable(load, true, jltype); - if (tbaa) - tbaa_decorate(tbaa, load); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.scope = MDNode::concatenate(aliasscope, ai.scope); + ai.decorateInst(load); instr = load; if (elty != realelty) instr = ctx.builder.CreateTrunc(instr, realelty); @@ -1908,20 +1904,18 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, auto *load = ctx.builder.CreateAlignedLoad(elty, ptr, Align(alignment)); if (isboxed) load->setOrdering(AtomicOrdering::Unordered); - if (aliasscope) - load->setMetadata("noalias", aliasscope); - if (tbaa) - tbaa_decorate(tbaa, load); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); + ai.decorateInst(load); assert(realelty == elty); instr = load; } if (r) { StoreInst *store = ctx.builder.CreateAlignedStore(r, ptr, Align(alignment)); store->setOrdering(Order == AtomicOrdering::NotAtomic && isboxed ? AtomicOrdering::Release : Order); - if (aliasscope) - store->setMetadata("noalias", aliasscope); - if (tbaa) - tbaa_decorate(tbaa, store); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); + ai.decorateInst(store); } else { assert(Order == AtomicOrdering::NotAtomic && !isboxed && rhs.typ == jltype); @@ -1938,10 +1932,9 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, auto *store = ctx.builder.CreateAtomicRMW(AtomicRMWInst::Xchg, ptr, r, Order); store->setAlignment(Align(alignment)); #endif - if (aliasscope) - store->setMetadata("noalias", aliasscope); - if (tbaa) - tbaa_decorate(tbaa, store); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); + ai.decorateInst(store); instr = store; } else { @@ -1964,11 +1957,9 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ctx.builder.SetInsertPoint(SkipBB); LoadInst *load = ctx.builder.CreateAlignedLoad(elty, ptr, Align(alignment)); load->setOrdering(FailOrder == AtomicOrdering::NotAtomic && isboxed ? AtomicOrdering::Monotonic : FailOrder); - if (aliasscope) - load->setMetadata("noalias", aliasscope); - if (tbaa) - tbaa_decorate(tbaa, load); - instr = load; + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); + instr = ai.decorateInst(load); ctx.builder.CreateBr(DoneBB); ctx.builder.SetInsertPoint(DoneBB); Succ = ctx.builder.CreatePHI(getInt1Ty(ctx.builder.getContext()), 2); @@ -1995,11 +1986,9 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, else { // swap or modify LoadInst *Current = ctx.builder.CreateAlignedLoad(elty, ptr, Align(alignment)); Current->setOrdering(Order == AtomicOrdering::NotAtomic && !isboxed ? Order : AtomicOrdering::Monotonic); - if (aliasscope) - Current->setMetadata("noalias", aliasscope); - if (tbaa) - tbaa_decorate(tbaa, Current); - Compare = Current; + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); + Compare = ai.decorateInst(Current); needloop = !isswapfield || Order != AtomicOrdering::NotAtomic; } BasicBlock *BB = NULL; @@ -2051,12 +2040,11 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, // modifyfield or replacefield assert(elty == realelty && !intcast); auto *load = ctx.builder.CreateAlignedLoad(elty, ptr, Align(alignment)); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); + ai.decorateInst(load); if (isboxed) load->setOrdering(AtomicOrdering::Monotonic); - if (aliasscope) - load->setMetadata("noalias", aliasscope); - if (tbaa) - tbaa_decorate(tbaa, load); Value *first_ptr = nullptr; if (maybe_null_if_boxed && !ismodifyfield) first_ptr = isboxed ? load : extract_first_ptr(ctx, load); @@ -2072,10 +2060,9 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ctx.builder.SetInsertPoint(XchgBB); if (r) { auto *store = ctx.builder.CreateAlignedStore(r, ptr, Align(alignment)); - if (aliasscope) - store->setMetadata("noalias", aliasscope); - if (tbaa) - tbaa_decorate(tbaa, store); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); + ai.decorateInst(store); } else { assert(!isboxed && rhs.typ == jltype); @@ -2100,10 +2087,9 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, auto *store = ctx.builder.CreateAtomicCmpXchg(ptr, Compare, r, Order, FailOrder); store->setAlignment(Align(alignment)); #endif - if (aliasscope) - store->setMetadata("noalias", aliasscope); - if (tbaa) - tbaa_decorate(tbaa, store); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.noalias = MDNode::concatenate(aliasscope, ai.noalias); + ai.decorateInst(store); instr = ctx.builder.Insert(ExtractValueInst::Create(store, 0)); Success = ctx.builder.Insert(ExtractValueInst::Create(store, 1)); Done = Success; @@ -2329,7 +2315,8 @@ static bool emit_getfield_unknownidx(jl_codectx_t &ctx, idx0()); LoadInst *fld = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, fldptr, Align(sizeof(void*))); fld->setOrdering(AtomicOrdering::Unordered); - tbaa_decorate(strct.tbaa, fld); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, strct.tbaa); + ai.decorateInst(fld); maybe_mark_load_dereferenceable(fld, maybe_null, minimum_field_size, minimum_align); if (maybe_null) null_pointer_check(ctx, fld); @@ -2368,7 +2355,8 @@ static jl_cgval_t emit_unionload(jl_codectx_t &ctx, Value *addr, Value *ptindex, unsigned union_max, MDNode *tbaa_ptindex) { ++EmittedUnionLoads; - Instruction *tindex0 = tbaa_decorate(tbaa_ptindex, ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), ptindex, Align(1))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa_ptindex); + Instruction *tindex0 = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), ptindex, Align(1))); tindex0->setMetadata(LLVMContext::MD_range, MDNode::get(ctx.builder.getContext(), { ConstantAsMetadata::get(ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0)), ConstantAsMetadata::get(ConstantInt::get(getInt8Ty(ctx.builder.getContext()), union_max)) })); @@ -2379,7 +2367,8 @@ static jl_cgval_t emit_unionload(jl_codectx_t &ctx, Value *addr, Value *ptindex, AllocaInst *lv = emit_static_alloca(ctx, AT); if (al > 1) lv->setAlignment(Align(al)); - emit_memcpy(ctx, lv, tbaa, addr, tbaa, fsz, al); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + emit_memcpy(ctx, lv, ai, addr, ai, fsz, al); addr = lv; } return mark_julia_slot(fsz > 0 ? addr : nullptr, jfty, tindex, tbaa); @@ -2449,7 +2438,8 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st LoadInst *Load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, maybe_bitcast(ctx, addr, ctx.types().T_pprjlvalue), Align(sizeof(void*))); Load->setOrdering(order <= jl_memory_order_notatomic ? AtomicOrdering::Unordered : get_llvm_atomic_order(order)); maybe_mark_load_dereferenceable(Load, maybe_null, jl_field_type(jt, idx)); - Value *fldv = tbaa_decorate(tbaa, Load); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + Value *fldv = ai.decorateInst(Load); if (maybe_null) null_pointer_check(ctx, fldv, nullcheck); return mark_julia_type(ctx, fldv, true, jfty); @@ -2691,7 +2681,8 @@ static Value *emit_arraylen_prim(jl_codectx_t &ctx, const jl_cgval_t &tinfo) MDBuilder MDB(ctx.builder.getContext()); auto rng = MDB.createRange(Constant::getNullValue(getSizeTy(ctx.builder.getContext())), ConstantInt::get(getSizeTy(ctx.builder.getContext()), arraytype_maxsize(tinfo.typ))); len->setMetadata(LLVMContext::MD_range, rng); - return tbaa_decorate(tbaa, len); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + return ai.decorateInst(len); } static Value *emit_arraylen(jl_codectx_t &ctx, const jl_cgval_t &tinfo) @@ -2762,7 +2753,8 @@ static Value *emit_arrayflags(jl_codectx_t &ctx, const jl_cgval_t &tinfo) ctx.types().T_jlarray, emit_bitcast(ctx, decay_derived(ctx, t), ctx.types().T_pjlarray), arrayflag_field); - return tbaa_decorate(ctx.tbaa().tbaa_arrayflags, ctx.builder.CreateAlignedLoad(getInt16Ty(ctx.builder.getContext()), addr, Align(sizeof(int16_t)))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_arrayflags); + return ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt16Ty(ctx.builder.getContext()), addr, Align(sizeof(int16_t)))); } static Value *emit_arrayndims(jl_codectx_t &ctx, const jl_cgval_t &ary) @@ -2783,7 +2775,8 @@ static Value *emit_arrayelsize(jl_codectx_t &ctx, const jl_cgval_t &tinfo) Value *addr = ctx.builder.CreateStructGEP(ctx.types().T_jlarray, emit_bitcast(ctx, decay_derived(ctx, t), ctx.types().T_pjlarray), elsize_field); - return tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(getInt16Ty(ctx.builder.getContext()), addr, Align(sizeof(int16_t)))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + return ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt16Ty(ctx.builder.getContext()), addr, Align(sizeof(int16_t)))); } static Value *emit_arrayoffset(jl_codectx_t &ctx, const jl_cgval_t &tinfo, int nd) @@ -2798,7 +2791,8 @@ static Value *emit_arrayoffset(jl_codectx_t &ctx, const jl_cgval_t &tinfo, int n ctx.types().T_jlarray, emit_bitcast(ctx, decay_derived(ctx, t), ctx.types().T_pjlarray), offset_field); - return tbaa_decorate(ctx.tbaa().tbaa_arrayoffset, ctx.builder.CreateAlignedLoad(getInt32Ty(ctx.builder.getContext()), addr, Align(sizeof(int32_t)))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_arrayoffset); + return ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt32Ty(ctx.builder.getContext()), addr, Align(sizeof(int32_t)))); } // Returns the size of the array represented by `tinfo` for the given dimension `dim` if @@ -2911,8 +2905,9 @@ static Value *emit_allocobj(jl_codectx_t &ctx, size_t static_size, Value *jt); static void init_bits_value(jl_codectx_t &ctx, Value *newv, Value *v, MDNode *tbaa, unsigned alignment = sizeof(void*)) // min alignment in julia's gc is pointer-aligned { + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); // newv should already be tagged - tbaa_decorate(tbaa, ctx.builder.CreateAlignedStore(v, emit_bitcast(ctx, newv, + ai.decorateInst(ctx.builder.CreateAlignedStore(v, emit_bitcast(ctx, newv, PointerType::get(v->getType(), 0)), Align(alignment))); } @@ -2920,7 +2915,7 @@ static void init_bits_cgval(jl_codectx_t &ctx, Value *newv, const jl_cgval_t& v, { // newv should already be tagged if (v.ispointer()) { - emit_memcpy(ctx, newv, tbaa, v, jl_datatype_size(v.typ), sizeof(void*)); + emit_memcpy(ctx, newv, jl_aliasinfo_t::fromTBAA(ctx, tbaa), v, jl_datatype_size(v.typ), sizeof(void*)); } else { init_bits_value(ctx, newv, v.V, tbaa); @@ -3028,7 +3023,8 @@ static Value *load_i8box(jl_codectx_t &ctx, Value *v, jl_datatype_t *ty) GlobalVariable *gv = prepare_global_in(jl_Module, jvar); Value *idx[] = {ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 0), ctx.builder.CreateZExt(v, getInt32Ty(ctx.builder.getContext()))}; auto slot = ctx.builder.CreateInBoundsGEP(gv->getValueType(), gv, idx); - return tbaa_decorate(ctx.tbaa().tbaa_const, maybe_mark_load_dereferenceable( + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + return ai.decorateInst(maybe_mark_load_dereferenceable( ctx.builder.CreateAlignedLoad(ctx.types().T_pjlvalue, slot, Align(sizeof(void*))), false, (jl_value_t*)ty)); } @@ -3383,7 +3379,8 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con // select copy dest -> dest to simulate an undef value / conditional copy // if (skip) src_ptr = ctx.builder.CreateSelect(skip, dest, src_ptr); auto f = [&] { - (void)emit_memcpy(ctx, dest, tbaa_dst, src_ptr, src.tbaa, nb, alignment, isVolatile); + (void)emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src_ptr, + jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, alignment, isVolatile); return nullptr; }; if (skip) @@ -3419,8 +3416,8 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con ctx.builder.CreateUnreachable(); return; } else { - emit_memcpy(ctx, dest, tbaa_dst, src_ptr, - src.tbaa, nb, alignment, isVolatile); + emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src_ptr, + jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, alignment, isVolatile); } } ctx.builder.CreateBr(postBB); @@ -3445,7 +3442,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con auto f = [&] { Value *datatype = emit_typeof_boxed(ctx, src); Value *copy_bytes = emit_datatype_size(ctx, datatype); - emit_memcpy(ctx, dest, tbaa_dst, src, copy_bytes, /*TODO: min-align*/1, isVolatile); + emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src, copy_bytes, /*TODO: min-align*/1, isVolatile); return nullptr; }; if (skip) @@ -3633,7 +3630,8 @@ static jl_cgval_t emit_setfield(jl_codectx_t &ctx, } Value *tindex = compute_tindex_unboxed(ctx, rhs_union, jfty); tindex = ctx.builder.CreateNUWSub(tindex, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 1)); - tbaa_decorate(ctx.tbaa().tbaa_unionselbyte, ctx.builder.CreateAlignedStore(tindex, ptindex, Align(1))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_unionselbyte); + ai.decorateInst(ctx.builder.CreateAlignedStore(tindex, ptindex, Align(1))); // copy data if (!rhs.isghost) { emit_unionmove(ctx, addr, strct.tbaa, rhs, nullptr); @@ -3707,7 +3705,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg else { strct = emit_static_alloca(ctx, lt); if (tracked.count) - undef_derived_strct(ctx.builder, strct, sty, ctx.tbaa().tbaa_stack); + undef_derived_strct(ctx, strct, sty, ctx.tbaa().tbaa_stack); } for (unsigned i = 0; i < na; i++) { @@ -3761,10 +3759,12 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg Value *fval = NULL; if (jl_field_isptr(sty, i)) { fval = boxed(ctx, fval_info, field_promotable); - if (!init_as_value) - cast(tbaa_decorate(ctx.tbaa().tbaa_stack, - ctx.builder.CreateAlignedStore(fval, dest, Align(jl_field_align(sty, i))))) - ->setOrdering(AtomicOrdering::Unordered); + if (!init_as_value) { + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack); + StoreInst *SI = cast(ai.decorateInst( + ctx.builder.CreateAlignedStore(fval, dest, Align(jl_field_align(sty, i))))); + SI->setOrdering(AtomicOrdering::Unordered); + } } else if (jl_is_uniontype(jtype)) { // compute tindex from rhs @@ -3792,7 +3792,8 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg unsigned i = 0; for (; i < fsz / al; i++) { Value *fldp = ctx.builder.CreateConstInBoundsGEP1_32(ET, lv, i); - Value *fldv = tbaa_decorate(ctx.tbaa().tbaa_stack, ctx.builder.CreateAlignedLoad(ET, fldp, Align(al))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack); + Value *fldv = ai.decorateInst(ctx.builder.CreateAlignedLoad(ET, fldp, Align(al))); strct = ctx.builder.CreateInsertValue(strct, fldv, makeArrayRef(llvm_idx + i)); } // emit remaining bytes up to tindex @@ -3801,7 +3802,8 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg staddr = ctx.builder.CreateBitCast(staddr, getInt8PtrTy(ctx.builder.getContext())); for (; i < ptindex - llvm_idx; i++) { Value *fldp = ctx.builder.CreateConstInBoundsGEP1_32(getInt8Ty(ctx.builder.getContext()), staddr, i); - Value *fldv = tbaa_decorate(ctx.tbaa().tbaa_stack, ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), fldp, Align(1))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack); + Value *fldv = ai.decorateInst(ctx.builder.CreateAlignedLoad(getInt8Ty(ctx.builder.getContext()), fldp, Align(1))); strct = ctx.builder.CreateInsertValue(strct, fldv, makeArrayRef(llvm_idx + i)); } } @@ -3813,7 +3815,8 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg } else { Value *ptindex = emit_struct_gep(ctx, lt, strct, offs + fsz); - tbaa_decorate(ctx.tbaa().tbaa_unionselbyte, ctx.builder.CreateAlignedStore(tindex, ptindex, Align(1))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_unionselbyte); + ai.decorateInst(ctx.builder.CreateAlignedStore(tindex, ptindex, Align(1))); if (!rhs_union.isghost) emit_unionmove(ctx, dest, ctx.tbaa().tbaa_stack, fval_info, nullptr); } @@ -3850,11 +3853,13 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg unsigned llvm_idx = convert_struct_offset(ctx, cast(lt), offs + fsz); if (init_as_value) strct = ctx.builder.CreateInsertValue(strct, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0), makeArrayRef(llvm_idx)); - else - tbaa_decorate(ctx.tbaa().tbaa_unionselbyte, ctx.builder.CreateAlignedStore( + else { + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_unionselbyte); + ai.decorateInst(ctx.builder.CreateAlignedStore( ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0), ctx.builder.CreateConstInBoundsGEP2_32(lt, strct, 0, llvm_idx), Align(1))); + } } } if (type_is_ghost(lt)) @@ -3874,10 +3879,11 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg literal_pointer_val(ctx, (jl_value_t*)ty)); jl_cgval_t strctinfo = mark_julia_type(ctx, strct, true, ty); strct = decay_derived(ctx, strct); - undef_derived_strct(ctx.builder, strct, sty, strctinfo.tbaa); + undef_derived_strct(ctx, strct, sty, strctinfo.tbaa); for (size_t i = nargs; i < nf; i++) { if (!jl_field_isptr(sty, i) && jl_is_uniontype(jl_field_type(sty, i))) { - tbaa_decorate(ctx.tbaa().tbaa_unionselbyte, ctx.builder.CreateAlignedStore( + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_unionselbyte); + ai.decorateInst(ctx.builder.CreateAlignedStore( ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0), ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), emit_bitcast(ctx, strct, getInt8PtrTy(ctx.builder.getContext())), ConstantInt::get(getSizeTy(ctx.builder.getContext()), jl_field_offset(sty, i) + jl_field_size(sty, i) - 1)), diff --git a/src/codegen.cpp b/src/codegen.cpp index 5885405179c4b..4fb5c7bc0cdee 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1705,7 +1705,6 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static GlobalVariable *prepare_global_in(Module *M, GlobalVariable *G); -Instruction *tbaa_decorate(MDNode *md, Instruction *inst); static GlobalVariable *prepare_global_in(Module *M, JuliaVariable *G) { @@ -1774,20 +1773,21 @@ static AllocaInst *emit_static_alloca(jl_codectx_t &ctx, Type *lty) return new AllocaInst(lty, ctx.topalloca->getModule()->getDataLayout().getAllocaAddrSpace(), "", /*InsertBefore=*/ctx.topalloca); } -static void undef_derived_strct(IRBuilder<> &irbuilder, Value *ptr, jl_datatype_t *sty, MDNode *tbaa) +static void undef_derived_strct(jl_codectx_t &ctx, Value *ptr, jl_datatype_t *sty, MDNode *tbaa) { assert(ptr->getType()->getPointerAddressSpace() != AddressSpace::Tracked); size_t first_offset = sty->layout->nfields ? jl_field_offset(sty, 0) : 0; if (first_offset != 0) - irbuilder.CreateMemSet(ptr, ConstantInt::get(getInt8Ty(irbuilder.getContext()), 0), first_offset, MaybeAlign(0)); + ctx.builder.CreateMemSet(ptr, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0), first_offset, MaybeAlign(0)); size_t i, np = sty->layout->npointers; if (np == 0) return; - auto T_prjlvalue = JuliaType::get_prjlvalue_ty(irbuilder.getContext()); - ptr = irbuilder.CreateBitCast(ptr, T_prjlvalue->getPointerTo(ptr->getType()->getPointerAddressSpace())); + auto T_prjlvalue = JuliaType::get_prjlvalue_ty(ctx.builder.getContext()); + ptr = ctx.builder.CreateBitCast(ptr, T_prjlvalue->getPointerTo(ptr->getType()->getPointerAddressSpace())); for (i = 0; i < np; i++) { - Value *fld = irbuilder.CreateConstInBoundsGEP1_32(T_prjlvalue, ptr, jl_ptr_offset(sty, i)); - tbaa_decorate(tbaa, irbuilder.CreateStore(Constant::getNullValue(T_prjlvalue), fld)); + Value *fld = ctx.builder.CreateConstInBoundsGEP1_32(T_prjlvalue, ptr, jl_ptr_offset(sty, i)); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.decorateInst(ctx.builder.CreateStore(Constant::getNullValue(T_prjlvalue), fld)); } } @@ -2777,7 +2777,8 @@ static jl_cgval_t emit_globalref(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t * } LoadInst *v = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*))); v->setOrdering(order); - tbaa_decorate(ctx.tbaa().tbaa_binding, v); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_binding); + ai.decorateInst(v); return mark_julia_type(ctx, v, true, bnd->ty); } // todo: use type info to avoid undef check @@ -2790,7 +2791,8 @@ static void emit_globalset(jl_codectx_t &ctx, jl_binding_t *bnd, Value *bp, cons if (bnd && !bnd->constp && bnd->ty && jl_subtype(rval_info.typ, bnd->ty)) { StoreInst *v = ctx.builder.CreateAlignedStore(rval, julia_binding_pvalue(ctx, bp), Align(sizeof(void*))); v->setOrdering(Order); - tbaa_decorate(ctx.tbaa().tbaa_binding, v); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_binding); + ai.decorateInst(v); emit_write_barrier_binding(ctx, bp, rval); } else { @@ -2929,18 +2931,22 @@ static Value *emit_bits_compare(jl_codectx_t &ctx, jl_cgval_t arg1, jl_cgval_t a ctx.builder.CreateBitCast(varg2, getInt8PtrTy(ctx.builder.getContext())), ConstantInt::get(getSizeTy(ctx.builder.getContext()), sz) }, ArrayRef(&OpBundle, nroots ? 1 : 0)); - MDNode *tbaa = nullptr; - if (!arg1.tbaa) { - tbaa = arg2.tbaa; - } - else if (!arg2.tbaa) { - tbaa = arg1.tbaa; - } - else { - tbaa = MDNode::getMostGenericTBAA(arg1.tbaa, arg2.tbaa); + + if (arg1.tbaa || arg2.tbaa) { + jl_aliasinfo_t ai; + if (!arg1.tbaa) { + ai = jl_aliasinfo_t::fromTBAA(ctx, arg2.tbaa); + } + else if (!arg2.tbaa) { + ai = jl_aliasinfo_t::fromTBAA(ctx, arg1.tbaa); + } + else { + jl_aliasinfo_t arg1_ai = jl_aliasinfo_t::fromTBAA(ctx, arg1.tbaa); + jl_aliasinfo_t arg2_ai = jl_aliasinfo_t::fromTBAA(ctx, arg2.tbaa); + ai = arg1_ai.merge(arg2_ai); + } + ai.decorateInst(answer); } - if (tbaa) - tbaa_decorate(tbaa, answer); return ctx.builder.CreateICmpEQ(answer, ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 0)); } else { @@ -3495,7 +3501,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_bitcast(ctx, decay_derived(ctx, aryv), ctx.types().T_pprjlvalue), jl_array_data_owner_offset(nd) / sizeof(jl_value_t*)), Align(sizeof(void*))); - tbaa_decorate(ctx.tbaa().tbaa_const, maybe_mark_load_dereferenceable(own_ptr, false, (jl_value_t*)jl_array_any_type)); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + ai.decorateInst(maybe_mark_load_dereferenceable(own_ptr, false, (jl_value_t*)jl_array_any_type)); } else { own_ptr = ctx.builder.CreateCall( @@ -3532,7 +3539,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, ptindex = emit_bitcast(ctx, ptindex, getInt8PtrTy(ctx.builder.getContext())); ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, offset); ptindex = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), ptindex, idx); - tbaa_decorate(ctx.tbaa().tbaa_arrayselbyte, ctx.builder.CreateStore(tindex, ptindex)); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_arrayselbyte); + ai.decorateInst(ctx.builder.CreateStore(tindex, ptindex)); if (elsz > 0 && (!jl_is_datatype(val.typ) || jl_datatype_size(val.typ) > 0)) { // copy data (if any) emit_unionmove(ctx, data, ctx.tbaa().tbaa_arraybuf, val, nullptr); @@ -3630,7 +3638,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, idx = ctx.builder.CreateAdd(idx, ConstantInt::get(getSizeTy(ctx.builder.getContext()), ctx.nReqArgs)); Instruction *v = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, ctx.argArray, idx), Align(sizeof(void*))); // if we know the result type of this load, we will mark that information here too - tbaa_decorate(ctx.tbaa().tbaa_value, maybe_mark_load_dereferenceable(v, false, rt)); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_value); + ai.decorateInst(maybe_mark_load_dereferenceable(v, false, rt)); *ret = mark_julia_type(ctx, v, /*boxed*/ true, rt); return true; } @@ -3793,7 +3802,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, emit_typecheck(ctx, argv[3], (jl_value_t*)jl_bool_type, "fieldtype"); emit_bounds_check(ctx, typ, (jl_value_t*)jl_datatype_type, idx, types_len, boundscheck); Value *fieldtyp_p = ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, decay_derived(ctx, emit_bitcast(ctx, types_svec, ctx.types().T_pprjlvalue)), idx); - Value *fieldtyp = tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, fieldtyp_p, Align(sizeof(void*)))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + Value *fieldtyp = ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, fieldtyp_p, Align(sizeof(void*)))); *ret = mark_julia_type(ctx, fieldtyp, true, (jl_value_t*)jl_type_type); return true; } @@ -3818,7 +3828,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, } // String and SimpleVector's length fields have the same layout auto ptr = emit_bitcast(ctx, boxed(ctx, obj), getSizePtrTy(ctx.builder.getContext())); - Value *len = tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(getSizeTy(ctx.builder.getContext()), ptr, Align(sizeof(size_t)))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + Value *len = ai.decorateInst(ctx.builder.CreateAlignedLoad(getSizeTy(ctx.builder.getContext()), ptr, Align(sizeof(size_t)))); MDBuilder MDB(ctx.builder.getContext()); if (sty == jl_simplevector_type) { auto rng = MDB.createRange( @@ -3949,7 +3960,8 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, Value *addr = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, ptr, offs); // emit this using the same type as emit_getfield_knownidx // so that LLVM may be able to load-load forward them and fold the result - fldv = tbaa_decorate(tbaa, ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, addr, Align(sizeof(size_t)))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + fldv = ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, addr, Align(sizeof(size_t)))); cast(fldv)->setOrdering(order <= jl_memory_order_notatomic ? AtomicOrdering::Unordered : get_llvm_atomic_order(order)); } else { @@ -4439,8 +4451,10 @@ static jl_cgval_t emit_checked_var(jl_codectx_t &ctx, Value *bp, jl_sym_t *name, if (isvol) v->setVolatile(true); v->setOrdering(AtomicOrdering::Unordered); - if (tbaa) - tbaa_decorate(tbaa, v); + if (tbaa) { + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.decorateInst(v); + } undef_var_error_ifnot(ctx, ctx.builder.CreateIsNotNull(v), name); return mark_julia_type(ctx, v, true, jl_any_type); } @@ -4458,7 +4472,8 @@ static jl_cgval_t emit_sparam(jl_codectx_t &ctx, size_t i) ctx.types().T_prjlvalue, ctx.spvals_ptr, i + sizeof(jl_svec_t) / sizeof(jl_value_t*)); - Value *sp = tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*)))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + Value *sp = ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*)))); Value *isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp, false), track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jl_tvar_type))); jl_unionall_t *sparam = (jl_unionall_t*)ctx.linfo->def.method->sig; @@ -4513,7 +4528,8 @@ static jl_cgval_t emit_isdefined(jl_codectx_t &ctx, jl_value_t *sym) ctx.types().T_prjlvalue, ctx.spvals_ptr, i + sizeof(jl_svec_t) / sizeof(jl_value_t*)); - Value *sp = tbaa_decorate(ctx.tbaa().tbaa_const, ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*)))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + Value *sp = ai.decorateInst(ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*)))); isnull = ctx.builder.CreateICmpNE(emit_typeof(ctx, sp, false), track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)jl_tvar_type))); } @@ -4536,7 +4552,8 @@ static jl_cgval_t emit_isdefined(jl_codectx_t &ctx, jl_value_t *sym) Value *bp = julia_binding_gv(ctx, bnd); bp = julia_binding_pvalue(ctx, bp); LoadInst *v = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*))); - tbaa_decorate(ctx.tbaa().tbaa_binding, v); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_binding); + ai.decorateInst(v); v->setOrdering(AtomicOrdering::Unordered); isnull = ctx.builder.CreateICmpNE(v, Constant::getNullValue(ctx.types().T_prjlvalue)); } @@ -4577,7 +4594,7 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va else { const DataLayout &DL = jl_Module->getDataLayout(); uint64_t sz = DL.getTypeStoreSize(T); - emit_memcpy(ctx, ssaslot, ctx.tbaa().tbaa_stack, vi.value, sz, ssaslot->getAlign().value()); + emit_memcpy(ctx, ssaslot, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), vi.value, sz, ssaslot->getAlign().value()); } Value *tindex = NULL; if (vi.pTIndex) @@ -4667,7 +4684,8 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu Type *dest_ty = store_ty->getPointerTo(); if (dest_ty != dest->getType()) dest = emit_bitcast(ctx, dest, dest_ty); - tbaa_decorate(ctx.tbaa().tbaa_stack, ctx.builder.CreateStore( + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack); + ai.decorateInst(ctx.builder.CreateStore( emit_unbox(ctx, store_ty, rval_info, rval_info.typ), dest, vi.isVolatile)); @@ -4683,7 +4701,7 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu // This check should probably mostly catch the relevant situations. if (vi.value.V != rval_info.V) { Value *copy_bytes = ConstantInt::get(getInt32Ty(ctx.builder.getContext()), jl_datatype_size(vi.value.typ)); - emit_memcpy(ctx, vi.value.V, ctx.tbaa().tbaa_stack, rval_info, copy_bytes, + emit_memcpy(ctx, vi.value.V, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), rval_info, copy_bytes, julia_alignment(rval_info.typ), vi.isVolatile); } } @@ -5397,11 +5415,9 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ std::tie(F, specF) = get_oc_function(ctx, (jl_method_t*)source.constant, env_t, argt_typ, ub.constant); if (F) { jl_cgval_t jlcall_ptr = mark_julia_type(ctx, F, false, jl_voidpointer_type); - jl_cgval_t world_age = mark_julia_type(ctx, - tbaa_decorate(ctx.tbaa().tbaa_gcframe, - ctx.builder.CreateAlignedLoad(getSizeTy(ctx.builder.getContext()), get_last_age_field(ctx), Align(sizeof(size_t)))), - false, - jl_long_type); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe); + Instruction *I = ctx.builder.CreateAlignedLoad(getSizeTy(ctx.builder.getContext()), get_last_age_field(ctx), Align(sizeof(size_t))); + jl_cgval_t world_age = mark_julia_type(ctx, ai.decorateInst(I), false, jl_long_type); jl_cgval_t fptr; if (specF) fptr = mark_julia_type(ctx, specF, false, jl_voidpointer_type); @@ -5690,7 +5706,8 @@ static void emit_cfunc_invalidate( root1 = ctx.builder.CreateConstInBoundsGEP2_32(get_returnroots_type(ctx, return_roots), root1, 0, 0); ctx.builder.CreateStore(gf_ret, root1); } - emit_memcpy(ctx, &*gf_thunk->arg_begin(), nullptr, gf_ret, nullptr, jl_datatype_size(rettype), julia_alignment(rettype)); + emit_memcpy(ctx, &*gf_thunk->arg_begin(), jl_aliasinfo_t::fromTBAA(ctx, nullptr), gf_ret, + jl_aliasinfo_t::fromTBAA(ctx, nullptr), jl_datatype_size(rettype), julia_alignment(rettype)); ctx.builder.CreateRetVoid(); break; } @@ -5857,7 +5874,8 @@ static Function* gen_cfun_wrapper( allocate_gc_frame(ctx, b0, true); Value *world_age_field = get_last_age_field(ctx); - Value *last_age = tbaa_decorate(ctx.tbaa().tbaa_gcframe, + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe); + Value *last_age = ai.decorateInst( ctx.builder.CreateAlignedLoad(getSizeTy(ctx.builder.getContext()), world_age_field, Align(sizeof(size_t)))); Value *world_v = ctx.builder.CreateAlignedLoad(getSizeTy(ctx.builder.getContext()), @@ -6413,13 +6431,14 @@ static jl_cgval_t emit_cfunction(jl_codectx_t &ctx, jl_value_t *output_type, con literal_pointer_val(ctx, (jl_value_t*)output_type)); Value *derived_strct = emit_bitcast(ctx, decay_derived(ctx, strct), getSizePtrTy(ctx.builder.getContext())); MDNode *tbaa = best_tbaa(ctx.tbaa(), output_type); - tbaa_decorate(tbaa, ctx.builder.CreateStore(F, derived_strct)); - tbaa_decorate(tbaa, ctx.builder.CreateStore( + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.decorateInst(ctx.builder.CreateStore(F, derived_strct)); + ai.decorateInst(ctx.builder.CreateStore( ctx.builder.CreatePtrToInt(literal_pointer_val(ctx, fexpr_rt.constant), getSizeTy(ctx.builder.getContext())), ctx.builder.CreateConstInBoundsGEP1_32(getSizeTy(ctx.builder.getContext()), derived_strct, 1))); - tbaa_decorate(tbaa, ctx.builder.CreateStore(Constant::getNullValue(getSizeTy(ctx.builder.getContext())), + ai.decorateInst(ctx.builder.CreateStore(Constant::getNullValue(getSizeTy(ctx.builder.getContext())), ctx.builder.CreateConstInBoundsGEP1_32(getSizeTy(ctx.builder.getContext()), derived_strct, 2))); - tbaa_decorate(tbaa, ctx.builder.CreateStore(Constant::getNullValue(getSizeTy(ctx.builder.getContext())), + ai.decorateInst(ctx.builder.CreateStore(Constant::getNullValue(getSizeTy(ctx.builder.getContext())), ctx.builder.CreateConstInBoundsGEP1_32(getSizeTy(ctx.builder.getContext()), derived_strct, 3))); F = strct; } @@ -6553,7 +6572,8 @@ static Function *gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *jlret } else { Value *argPtr = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, argArray, i - 1); - theArg = tbaa_decorate(ctx.tbaa().tbaa_const, maybe_mark_load_dereferenceable( + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + theArg = ai.decorateInst(maybe_mark_load_dereferenceable( ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, argPtr, Align(sizeof(void*))), false, ty)); @@ -7196,7 +7216,8 @@ static jl_llvm_functions_t Value *last_age = NULL; Value *world_age_field = get_last_age_field(ctx); if (toplevel || ctx.is_opaque_closure) { - last_age = tbaa_decorate(ctx.tbaa().tbaa_gcframe, ctx.builder.CreateAlignedLoad( + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe); + last_age = ai.decorateInst(ctx.builder.CreateAlignedLoad( getSizeTy(ctx.builder.getContext()), world_age_field, Align(sizeof(size_t)))); } @@ -7437,7 +7458,8 @@ static jl_llvm_functions_t } else { Value *argPtr = ctx.builder.CreateConstInBoundsGEP1_32(ctx.types().T_prjlvalue, argArray, i - 1); - Value *load = tbaa_decorate(ctx.tbaa().tbaa_const, maybe_mark_load_dereferenceable( + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_const); + Value *load = ai.decorateInst(maybe_mark_load_dereferenceable( ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, argPtr, Align(sizeof(void*))), false, vi.value.typ)); theArg = mark_julia_type(ctx, load, true, vi.value.typ); @@ -7921,8 +7943,8 @@ static jl_llvm_functions_t } if (returninfo.cc == jl_returninfo_t::SRet) { assert(jl_is_concrete_type(jlrettype)); - emit_memcpy(ctx, sret, nullptr, retvalinfo, jl_datatype_size(jlrettype), - julia_alignment(jlrettype)); + emit_memcpy(ctx, sret, jl_aliasinfo_t::fromTBAA(ctx, nullptr), retvalinfo, + jl_datatype_size(jlrettype), julia_alignment(jlrettype)); } else { // must be jl_returninfo_t::Union emit_unionmove(ctx, sret, nullptr, retvalinfo, /*skip*/isboxed_union); diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 38d923cb5a99e..b822907e63524 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -397,7 +397,8 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va Value *p = x.constant ? literal_pointer_val(ctx, x.constant) : x.V; if (jt == (jl_value_t*)jl_bool_type || to->isIntegerTy(1)) { - Instruction *unbox_load = tbaa_decorate(x.tbaa, ctx.builder.CreateLoad(getInt8Ty(ctx.builder.getContext()), maybe_bitcast(ctx, p, getInt8PtrTy(ctx.builder.getContext())))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, x.tbaa); + Instruction *unbox_load = ai.decorateInst(ctx.builder.CreateLoad(getInt8Ty(ctx.builder.getContext()), maybe_bitcast(ctx, p, getInt8PtrTy(ctx.builder.getContext())))); if (jt == (jl_value_t*)jl_bool_type) unbox_load->setMetadata(LLVMContext::MD_range, MDNode::get(ctx.builder.getContext(), { ConstantAsMetadata::get(ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0)), @@ -425,12 +426,14 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va (to->isFloatingPointTy() || to->isIntegerTy() || to->isPointerTy()) && DL.getTypeSizeInBits(AllocType) == DL.getTypeSizeInBits(to)) { Instruction *load = ctx.builder.CreateAlignedLoad(AllocType, p, Align(alignment)); - return emit_unboxed_coercion(ctx, to, tbaa_decorate(x.tbaa, load)); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, x.tbaa); + return emit_unboxed_coercion(ctx, to, ai.decorateInst(load)); } } p = maybe_bitcast(ctx, p, ptype); Instruction *load = ctx.builder.CreateAlignedLoad(to, p, Align(alignment)); - return tbaa_decorate(x.tbaa, load); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, x.tbaa); + return ai.decorateInst(load); } // emit code to store a raw value into a destination @@ -459,12 +462,13 @@ static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest dest = emit_bitcast(ctx, dest, dest_ty); StoreInst *store = ctx.builder.CreateAlignedStore(unboxed, dest, Align(alignment)); store->setVolatile(isVolatile); - tbaa_decorate(tbaa_dest, store); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa_dest); + ai.decorateInst(store); return; } Value *src = data_pointer(ctx, x); - emit_memcpy(ctx, dest, tbaa_dest, src, x.tbaa, jl_datatype_size(x.typ), alignment, isVolatile); + emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dest), src, jl_aliasinfo_t::fromTBAA(ctx, x.tbaa), jl_datatype_size(x.typ), alignment, isVolatile); } static jl_datatype_t *staticeval_bitstype(const jl_cgval_t &targ) @@ -546,7 +550,8 @@ static jl_cgval_t generic_bitcast(jl_codectx_t &ctx, const jl_cgval_t *argv) if (isboxed) vxt = llvmt; auto storage_type = vxt->isIntegerTy(1) ? getInt8Ty(ctx.builder.getContext()) : vxt; - vx = tbaa_decorate(v.tbaa, ctx.builder.CreateLoad( + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, v.tbaa); + vx = ai.decorateInst(ctx.builder.CreateLoad( storage_type, emit_bitcast(ctx, data_pointer(ctx, v), storage_type->getPointerTo()))); @@ -662,7 +667,8 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) if (ety == (jl_value_t*)jl_any_type) { Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, e.typ); LoadInst *load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, ctx.builder.CreateInBoundsGEP(ctx.types().T_prjlvalue, thePtr, im1), Align(align_nb)); - tbaa_decorate(ctx.tbaa().tbaa_data, load); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_data); + ai.decorateInst(load); return mark_julia_type(ctx, load, true, ety); } else if (!jl_isbits(ety)) { @@ -675,7 +681,7 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) Value *thePtr = emit_unbox(ctx, getInt8PtrTy(ctx.builder.getContext()), e, e.typ); thePtr = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), emit_bitcast(ctx, thePtr, getInt8PtrTy(ctx.builder.getContext())), im1); MDNode *tbaa = best_tbaa(ctx.tbaa(), ety); - emit_memcpy(ctx, strct, tbaa, thePtr, nullptr, size, 1); + emit_memcpy(ctx, strct, jl_aliasinfo_t::fromTBAA(ctx, tbaa), thePtr, jl_aliasinfo_t::fromTBAA(ctx, nullptr), size, 1); return mark_julia_type(ctx, strct, true, ety); } else { @@ -735,14 +741,15 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, jl_cgval_t *argv) Instruction *store = ctx.builder.CreateAlignedStore( ctx.builder.CreatePtrToInt(emit_pointer_from_objref(ctx, boxed(ctx, x)), getSizeTy(ctx.builder.getContext())), ctx.builder.CreateInBoundsGEP(getSizeTy(ctx.builder.getContext()), thePtr, im1), Align(align_nb)); - tbaa_decorate(ctx.tbaa().tbaa_data, store); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_data); + ai.decorateInst(store); } else if (!jl_isbits(ety)) { thePtr = emit_unbox(ctx, getInt8PtrTy(ctx.builder.getContext()), e, e.typ); uint64_t size = jl_datatype_size(ety); im1 = ctx.builder.CreateMul(im1, ConstantInt::get(getSizeTy(ctx.builder.getContext()), LLT_ALIGN(size, jl_datatype_align(ety)))); - emit_memcpy(ctx, ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), thePtr, im1), nullptr, x, size, align_nb); + emit_memcpy(ctx, ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), thePtr, im1), jl_aliasinfo_t::fromTBAA(ctx, nullptr), x, size, align_nb); } else { bool isboxed; @@ -793,7 +800,8 @@ static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) if (ety == (jl_value_t*)jl_any_type) { Value *thePtr = emit_unbox(ctx, ctx.types().T_pprjlvalue, e, e.typ); LoadInst *load = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, thePtr, Align(sizeof(jl_value_t*))); - tbaa_decorate(ctx.tbaa().tbaa_data, load); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_data); + ai.decorateInst(load); load->setOrdering(llvm_order); return mark_julia_type(ctx, load, true, ety); } @@ -819,11 +827,12 @@ static jl_cgval_t emit_atomic_pointerref(jl_codectx_t &ctx, jl_cgval_t *argv) thePtr = emit_bitcast(ctx, thePtr, loadT->getPointerTo()); MDNode *tbaa = best_tbaa(ctx.tbaa(), ety); LoadInst *load = ctx.builder.CreateAlignedLoad(loadT, thePtr, Align(nb)); - tbaa_decorate(tbaa, load); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); + ai.decorateInst(load); load->setOrdering(llvm_order); thePtr = emit_bitcast(ctx, strct, thePtr->getType()); StoreInst *store = ctx.builder.CreateAlignedStore(load, thePtr, Align(julia_alignment(ety))); - tbaa_decorate(tbaa, store); + ai.decorateInst(store); return mark_julia_type(ctx, strct, true, ety); } else { diff --git a/test/llvmpasses/aliasscopes.jl b/test/llvmpasses/aliasscopes.jl index 5c0fe48091ade..751e351dfad1e 100644 --- a/test/llvmpasses/aliasscopes.jl +++ b/test/llvmpasses/aliasscopes.jl @@ -18,8 +18,8 @@ import Base.Experimental: Const, @aliasscope function simple(A, B) @aliasscope @inbounds for I in eachindex(A, B) A[I] = Const(B)[I] -# CHECK: load double, {{.*}} !alias.scope [[SCOPE:![0-9]+]] -# CHECK: store double {{.*}} !noalias [[SCOPE]] +# CHECK: load double, {{.*}} !alias.scope [[SCOPE_LD:![0-9]+]] +# CHECK: store double {{.*}} !noalias [[SCOPE_ST:![0-9]+]] end return 0 # return nothing causes japi1 end @@ -28,8 +28,8 @@ end function constargs(A, B::Const) @aliasscope @inbounds for I in eachindex(A, B) A[I] = B[I] -# CHECK: load double, {{.*}} !alias.scope [[SCOPE2:![0-9]+]] -# CHECK: store double {{.*}} !noalias [[SCOPE2]] +# CHECK: load double, {{.*}} !alias.scope [[SCOPE2_LD:![0-9]+]] +# CHECK: store double {{.*}} !noalias [[SCOPE2_ST:![0-9]+]] end return 0 end @@ -40,10 +40,10 @@ function micro_ker!(AB, Ac, Bc, kc, offSetA, offSetB) @inbounds @aliasscope for k in 1:kc for j in 1:NR, i in 1:MR AB[i+(j-1)*MR] = muladd(Const(Ac)[offSetA+i], Const(Bc)[offSetB+j], Const(AB)[i+(j-1)*MR]) -# CHECK: load double, {{.*}} !alias.scope [[SCOPE3:![0-9]+]] -# CHECK: load double, {{.*}} !alias.scope [[SCOPE3]] -# CHECK: load double, {{.*}} !alias.scope [[SCOPE3]] -# CHECK: store double {{.*}} !noalias [[SCOPE3]] +# CHECK: load double, {{.*}} !alias.scope [[SCOPE3_LD:![0-9]+]] +# CHECK: load double, {{.*}} !alias.scope [[SCOPE3_LD]] +# CHECK: load double, {{.*}} !alias.scope [[SCOPE3_LD]] +# CHECK: store double {{.*}} !noalias [[SCOPE3_ST:![0-9]+]] end offSetA += MR offSetB += NR @@ -51,9 +51,14 @@ function micro_ker!(AB, Ac, Bc, kc, offSetA, offSetB) return end -# CHECK: [[SCOPE]] = !{[[ALIASSCOPE:![0-9]+]]} -# CHECK: [[ALIASSCOPE]] = !{!"aliasscope", [[MDNODE:![0-9]+]]} -# CHECK: [[MDNODE]] = !{!"simple"} +# CHECK-DAG: [[SCOPE_LD]] = !{[[ALIASSCOPE:![0-9]+]] +# CHECK-DAG: [[SCOPE_ST]] = !{[[ALIASSCOPE]] +# CHECK-DAG: [[SCOPE2_LD]] = !{[[ALIASSCOPE2:![0-9]+]] +# CHECK-DAG: [[SCOPE2_ST]] = !{[[ALIASSCOPE2]] +# CHECK-DAG: [[SCOPE3_LD]] = !{[[ALIASSCOPE3:![0-9]+]] +# CHECK-DAG: [[SCOPE3_ST]] = !{[[ALIASSCOPE3]] +# CHECK-DAG: [[ALIASSCOPE]] = !{!"aliasscope", [[MDNODE:![0-9]+]]} +# CHECK-DAG: [[MDNODE]] = !{!"simple"} emit(simple, Vector{Float64}, Vector{Float64}) emit(constargs, Vector{Float64}, Const{Float64, 1})