diff --git a/src/abi_aarch64.cpp b/src/abi_aarch64.cpp index 7c31b6606139a..0a193ee132556 100644 --- a/src/abi_aarch64.cpp +++ b/src/abi_aarch64.cpp @@ -16,7 +16,7 @@ struct ABI_AArch64Layout : AbiLayout { Type *get_llvm_vectype(jl_datatype_t *dt, LLVMContext &ctx) const { // Assume jl_is_datatype(dt) && !jl_is_abstracttype(dt) - // `!dt->name->mutabl && dt->pointerfree && !dt->haspadding && dt->nfields > 0` + // `!dt->name->mutabl && dt->pointerfree && !dt->haspadding && dt->isbitsegal && dt->nfields > 0` if (dt->layout == NULL || jl_is_layout_opaque(dt->layout)) return nullptr; size_t nfields = dt->layout->nfields; @@ -62,7 +62,7 @@ Type *get_llvm_vectype(jl_datatype_t *dt, LLVMContext &ctx) const Type *get_llvm_fptype(jl_datatype_t *dt, LLVMContext &ctx) const { // Assume jl_is_datatype(dt) && !jl_is_abstracttype(dt) - // `!dt->name->mutabl && dt->pointerfree && !dt->haspadding && dt->nfields == 0` + // `!dt->name->mutabl && dt->pointerfree && !dt->haspadding && dt->isbitsegal && dt->nfields == 0` Type *lltype; // Check size first since it's cheaper. switch (jl_datatype_size(dt)) { @@ -88,7 +88,7 @@ Type *get_llvm_fptype(jl_datatype_t *dt, LLVMContext &ctx) const Type *get_llvm_fp_or_vectype(jl_datatype_t *dt, LLVMContext &ctx) const { // Assume jl_is_datatype(dt) && !jl_is_abstracttype(dt) - if (dt->name->mutabl || dt->layout->npointers || dt->layout->flags.haspadding) + if (dt->name->mutabl || dt->layout->npointers || !dt->layout->flags.isbitsegal || dt->layout->flags.haspadding) return nullptr; return dt->layout->nfields ? get_llvm_vectype(dt, ctx) : get_llvm_fptype(dt, ctx); } @@ -184,7 +184,7 @@ Type *isHFAorHVA(jl_datatype_t *dt, size_t &nele, LLVMContext &ctx) const // uniquely addressable members. // Maximum HFA and HVA size is 64 bytes (4 x fp128 or 16bytes vector) size_t dsz = jl_datatype_size(dt); - if (dsz > 64 || !dt->layout || dt->layout->npointers || dt->layout->flags.haspadding) + if (dsz > 64 || !dt->layout || dt->layout->npointers || !dt->layout->flags.isbitsegal || dt->layout->flags.haspadding) return NULL; nele = 0; ElementType eltype; diff --git a/src/abi_arm.cpp b/src/abi_arm.cpp index 68f980d7b40da..8839a37da6e13 100644 --- a/src/abi_arm.cpp +++ b/src/abi_arm.cpp @@ -82,7 +82,7 @@ size_t isLegalHA(jl_datatype_t *dt, Type *&base, LLVMContext &ctx) const if (jl_is_structtype(dt)) { // Fast path checks before descending the type hierarchy // (4 x 128b vector == 64B max size) - if (jl_datatype_size(dt) > 64 || dt->layout->npointers || dt->layout->flags.haspadding) + if (jl_datatype_size(dt) > 64 || dt->layout->npointers || !dt->layout->flags.isbitsegal || dt->layout->flags.haspadding) return 0; base = NULL; diff --git a/src/abi_ppc64le.cpp b/src/abi_ppc64le.cpp index 1f10817cfeeee..f02e1022ddc2d 100644 --- a/src/abi_ppc64le.cpp +++ b/src/abi_ppc64le.cpp @@ -44,7 +44,7 @@ struct ABI_PPC64leLayout : AbiLayout { // count the homogeneous floating aggregate size (saturating at max count of 8) unsigned isHFA(jl_datatype_t *ty, jl_datatype_t **ty0, bool *hva) const { - if (jl_datatype_size(ty) > 128 || ty->layout->npointers || ty->layout->flags.haspadding) + if (jl_datatype_size(ty) > 128 || ty->layout->npointers || !ty->layout->flags.isbitsegal || ty->layout->flags.haspadding) return 9; size_t i, l = ty->layout->nfields; diff --git a/src/ccall.cpp b/src/ccall.cpp index db7bcb3a408d0..97315d46b6ead 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -554,8 +554,8 @@ static Value *julia_to_native( // pass the address of an alloca'd thing, not a box // since those are immutable. Value *slot = emit_static_alloca(ctx, to); - unsigned align = julia_alignment(jlto); - cast(slot)->setAlignment(Align(align)); + Align align(julia_alignment(jlto)); + cast(slot)->setAlignment(align); setName(ctx.emission_context, slot, "native_convert_buffer"); if (!jvinfo.ispointer()) { jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, jvinfo.tbaa); @@ -2230,7 +2230,7 @@ jl_cgval_t function_sig_t::emit_a_ccall( Value *strct = emit_allocobj(ctx, (jl_datatype_t*)rt, true); setName(ctx.emission_context, strct, "ccall_ret_box"); MDNode *tbaa = jl_is_mutable(rt) ? ctx.tbaa().tbaa_mutab : ctx.tbaa().tbaa_immut; - int boxalign = julia_alignment(rt); + Align boxalign(julia_alignment(rt)); // copy the data from the return value to the new struct const DataLayout &DL = ctx.builder.GetInsertBlock()->getModule()->getDataLayout(); auto resultTy = result->getType(); @@ -2240,8 +2240,8 @@ jl_cgval_t function_sig_t::emit_a_ccall( // When this happens, cast through memory. auto slot = emit_static_alloca(ctx, resultTy); setName(ctx.emission_context, slot, "type_pun_slot"); - slot->setAlignment(Align(boxalign)); - ctx.builder.CreateAlignedStore(result, slot, Align(boxalign)); + slot->setAlignment(boxalign); + ctx.builder.CreateAlignedStore(result, slot, boxalign); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); emit_memcpy(ctx, strct, ai, slot, ai, rtsz, boxalign, boxalign); } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 613d7ae719448..08d51f52b613b 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -315,7 +315,7 @@ static Value *emit_pointer_from_objref(jl_codectx_t &ctx, Value *V) } static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_value_t *jt); -static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value* dest, MDNode *tbaa_dest, unsigned alignment, bool isVolatile=false); +static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value* dest, MDNode *tbaa_dest, Align alignment, bool isVolatile=false); static bool type_is_permalloc(jl_value_t *typ) { @@ -1006,11 +1006,10 @@ static Value *data_pointer(jl_codectx_t &ctx, const jl_cgval_t &x) } static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src, - jl_aliasinfo_t const &src_ai, uint64_t sz, unsigned align_dst, unsigned align_src, bool is_volatile) + jl_aliasinfo_t const &src_ai, uint64_t sz, Align align_dst, Align align_src, bool is_volatile) { if (sz == 0) return; - assert(align_dst && "align must be specified"); #if JL_LLVM_VERSION < 170000 // If the types are small and simple, use load and store directly. // Going through memcpy can cause LLVM (e.g. SROA) to create bitcasts between float and int @@ -1053,7 +1052,7 @@ static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const if (isa(dst) && !dst->hasName()) setName(ctx.emission_context, dst, "memcpy_refined_dst"); auto val = src_ai.decorateInst(ctx.builder.CreateAlignedLoad(directel, src, MaybeAlign(align_src), is_volatile)); - dst_ai.decorateInst(ctx.builder.CreateAlignedStore(val, dst, Align(align_dst), is_volatile)); + dst_ai.decorateInst(ctx.builder.CreateAlignedStore(val, dst, align_dst, is_volatile)); ++SkippedMemcpys; return; } @@ -1072,12 +1071,12 @@ static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const // above problem won't be as serious. auto merged_ai = dst_ai.merge(src_ai); - ctx.builder.CreateMemCpy(dst, Align(align_dst), src, Align(align_src), sz, is_volatile, + ctx.builder.CreateMemCpy(dst, align_dst, src, align_src, sz, is_volatile, merged_ai.tbaa, merged_ai.tbaa_struct, merged_ai.scope, merged_ai.noalias); } static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src, - jl_aliasinfo_t const &src_ai, Value *sz, unsigned align_dst, unsigned align_src, bool is_volatile) + jl_aliasinfo_t const &src_ai, Value *sz, Align align_dst, Align align_src, bool is_volatile) { if (auto const_sz = dyn_cast(sz)) { emit_memcpy_llvm(ctx, dst, dst_ai, src, src_ai, const_sz->getZExtValue(), align_dst, align_src, is_volatile); @@ -1086,20 +1085,20 @@ static void emit_memcpy_llvm(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const ++EmittedMemcpys; auto merged_ai = dst_ai.merge(src_ai); - ctx.builder.CreateMemCpy(dst, MaybeAlign(align_dst), src, MaybeAlign(align_src), sz, is_volatile, + ctx.builder.CreateMemCpy(dst, align_dst, src, align_src, sz, is_volatile, merged_ai.tbaa, merged_ai.tbaa_struct, merged_ai.scope, merged_ai.noalias); } template static void emit_memcpy(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, Value *src, - jl_aliasinfo_t const &src_ai, T1 &&sz, unsigned align_dst, unsigned align_src, bool is_volatile=false) + jl_aliasinfo_t const &src_ai, T1 &&sz, Align align_dst, Align align_src, bool is_volatile=false) { emit_memcpy_llvm(ctx, dst, dst_ai, src, src_ai, sz, align_dst, align_src, is_volatile); } template static void emit_memcpy(jl_codectx_t &ctx, Value *dst, jl_aliasinfo_t const &dst_ai, const jl_cgval_t &src, - T1 &&sz, unsigned align_dst, unsigned align_src, bool is_volatile=false) + T1 &&sz, Align align_dst, Align align_src, bool is_volatile=false) { auto src_ai = jl_aliasinfo_t::fromTBAA(ctx, src.tbaa); emit_memcpy_llvm(ctx, dst, dst_ai, data_pointer(ctx, src), src_ai, sz, align_dst, align_src, is_volatile); @@ -1999,7 +1998,7 @@ static jl_cgval_t typed_load(jl_codectx_t &ctx, Value *ptr, Value *idx_0based, j else if (!alignment) alignment = julia_alignment(jltype); if (intcast && Order == AtomicOrdering::NotAtomic) { - emit_memcpy(ctx, intcast, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), data, jl_aliasinfo_t::fromTBAA(ctx, tbaa), nb, alignment, intcast->getAlign().value()); + emit_memcpy(ctx, intcast, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), data, jl_aliasinfo_t::fromTBAA(ctx, tbaa), nb, Align(alignment), intcast->getAlign()); } else { if (!isboxed && jl_is_genericmemoryref_type(jltype)) { @@ -2120,12 +2119,15 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, FailOrder = AtomicOrdering::Monotonic; unsigned nb = isboxed ? sizeof(void*) : jl_datatype_size(jltype); AllocaInst *intcast = nullptr; + Type *intcast_eltyp = nullptr; + bool tracked_pointers = isboxed || CountTrackedPointers(elty).count > 0; if (!isboxed && Order != AtomicOrdering::NotAtomic && !elty->isIntOrPtrTy()) { + intcast_eltyp = elty; + elty = Type::getIntNTy(ctx.builder.getContext(), 8 * nb); if (!issetfield) { intcast = emit_static_alloca(ctx, elty); setName(ctx.emission_context, intcast, "atomic_store_box"); } - elty = Type::getIntNTy(ctx.builder.getContext(), 8 * nb); } Type *realelty = elty; if (Order != AtomicOrdering::NotAtomic && isa(elty)) { @@ -2134,14 +2136,20 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, elty = Type::getIntNTy(ctx.builder.getContext(), 8 * nb2); } Value *r = nullptr; - if (issetfield || isswapfield || isreplacefield || issetfieldonce) { - if (isboxed) + if (issetfield || isswapfield || isreplacefield || issetfieldonce) { // e.g. !ismodifyfield + assert(isboxed || rhs.typ == jltype); + if (isboxed) { r = boxed(ctx, rhs); - else if (aliasscope || Order != AtomicOrdering::NotAtomic || CountTrackedPointers(realelty).count) { + } + else if (intcast) { + emit_unbox_store(ctx, rhs, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + r = ctx.builder.CreateLoad(realelty, intcast); + } + else if (aliasscope || Order != AtomicOrdering::NotAtomic || tracked_pointers) { r = emit_unbox(ctx, realelty, rhs, jltype); - if (realelty != elty) - r = ctx.builder.CreateZExt(r, elty); } + if (realelty != elty) + r = ctx.builder.CreateZExt(r, elty); } if (isboxed) alignment = sizeof(void*); @@ -2176,7 +2184,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } else { assert(Order == AtomicOrdering::NotAtomic && !isboxed && rhs.typ == jltype); - emit_unbox_store(ctx, rhs, ptr, tbaa, alignment); + emit_unbox_store(ctx, rhs, ptr, tbaa, Align(alignment)); } } else if (isswapfield) { @@ -2223,7 +2231,14 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, Current->addIncoming(instr, SkipBB); ctx.builder.SetInsertPoint(BB); } - Compare = emit_unbox(ctx, realelty, cmp, jltype); + cmp = update_julia_type(ctx, cmp, jltype); + if (intcast) { + emit_unbox_store(ctx, cmp, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + Compare = ctx.builder.CreateLoad(realelty, intcast); + } + else { + Compare = emit_unbox(ctx, realelty, cmp, jltype); + } if (realelty != elty) Compare = ctx.builder.CreateZExt(Compare, elty); } @@ -2270,16 +2285,17 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (realelty != elty) realCompare = ctx.builder.CreateTrunc(realCompare, realelty); if (intcast) { + assert(!isboxed); ctx.builder.CreateStore(realCompare, intcast); - if (maybe_null_if_boxed) - realCompare = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); + if (tracked_pointers) + realCompare = ctx.builder.CreateLoad(intcast_eltyp, intcast); } - if (maybe_null_if_boxed) { - Value *first_ptr = isboxed ? Compare : extract_first_ptr(ctx, Compare); - if (first_ptr) - null_load_check(ctx, first_ptr, mod, var); + if (maybe_null_if_boxed && tracked_pointers) { + Value *first_ptr = isboxed ? realCompare : extract_first_ptr(ctx, realCompare); + assert(first_ptr); + null_load_check(ctx, first_ptr, mod, var); } - if (intcast) + if (intcast && !tracked_pointers) oldval = mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); else oldval = mark_julia_type(ctx, realCompare, isboxed, jltype); @@ -2287,11 +2303,18 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, if (isboxed) { r = boxed(ctx, rhs); } - else if (Order != AtomicOrdering::NotAtomic || CountTrackedPointers(realelty).count) { + else if (intcast) { + emit_unbox_store(ctx, rhs, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + r = ctx.builder.CreateLoad(realelty, intcast); + if (!tracked_pointers) // oldval is a slot, so put the oldval back + ctx.builder.CreateStore(realCompare, intcast); + } + else if (Order != AtomicOrdering::NotAtomic) { + assert(!tracked_pointers); r = emit_unbox(ctx, realelty, rhs, jltype); - if (realelty != elty) - r = ctx.builder.CreateZExt(r, elty); } + if (realelty != elty) + r = ctx.builder.CreateZExt(r, elty); if (needlock) emit_lockstate_value(ctx, needlock, true); cmp = oldval; @@ -2329,7 +2352,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } else { assert(!isboxed && rhs.typ == jltype); - emit_unbox_store(ctx, rhs, ptr, tbaa, alignment); + emit_unbox_store(ctx, rhs, ptr, tbaa, Align(alignment)); } ctx.builder.CreateBr(DoneBB); instr = load; @@ -2357,9 +2380,10 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, realinstr = ctx.builder.CreateTrunc(realinstr, realelty); if (intcast) { ctx.builder.CreateStore(realinstr, intcast); + // n.b. this oldval is only used for emit_f_is in this branch, so we know a priori that it does not need a gc-root oldval = mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); if (maybe_null_if_boxed) - realinstr = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); + realinstr = ctx.builder.CreateLoad(intcast_eltyp, intcast); } else { oldval = mark_julia_type(ctx, realinstr, isboxed, jltype); @@ -2399,20 +2423,23 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ctx.builder.SetInsertPoint(DoneBB); if (needlock) emit_lockstate_value(ctx, needlock, false); - if (parent != NULL) { + if (parent != NULL && r && tracked_pointers && (!isboxed || !type_is_permalloc(rhs.typ))) { if (isreplacefield || issetfieldonce) { - // TODO: avoid this branch if we aren't making a write barrier BasicBlock *BB = BasicBlock::Create(ctx.builder.getContext(), "xchg_wb", ctx.f); DoneBB = BasicBlock::Create(ctx.builder.getContext(), "done_xchg_wb", ctx.f); ctx.builder.CreateCondBr(Success, BB, DoneBB); ctx.builder.SetInsertPoint(BB); } - if (r) { - if (!isboxed) - emit_write_multibarrier(ctx, parent, r, rhs.typ); - else if (!type_is_permalloc(rhs.typ)) - emit_write_barrier(ctx, parent, r); + if (realelty != elty) + r = ctx.builder.Insert(CastInst::Create(Instruction::Trunc, r, realelty)); + if (intcast) { + ctx.builder.CreateStore(r, intcast); + r = ctx.builder.CreateLoad(intcast_eltyp, intcast); } + if (!isboxed) + emit_write_multibarrier(ctx, parent, r, rhs.typ); + else if (!type_is_permalloc(rhs.typ)) + emit_write_barrier(ctx, parent, r); if (isreplacefield || issetfieldonce) { ctx.builder.CreateBr(DoneBB); ctx.builder.SetInsertPoint(DoneBB); @@ -2431,21 +2458,18 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, instr = ctx.builder.Insert(CastInst::Create(Instruction::Trunc, instr, realelty)); if (intcast) { ctx.builder.CreateStore(instr, intcast); - instr = nullptr; + if (tracked_pointers) + instr = ctx.builder.CreateLoad(intcast_eltyp, intcast); } - if (maybe_null_if_boxed) { - if (intcast) - instr = ctx.builder.CreateLoad(intcast->getAllocatedType(), intcast); + if (maybe_null_if_boxed && tracked_pointers) { Value *first_ptr = isboxed ? instr : extract_first_ptr(ctx, instr); - if (first_ptr) - null_load_check(ctx, first_ptr, mod, var); - if (intcast && !first_ptr) - instr = nullptr; + assert(first_ptr); + null_load_check(ctx, first_ptr, mod, var); } - if (instr) - oldval = mark_julia_type(ctx, instr, isboxed, jltype); - else + if (intcast && !tracked_pointers) oldval = mark_julia_slot(intcast, jltype, NULL, ctx.tbaa().tbaa_stack); + else + oldval = mark_julia_type(ctx, instr, isboxed, jltype); if (isreplacefield) { Success = ctx.builder.CreateZExt(Success, getInt8Ty(ctx.builder.getContext())); const jl_cgval_t argv[2] = {oldval, mark_julia_type(ctx, Success, false, jl_bool_type)}; @@ -2668,7 +2692,7 @@ static jl_cgval_t emit_unionload(jl_codectx_t &ctx, Value *addr, Value *ptindex, if (al > 1) lv->setAlignment(Align(al)); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); - emit_memcpy(ctx, lv, ai, addr, ai, fsz, al, al); + emit_memcpy(ctx, lv, ai, addr, ai, fsz, Align(al), Align(al)); addr = lv; } return mark_julia_slot(fsz > 0 ? addr : nullptr, jfty, tindex, tbaa); @@ -3104,11 +3128,11 @@ static Value *emit_genericmemoryowner(jl_codectx_t &ctx, Value *t) static Value *emit_allocobj(jl_codectx_t &ctx, jl_datatype_t *jt, bool fully_initialized); static void init_bits_value(jl_codectx_t &ctx, Value *newv, Value *v, MDNode *tbaa, - unsigned alignment = sizeof(void*)) // min alignment in julia's gc is pointer-aligned + Align alignment = Align(sizeof(void*))) // min alignment in julia's gc is pointer-aligned { jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa); // newv should already be tagged - ai.decorateInst(ctx.builder.CreateAlignedStore(v, newv, Align(alignment))); + ai.decorateInst(ctx.builder.CreateAlignedStore(v, newv, alignment)); } static void init_bits_cgval(jl_codectx_t &ctx, Value *newv, const jl_cgval_t& v, MDNode *tbaa) @@ -3116,7 +3140,7 @@ static void init_bits_cgval(jl_codectx_t &ctx, Value *newv, const jl_cgval_t& v, // newv should already be tagged if (v.ispointer()) { unsigned align = std::max(julia_alignment(v.typ), (unsigned)sizeof(void*)); - emit_memcpy(ctx, newv, jl_aliasinfo_t::fromTBAA(ctx, tbaa), v, jl_datatype_size(v.typ), align, julia_alignment(v.typ)); + emit_memcpy(ctx, newv, jl_aliasinfo_t::fromTBAA(ctx, tbaa), v, jl_datatype_size(v.typ), Align(align), Align(julia_alignment(v.typ))); } else { init_bits_value(ctx, newv, v.V, tbaa); @@ -3582,7 +3606,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con if (jl_is_pointerfree(typ)) { unsigned alignment = julia_alignment(typ); if (!src.ispointer() || src.constant) { - emit_unbox_store(ctx, src, dest, tbaa_dst, alignment, isVolatile); + emit_unbox_store(ctx, src, dest, tbaa_dst, Align(alignment), isVolatile); } else { Value *src_ptr = data_pointer(ctx, src); @@ -3592,7 +3616,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con // if (skip) src_ptr = ctx.builder.CreateSelect(skip, dest, src_ptr); auto f = [&] { (void)emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src_ptr, - jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, alignment, alignment, isVolatile); + jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, Align(alignment), Align(alignment), isVolatile); return nullptr; }; if (skip) @@ -3627,7 +3651,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con return; } else { emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), src_ptr, - jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, alignment, alignment, isVolatile); + jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), nb, Align(alignment), Align(alignment), isVolatile); } } ctx.builder.CreateBr(postBB); @@ -3653,7 +3677,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con Value *datatype = emit_typeof(ctx, src, false, false); Value *copy_bytes = emit_datatype_size(ctx, datatype); (void)emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dst), data_pointer(ctx, src), - jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), copy_bytes, 1, 1, isVolatile); + jl_aliasinfo_t::fromTBAA(ctx, src.tbaa), copy_bytes, Align(1), Align(1), isVolatile); return nullptr; }; if (skip) @@ -4046,7 +4070,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg } else if (init_as_value) { fval = emit_unbox(ctx, fty, fval_info, jtype); } else { - emit_unbox_store(ctx, fval_info, dest, ctx.tbaa().tbaa_stack, jl_field_align(sty, i)); + emit_unbox_store(ctx, fval_info, dest, ctx.tbaa().tbaa_stack, Align(jl_field_align(sty, i))); } } if (init_as_value) { diff --git a/src/codegen.cpp b/src/codegen.cpp index 3a363dd3fd0c1..6551f13ea566e 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3374,11 +3374,14 @@ static size_t emit_masked_bits_compare(callback &emit_desc, jl_datatype_t *aty, size_t padding_bytes = 0; size_t nfields = jl_datatype_nfields(aty); size_t total_size = jl_datatype_size(aty); + assert(aty->layout->flags.isbitsegal); for (size_t i = 0; i < nfields; ++i) { size_t offset = jl_field_offset(aty, i); size_t fend = i == nfields - 1 ? total_size : jl_field_offset(aty, i + 1); size_t fsz = jl_field_size(aty, i); jl_datatype_t *fty = (jl_datatype_t*)jl_field_type(aty, i); + assert(jl_is_datatype(fty)); // union fields should never reach here + assert(fty->layout->flags.isbitsegal); if (jl_field_isptr(aty, i) || !fty->layout->flags.haspadding) { // The field has no internal padding data_bytes += fsz; @@ -5652,7 +5655,7 @@ static jl_cgval_t emit_varinfo(jl_codectx_t &ctx, jl_varinfo_t &vi, jl_sym_t *va else { const DataLayout &DL = jl_Module->getDataLayout(); uint64_t sz = DL.getTypeStoreSize(T); - emit_memcpy(ctx, ssaslot, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), vi.value, sz, ssaslot->getAlign().value(), varslot->getAlign().value()); + emit_memcpy(ctx, ssaslot, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), vi.value, sz, ssaslot->getAlign(), varslot->getAlign()); } Value *tindex = NULL; if (vi.pTIndex) @@ -5756,8 +5759,9 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu // This check should probably mostly catch the relevant situations. if (vi.value.V != rval_info.V) { Value *copy_bytes = ConstantInt::get(getInt32Ty(ctx.builder.getContext()), jl_datatype_size(vi.value.typ)); + Align alignment(julia_alignment(rval_info.typ)); emit_memcpy(ctx, vi.value.V, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), rval_info, copy_bytes, - julia_alignment(rval_info.typ), julia_alignment(rval_info.typ), vi.isVolatile); + alignment, alignment, vi.isVolatile); } } else { @@ -6868,8 +6872,9 @@ static void emit_cfunc_invalidate( root1 = ctx.builder.CreateConstInBoundsGEP2_32(get_returnroots_type(ctx, return_roots), root1, 0, 0); ctx.builder.CreateStore(gf_ret, root1); } + Align alignment(julia_alignment(rettype)); emit_memcpy(ctx, &*gf_thunk->arg_begin(), jl_aliasinfo_t::fromTBAA(ctx, nullptr), gf_ret, - jl_aliasinfo_t::fromTBAA(ctx, nullptr), jl_datatype_size(rettype), julia_alignment(rettype), julia_alignment(rettype)); + jl_aliasinfo_t::fromTBAA(ctx, nullptr), jl_datatype_size(rettype), Align(alignment), Align(alignment)); ctx.builder.CreateRetVoid(); break; } @@ -8729,7 +8734,7 @@ static jl_llvm_functions_t jl_cgval_t closure_world = typed_load(ctx, worldaddr, NULL, (jl_value_t*)jl_long_type, nullptr, nullptr, false, AtomicOrdering::NotAtomic, false, ctx.types().alignof_ptr.value()); ctx.world_age_at_entry = closure_world.V; // The tls world in a OC is the world of the closure - emit_unbox_store(ctx, closure_world, world_age_field, ctx.tbaa().tbaa_gcframe, ctx.types().alignof_ptr.value()); + emit_unbox_store(ctx, closure_world, world_age_field, ctx.tbaa().tbaa_gcframe, ctx.types().alignof_ptr); // Load closure env Value *envaddr = ctx.builder.CreateInBoundsGEP( @@ -9272,8 +9277,9 @@ static jl_llvm_functions_t } if (returninfo.cc == jl_returninfo_t::SRet) { assert(jl_is_concrete_type(jlrettype)); + Align alignment(julia_alignment(jlrettype)); emit_memcpy(ctx, sret, jl_aliasinfo_t::fromTBAA(ctx, nullptr), retvalinfo, - jl_datatype_size(jlrettype), julia_alignment(jlrettype), julia_alignment(jlrettype)); + jl_datatype_size(jlrettype), alignment, alignment); } else { // must be jl_returninfo_t::Union emit_unionmove(ctx, sret, nullptr, retvalinfo, /*skip*/isboxed_union); @@ -9511,7 +9517,7 @@ static jl_llvm_functions_t // load of val) if the runtime type of val isn't phiType Value *isvalid = emit_isa_and_defined(ctx, val, phiType); emit_guarded_test(ctx, isvalid, nullptr, [&] { - emit_unbox_store(ctx, update_julia_type(ctx, val, phiType), dest, ctx.tbaa().tbaa_stack, julia_alignment(phiType)); + emit_unbox_store(ctx, update_julia_type(ctx, val, phiType), dest, ctx.tbaa().tbaa_stack, Align(julia_alignment(phiType))); return nullptr; }); } @@ -9538,7 +9544,7 @@ static jl_llvm_functions_t if (VN) V = Constant::getNullValue(ctx.types().T_prjlvalue); if (dest) - emit_unbox_store(ctx, val, dest, ctx.tbaa().tbaa_stack, julia_alignment(val.typ)); + emit_unbox_store(ctx, val, dest, ctx.tbaa().tbaa_stack, Align(julia_alignment(val.typ))); RTindex = ConstantInt::get(getInt8Ty(ctx.builder.getContext()), tindex); } } diff --git a/src/datatype.c b/src/datatype.c index 8a7b256db4bef..5d35a7b55057d 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -1254,7 +1254,7 @@ JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y /* pre- } else if (nb == 1) { uint8_t *y8 = (uint8_t*)y; - assert(!dt->layout->flags.haspadding); + assert(dt->layout->flags.isbitsegal && !dt->layout->flags.haspadding); if (dt == et) { *y8 = *(uint8_t*)expected; uint8_t z8 = *(uint8_t*)src; @@ -1267,7 +1267,7 @@ JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y /* pre- } else if (nb == 2) { uint16_t *y16 = (uint16_t*)y; - assert(!dt->layout->flags.haspadding); + assert(dt->layout->flags.isbitsegal && !dt->layout->flags.haspadding); if (dt == et) { *y16 = *(uint16_t*)expected; uint16_t z16 = *(uint16_t*)src; @@ -1285,7 +1285,7 @@ JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y /* pre- uint32_t z32 = zext_read32(src, nb); while (1) { success = jl_atomic_cmpswap((_Atomic(uint32_t)*)dst, y32, z32); - if (success || !dt->layout->flags.haspadding || !jl_egal__bits(y, expected, dt)) + if (success || (dt->layout->flags.isbitsegal && !dt->layout->flags.haspadding) || !jl_egal__bits(y, expected, dt)) break; } } @@ -1302,7 +1302,7 @@ JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y /* pre- uint64_t z64 = zext_read64(src, nb); while (1) { success = jl_atomic_cmpswap((_Atomic(uint64_t)*)dst, y64, z64); - if (success || !dt->layout->flags.haspadding || !jl_egal__bits(y, expected, dt)) + if (success || (dt->layout->flags.isbitsegal && !dt->layout->flags.haspadding) || !jl_egal__bits(y, expected, dt)) break; } } @@ -1320,7 +1320,7 @@ JL_DLLEXPORT int jl_atomic_cmpswap_bits(jl_datatype_t *dt, jl_value_t *y /* pre- jl_uint128_t z128 = zext_read128(src, nb); while (1) { success = jl_atomic_cmpswap((_Atomic(jl_uint128_t)*)dst, y128, z128); - if (success || !dt->layout->flags.haspadding || !jl_egal__bits(y, expected, dt)) + if (success || (dt->layout->flags.isbitsegal && !dt->layout->flags.haspadding) || !jl_egal__bits(y, expected, dt)) break; } } @@ -2014,7 +2014,7 @@ inline jl_value_t *modify_bits(jl_value_t *ty, char *p, uint8_t *psel, jl_value_ else { char *px = lock(p, parent, needlock, isatomic); int success = memcmp(px, (char*)r, fsz) == 0; - if (!success && ((jl_datatype_t*)rty)->layout->flags.haspadding) + if (!success && (!((jl_datatype_t*)rty)->layout->flags.isbitsegal || ((jl_datatype_t*)rty)->layout->flags.haspadding)) success = jl_egal__bits((jl_value_t*)px, r, (jl_datatype_t*)rty); if (success) { if (isunion) { @@ -2129,7 +2129,7 @@ inline jl_value_t *replace_bits(jl_value_t *ty, char *p, uint8_t *psel, jl_value success = (rty == jl_typeof(expected)); if (success) { success = memcmp((char*)r, (char*)expected, rsz) == 0; - if (!success && ((jl_datatype_t*)rty)->layout->flags.haspadding) + if (!success && (!((jl_datatype_t*)rty)->layout->flags.isbitsegal || ((jl_datatype_t*)rty)->layout->flags.haspadding)) success = jl_egal__bits(r, expected, (jl_datatype_t*)rty); } *((uint8_t*)r + fsz) = success ? 1 : 0; diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index ad89abf6ca1a2..4bfe3f184d24b 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -405,7 +405,7 @@ static Value *emit_unboxed_coercion(jl_codectx_t &ctx, Type *to, Value *unboxed) } else if (!ty->isIntOrPtrTy() && !ty->isFloatingPointTy()) { assert(DL.getTypeSizeInBits(ty) == DL.getTypeSizeInBits(to)); - AllocaInst *cast = ctx.builder.CreateAlloca(ty); + AllocaInst *cast = emit_static_alloca(ctx, ty); setName(ctx.emission_context, cast, "coercion"); ctx.builder.CreateStore(unboxed, cast); unboxed = ctx.builder.CreateLoad(to, cast); @@ -498,7 +498,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va } // emit code to store a raw value into a destination -static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest, MDNode *tbaa_dest, unsigned alignment, bool isVolatile) +static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest, MDNode *tbaa_dest, Align alignment, bool isVolatile) { if (x.isghost) { // this can happen when a branch yielding a different type ends @@ -510,7 +510,7 @@ static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest if (!x.ispointer()) { // already unboxed, but sometimes need conversion (e.g. f32 -> i32) assert(x.V); Value *unboxed = zext_struct(ctx, x.V); - StoreInst *store = ctx.builder.CreateAlignedStore(unboxed, dest, Align(alignment)); + StoreInst *store = ctx.builder.CreateAlignedStore(unboxed, dest, alignment); store->setVolatile(isVolatile); jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa_dest); ai.decorateInst(store); @@ -518,7 +518,7 @@ static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest } Value *src = data_pointer(ctx, x); - emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dest), src, jl_aliasinfo_t::fromTBAA(ctx, x.tbaa), jl_datatype_size(x.typ), alignment, julia_alignment(x.typ), isVolatile); + emit_memcpy(ctx, dest, jl_aliasinfo_t::fromTBAA(ctx, tbaa_dest), src, jl_aliasinfo_t::fromTBAA(ctx, x.tbaa), jl_datatype_size(x.typ), Align(alignment), Align(julia_alignment(x.typ)), isVolatile); } static jl_datatype_t *staticeval_bitstype(const jl_cgval_t &targ) @@ -770,7 +770,7 @@ static jl_cgval_t emit_pointerref(jl_codectx_t &ctx, ArrayRef argv) thePtr = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), thePtr, im1); setName(ctx.emission_context, thePtr, "pointerref_src"); MDNode *tbaa = best_tbaa(ctx.tbaa(), ety); - emit_memcpy(ctx, strct, jl_aliasinfo_t::fromTBAA(ctx, tbaa), thePtr, jl_aliasinfo_t::fromTBAA(ctx, nullptr), size, sizeof(jl_value_t*), align_nb); + emit_memcpy(ctx, strct, jl_aliasinfo_t::fromTBAA(ctx, tbaa), thePtr, jl_aliasinfo_t::fromTBAA(ctx, nullptr), size, Align(sizeof(jl_value_t*)), Align(align_nb)); return mark_julia_type(ctx, strct, true, ety); } else { @@ -799,7 +799,7 @@ static jl_cgval_t emit_runtime_pointerset(jl_codectx_t &ctx, ArrayRef argv) { const jl_cgval_t &e = argv[0]; - const jl_cgval_t &x = argv[1]; + jl_cgval_t x = argv[1]; const jl_cgval_t &i = argv[2]; const jl_cgval_t &align = argv[3]; @@ -822,6 +822,9 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, ArrayRef argv) return jl_cgval_t(); } emit_typecheck(ctx, x, ety, "pointerset"); + x = update_julia_type(ctx, x, ety); + if (x.typ == jl_bottom_type) + return jl_cgval_t(); Value *idx = emit_unbox(ctx, ctx.types().T_size, i, (jl_value_t*)jl_long_type); Value *im1 = ctx.builder.CreateSub(idx, ConstantInt::get(ctx.types().T_size, 1)); @@ -847,7 +850,7 @@ static jl_cgval_t emit_pointerset(jl_codectx_t &ctx, ArrayRef argv) setName(ctx.emission_context, im1, "pointerset_offset"); auto gep = ctx.builder.CreateInBoundsGEP(getInt8Ty(ctx.builder.getContext()), thePtr, im1); setName(ctx.emission_context, gep, "pointerset_ptr"); - emit_memcpy(ctx, gep, jl_aliasinfo_t::fromTBAA(ctx, nullptr), x, size, align_nb, julia_alignment(ety)); + emit_memcpy(ctx, gep, jl_aliasinfo_t::fromTBAA(ctx, nullptr), x, size, Align(align_nb), Align(julia_alignment(ety))); } else { bool isboxed; @@ -992,7 +995,7 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, ArrayRef bool ismodifyfield = f == atomic_pointermodify; const jl_cgval_t undefval; const jl_cgval_t &e = argv[0]; - const jl_cgval_t &x = isreplacefield || ismodifyfield ? argv[2] : argv[1]; + jl_cgval_t x = isreplacefield || ismodifyfield ? argv[2] : argv[1]; const jl_cgval_t &y = isreplacefield || ismodifyfield ? argv[1] : undefval; const jl_cgval_t &ord = isreplacefield || ismodifyfield ? argv[3] : argv[2]; const jl_cgval_t &failord = isreplacefield ? argv[4] : undefval; @@ -1034,8 +1037,12 @@ static jl_cgval_t emit_atomic_pointerop(jl_codectx_t &ctx, intrinsic f, ArrayRef emit_error(ctx, msg); return jl_cgval_t(); } - if (!ismodifyfield) + if (!ismodifyfield) { emit_typecheck(ctx, x, ety, std::string(jl_intrinsic_name((int)f))); + x = update_julia_type(ctx, x, ety); + if (x.typ == jl_bottom_type) + return jl_cgval_t(); + } size_t nb = jl_datatype_size(ety); if ((nb & (nb - 1)) != 0 || nb > MAX_POINTERATOMIC_SIZE) { diff --git a/test/atomics.jl b/test/atomics.jl index 3df9e7d0f63c0..165c8dc4e2dfc 100644 --- a/test/atomics.jl +++ b/test/atomics.jl @@ -129,6 +129,7 @@ test_field_operators(ARefxy{Any}(123_10, 123_20)) test_field_operators(ARefxy{Union{Nothing,Int}}(123_10, nothing)) test_field_operators(ARefxy{Complex{Int32}}(123_10, 123_20)) test_field_operators(ARefxy{Complex{Int128}}(123_10, 123_20)) +test_field_operators(ARefxy{Complex{Real}}(123_10, 123_20)) test_field_operators(ARefxy{PadIntA}(123_10, 123_20)) test_field_operators(ARefxy{PadIntB}(123_10, 123_20)) #FIXME: test_field_operators(ARefxy{Int24}(123_10, 123_20)) @@ -317,6 +318,7 @@ test_field_orderings(ARefxy{Any}(true, false), true, false) test_field_orderings(ARefxy{Union{Nothing,Missing}}(nothing, missing), nothing, missing) test_field_orderings(ARefxy{Union{Nothing,Int}}(nothing, 123_1), nothing, 123_1) test_field_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) +test_field_orderings(Complex{Real}(10, 30), Complex{Real}(20, 40)) test_field_orderings(10.0, 20.0) test_field_orderings(NaN, Inf) @@ -568,6 +570,7 @@ test_global_operators(Any) test_global_operators(Union{Nothing,Int}) test_global_operators(Complex{Int32}) test_global_operators(Complex{Int128}) +test_global_operators(Complex{Real}) test_global_operators(PadIntA) test_global_operators(PadIntB) #FIXME: test_global_operators(Int24) @@ -691,6 +694,7 @@ test_global_orderings(Any, true, false) test_global_orderings(Union{Nothing,Missing}, nothing, missing) test_global_orderings(Union{Nothing,Int}, nothing, 123_1) test_global_orderings(Complex{Int128}, Complex{Int128}(10, 30), Complex{Int128}(20, 40)) +test_global_orderings(Complex{Real}, Complex{Real}(10, 30), Complex{Real}(20, 40)) test_global_orderings(Float64, 10.0, 20.0) test_global_orderings(Float64, NaN, Inf) @@ -844,6 +848,7 @@ test_memory_operators(Any) test_memory_operators(Union{Nothing,Int}) test_memory_operators(Complex{Int32}) test_memory_operators(Complex{Int128}) +test_memory_operators(Complex{Real}) test_memory_operators(PadIntA) test_memory_operators(PadIntB) #FIXME: test_memory_operators(Int24) @@ -1031,6 +1036,7 @@ test_memory_orderings(Any, true, false) test_memory_orderings(Union{Nothing,Missing}, nothing, missing) test_memory_orderings(Union{Nothing,Int}, nothing, 123_1) test_memory_orderings(Complex{Int128}(10, 30), Complex{Int128}(20, 40)) +test_memory_orderings(Complex{Real}(10, 30), Complex{Real}(20, 40)) test_memory_orderings(10.0, 20.0) test_memory_orderings(NaN, Inf)