diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp index 0a049a41f87b392..e690f2a3eaef18e 100644 --- a/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp +++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp @@ -573,6 +573,7 @@ void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym, clauseOps->privateSyms.push_back(mlir::SymbolRefAttr::get(privatizerOp)); clauseOps->privateVars.push_back(hsb.getAddr()); } + symToPrivatizer[sym] = privatizerOp; } diff --git a/flang/lib/Lower/OpenMP/DataSharingProcessor.h b/flang/lib/Lower/OpenMP/DataSharingProcessor.h index 96500c9215d9eca..d1d3ae8743716dc 100644 --- a/flang/lib/Lower/OpenMP/DataSharingProcessor.h +++ b/flang/lib/Lower/OpenMP/DataSharingProcessor.h @@ -88,7 +88,6 @@ class DataSharingProcessor { bool useDelayedPrivatization; bool callsInitClone = false; lower::SymMap &symTable; - OMPConstructSymbolVisitor visitor; bool privatizationDone = false; diff --git a/flang/lib/Lower/OpenMP/OpenMP.cpp b/flang/lib/Lower/OpenMP/OpenMP.cpp index bbe1fce689fe347..6337dbbae0c2e2c 100644 --- a/flang/lib/Lower/OpenMP/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP/OpenMP.cpp @@ -1102,7 +1102,6 @@ static void createBodyOfOp(mlir::Operation &op, const OpWithBodyGenInfo &info, firOpBuilder.createBlock(&op.getRegion(0)); return {}; }(); - // Mark the earliest insertion point. mlir::Operation *marker = insertMarker(firOpBuilder); @@ -1858,7 +1857,6 @@ static mlir::omp::LoopNestOp genLoopNestOp( std::pair> wrapperArgs, llvm::omp::Directive directive, DataSharingProcessor &dsp) { - auto ivCallback = [&](mlir::Operation *op) { genLoopVars(op, converter, loc, iv, wrapperArgs); return llvm::SmallVector(iv); @@ -1867,15 +1865,13 @@ static mlir::omp::LoopNestOp genLoopNestOp( auto *nestedEval = getCollapsedLoopEval(eval, getCollapseValue(item->clauses)); - auto loopNestOp = genOpWithBody( + return genOpWithBody( OpWithBodyGenInfo(converter, symTable, semaCtx, loc, *nestedEval, directive) .setClauses(&item->clauses) .setDataSharingProcessor(&dsp) .setGenRegionEntryCb(ivCallback), queue, item, clauseOps); - - return loopNestOp; } static void genLoopOp(lower::AbstractConverter &converter, @@ -2202,69 +2198,69 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable, if (!converter.getSymbolAddress(sym)) return; - if (llvm::is_contained(mapSyms, &sym)) - return; - - if (const auto *details = - sym.template detailsIf()) - converter.copySymbolBinding(details->symbol(), sym); - std::stringstream name; - fir::ExtendedValue dataExv = converter.getSymbolExtendedValue(sym); - name << sym.name().ToString(); - - lower::AddrAndBoundsInfo info = getDataOperandBaseAddr( - converter, firOpBuilder, sym, converter.getCurrentLocation()); - llvm::SmallVector bounds = - lower::genImplicitBoundsOps( - firOpBuilder, info, dataExv, - semantics::IsAssumedSizeArray(sym.GetUltimate()), - converter.getCurrentLocation()); - - llvm::omp::OpenMPOffloadMappingFlags mapFlag = - llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT; - mlir::omp::VariableCaptureKind captureKind = - mlir::omp::VariableCaptureKind::ByRef; - - mlir::Value baseOp = info.rawInput; - mlir::Type eleType = baseOp.getType(); - if (auto refType = mlir::dyn_cast(baseOp.getType())) - eleType = refType.getElementType(); - - // If a variable is specified in declare target link and if device - // type is not specified as `nohost`, it needs to be mapped tofrom - mlir::ModuleOp mod = firOpBuilder.getModule(); - mlir::Operation *op = mod.lookupSymbol(converter.mangleName(sym)); - auto declareTargetOp = - llvm::dyn_cast_if_present(op); - if (declareTargetOp && declareTargetOp.isDeclareTarget()) { - if (declareTargetOp.getDeclareTargetCaptureClause() == - mlir::omp::DeclareTargetCaptureClause::link && - declareTargetOp.getDeclareTargetDeviceType() != - mlir::omp::DeclareTargetDeviceType::nohost) { + if (!llvm::is_contained(mapSyms, &sym)) { + if (const auto *details = + sym.template detailsIf()) + converter.copySymbolBinding(details->symbol(), sym); + std::stringstream name; + fir::ExtendedValue dataExv = converter.getSymbolExtendedValue(sym); + name << sym.name().ToString(); + + lower::AddrAndBoundsInfo info = getDataOperandBaseAddr( + converter, firOpBuilder, sym, converter.getCurrentLocation()); + llvm::SmallVector bounds = + lower::genImplicitBoundsOps( + firOpBuilder, info, dataExv, + semantics::IsAssumedSizeArray(sym.GetUltimate()), + converter.getCurrentLocation()); + + llvm::omp::OpenMPOffloadMappingFlags mapFlag = + llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT; + mlir::omp::VariableCaptureKind captureKind = + mlir::omp::VariableCaptureKind::ByRef; + + mlir::Value baseOp = info.rawInput; + mlir::Type eleType = baseOp.getType(); + if (auto refType = mlir::dyn_cast(baseOp.getType())) + eleType = refType.getElementType(); + + // If a variable is specified in declare target link and if device + // type is not specified as `nohost`, it needs to be mapped tofrom + mlir::ModuleOp mod = firOpBuilder.getModule(); + mlir::Operation *op = mod.lookupSymbol(converter.mangleName(sym)); + auto declareTargetOp = + llvm::dyn_cast_if_present(op); + if (declareTargetOp && declareTargetOp.isDeclareTarget()) { + if (declareTargetOp.getDeclareTargetCaptureClause() == + mlir::omp::DeclareTargetCaptureClause::link && + declareTargetOp.getDeclareTargetDeviceType() != + mlir::omp::DeclareTargetDeviceType::nohost) { + mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO; + mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM; + } + } else if (fir::isa_trivial(eleType) || fir::isa_char(eleType)) { + captureKind = mlir::omp::VariableCaptureKind::ByCopy; + } else if (!fir::isa_builtin_cptr_type(eleType)) { mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO; mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM; } - } else if (fir::isa_trivial(eleType) || fir::isa_char(eleType)) { - captureKind = mlir::omp::VariableCaptureKind::ByCopy; - } else if (!fir::isa_builtin_cptr_type(eleType)) { - mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO; - mapFlag |= llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_FROM; + auto location = + mlir::NameLoc::get(mlir::StringAttr::get(firOpBuilder.getContext(), + sym.name().ToString()), + baseOp.getLoc()); + mlir::Value mapOp = createMapInfoOp( + firOpBuilder, location, baseOp, /*varPtrPtr=*/mlir::Value{}, + name.str(), bounds, /*members=*/{}, + /*membersIndex=*/mlir::ArrayAttr{}, + static_cast< + std::underlying_type_t>( + mapFlag), + captureKind, baseOp.getType()); + + clauseOps.mapVars.push_back(mapOp); + mapSyms.push_back(&sym); } - auto location = mlir::NameLoc::get( - mlir::StringAttr::get(firOpBuilder.getContext(), sym.name().ToString()), - baseOp.getLoc()); - mlir::Value mapOp = createMapInfoOp( - firOpBuilder, location, baseOp, /*varPtrPtr=*/mlir::Value{}, name.str(), - bounds, /*members=*/{}, - /*membersIndex=*/mlir::ArrayAttr{}, - static_cast< - std::underlying_type_t>( - mapFlag), - captureKind, baseOp.getType()); - - clauseOps.mapVars.push_back(mapOp); - mapSyms.push_back(&sym); }; lower::pft::visitAllSymbols(eval, captureImplicitMap); @@ -2272,6 +2268,7 @@ genTargetOp(lower::AbstractConverter &converter, lower::SymMap &symTable, llvm::SmallVector mapBaseValues; extractMappedBaseValues(clauseOps.mapVars, mapBaseValues); + EntryBlockArgs args; args.hostEvalVars = clauseOps.hostEvalVars; // TODO: Add in_reduction syms and vars. @@ -2471,14 +2468,12 @@ genTeamsOp(lower::AbstractConverter &converter, lower::SymMap &symTable, return llvm::to_vector(args.getSyms()); }; - auto teamsOp = genOpWithBody( + return genOpWithBody( OpWithBodyGenInfo(converter, symTable, semaCtx, loc, eval, llvm::omp::Directive::OMPD_teams) .setClauses(&item->clauses) .setGenRegionEntryCb(genRegionEntryCB), queue, item, clauseOps); - - return teamsOp; } //===----------------------------------------------------------------------===// @@ -2528,6 +2523,7 @@ static void genStandaloneDo(lower::AbstractConverter &converter, const ConstructQueue &queue, ConstructQueue::const_iterator item) { lower::StatementContext stmtCtx; + mlir::omp::WsloopOperands wsloopClauseOps; llvm::SmallVector wsloopReductionSyms; genWsloopClauses(converter, semaCtx, stmtCtx, item->clauses, loc, @@ -3449,7 +3445,6 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable, ConstructQueue queue{ buildConstructQueue(converter.getFirOpBuilder().getModule(), semaCtx, eval, source, directive, clauses)}; - genOMPDispatch(converter, symTable, semaCtx, eval, currentLocation, queue, queue.begin()); } @@ -3475,7 +3470,6 @@ static void genOMP(lower::AbstractConverter &converter, lower::SymMap &symTable, std::get(sectionsConstruct.t); clauses.append(makeClauses( std::get(endSectionsDirective.t), semaCtx)); - mlir::Location currentLocation = converter.getCurrentLocation(); llvm::omp::Directive directive = diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h index 06b2b6d6f9183c0..f59fb75dcd4fcd9 100644 --- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h +++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h @@ -2904,6 +2904,7 @@ class OpenMPIRBuilder { Function *OutlinedFunction, StringRef EntryFnName, StringRef EntryFnIDName); + /// Type of BodyGen to use for region codegen /// /// Priv: If device pointer privatization is required, emit the body of the diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 73a1e9abb9408ca..41ed7122236cd81 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -153,7 +153,6 @@ static const omp::GV &getGridValue(const Triple &T, Function *Kernel) { if (T.isAMDGPU()) { StringRef Features = Kernel->getFnAttribute("target-features").getValueAsString(); - if (Features.count("+wavefrontsize64")) return omp::getAMDGPUGridValues<64>(); return omp::getAMDGPUGridValues<32>(); @@ -1248,6 +1247,7 @@ static void targetParallelCallback( CallInst *CI = cast(OutlinedFn.user_back()); assert(CI && "Expected call instruction to outlined function"); CI->getParent()->setName("omp_parallel"); + Builder.SetInsertPoint(CI); Type *PtrTy = OMPIRBuilder->VoidPtr; Value *NullPtrValue = Constant::getNullValue(PtrTy); @@ -4371,7 +4371,6 @@ OpenMPIRBuilder::applyStaticChunkedWorkshareLoop(DebugLoc DL, IsLastChunk, CountUntilOrigTripCount, ChunkRange, "omp_chunk.tripcount"); Value *BackcastedChunkTC = Builder.CreateTrunc(ChunkTripCount, IVTy, "omp_chunk.tripcount.trunc"); - CLI->setTripCount(BackcastedChunkTC); // Update all uses of the induction variable except the one in the condition @@ -6838,44 +6837,6 @@ FunctionCallee OpenMPIRBuilder::createDispatchDeinitFunction() { return getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_dispatch_deinit); } -static void emitUsed(StringRef Name, std::vector &List, - Type *Int8PtrTy, Module &M) { - if (List.empty()) - return; - - // Convert List to what ConstantArray needs. - SmallVector UsedArray; - UsedArray.resize(List.size()); - for (unsigned i = 0, e = List.size(); i != e; ++i) { - UsedArray[i] = ConstantExpr::getPointerBitCastOrAddrSpaceCast( - cast(&*List[i]), Int8PtrTy); - } - - if (UsedArray.empty()) - return; - ArrayType *ATy = ArrayType::get(Int8PtrTy, UsedArray.size()); - - auto *GV = - new GlobalVariable(M, ATy, false, llvm::GlobalValue::AppendingLinkage, - llvm::ConstantArray::get(ATy, UsedArray), Name); - - GV->setSection("llvm.metadata"); -} - -static void -emitExecutionMode(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, - StringRef FunctionName, omp::OMPTgtExecModeFlags ExecFlags, - std::vector &LLVMCompilerUsed) { - auto Int8Ty = Type::getInt8Ty(Builder.getContext()); - auto *GVMode = - new llvm::GlobalVariable(OMPBuilder.M, Int8Ty, /*isConstant=*/true, - llvm::GlobalValue::WeakAnyLinkage, - llvm::ConstantInt::get(Int8Ty, ExecFlags), - Twine(FunctionName, "_exec_mode")); - GVMode->setVisibility(llvm::GlobalVariable::ProtectedVisibility); - LLVMCompilerUsed.emplace_back(GVMode); -} - static Value *removeASCastIfPresent(Value *V) { if (Operator::getOpcode(V) == Instruction::AddrSpaceCast) return cast(V)->getOperand(0); @@ -7580,6 +7541,7 @@ emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, Result, Clause) : Clause; }; + // If a multi-dimensional THREAD_LIMIT is set, it is the OMPX_BARE case, so // the NUM_THREADS clause is overriden by THREAD_LIMIT. SmallVector NumThreadsC; diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp index 2140375d9edc9bd..994e3a1fdff5edd 100644 --- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp +++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp @@ -2388,7 +2388,6 @@ LogicalResult DistributeOp::verifyRegions() { if (!isComposite()) return emitError() << "'omp.composite' attribute missing from composite wrapper"; - // Check for the allowed leaf constructs that may appear in a composite // construct directly after DISTRIBUTE. if (isa(nested)) { diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index 7380a8341546d94..7ddedbb3bb79585 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -3557,6 +3557,7 @@ static llvm::omp::OpenMPOffloadMappingFlags mapParentWithMembers( // runtime information on the dynamically allocated data). auto parentClause = llvm::cast(mapData.MapClause[mapDataIndex]); + llvm::Value *lowAddr, *highAddr; if (!parentClause.getPartialMap()) { lowAddr = builder.CreatePointerCast(mapData.Pointers[mapDataIndex], @@ -4141,7 +4142,6 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder, // If device info is available then region has already been generated if (info.DevicePtrInfoMap.empty()) { builder.restoreIP(codeGenIP); - // For device pass, if use_device_ptr(addr) mappings were present, // we need to link them here before codegen. if (ompBuilder->Config.IsTargetDevice.value_or(false)) { @@ -4973,7 +4973,6 @@ convertOmpTarget(Operation &opInst, llvm::IRBuilderBase &builder, return exitBlock.takeError(); builder.SetInsertPoint(*exitBlock); - if (!privateCleanupRegions.empty()) { if (failed(inlineOmpRegionCleanup( privateCleanupRegions, llvmPrivateVars, moduleTranslation, @@ -5215,6 +5214,7 @@ static bool isTargetDeviceOp(Operation *op) { static LogicalResult convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) { + llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); return llvm::TypeSwitch(op) @@ -5340,19 +5340,6 @@ convertHostOrTargetOperation(Operation *op, llvm::IRBuilderBase &builder, }); } -template -bool matchOpNest(Operation *op, FirstOpType &firstOp, RestOpTypes &...restOps) { - if ((firstOp = mlir::dyn_cast(op))) { - if constexpr (sizeof...(RestOpTypes) == 0) { - return true; - } else { - Block &innerBlock = getContainedBlock(firstOp); - return matchOpScanNest(innerBlock, restOps...); - } - } - return false; -} - static LogicalResult convertTargetDeviceOp(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) {