Skip to content

Commit

Permalink
Wait for assets to be processed (o3de#400)
Browse files Browse the repository at this point in the history 
Signed-off-by: Artur Kamieniecki <[email protected]>
Signed-off-by: Artur Kamieniecki <[email protected]>
  • Loading branch information
@arturkamieniecki @Antoni-Robotec
arturkamieniecki authored and Antoni-Robotec committed Jul 24, 2023
1 parent ca4c8e4 commit 8f0973d
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 54 deletions.
71 changes: 71 additions & 0 deletions Gems/ROS2/Code/Source/RobotImporter/Utils/RobotImporterUtils.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,84 @@
#include <AzCore/Asset/AssetManager.h>
#include <AzCore/Asset/AssetManagerBus.h>
#include <AzCore/IO/Path/Path.h>
#include <AzCore/Settings/SettingsRegistry.h>
#include <AzCore/StringFunc/StringFunc.h>
#include <AzCore/std/string/regex.h>
#include <AzToolsFramework/API/EditorAssetSystemAPI.h>

namespace ROS2
{

bool Utils::WaitForAssetsToProcess(const AZStd::unordered_map<AZStd::string, AZ::IO::Path>& sourceAssetsPaths)
{
bool allAssetProcessed = false;
bool assetProcessorFailed = false;
auto loopStartTime = AZStd::chrono::system_clock::now();

AZStd::chrono::seconds assetJobTimeout = AZStd::chrono::seconds(30);
auto settingsRegistry = AZ::SettingsRegistry::Get();
AZ::s64 loopTimeoutValue;
if (settingsRegistry->Get(loopTimeoutValue, "/O3DE/ROS2/RobotImporter/AssetProcessorTimeoutInSeconds"))
{
assetJobTimeout = AZStd::chrono::seconds(loopTimeoutValue);
}

/* This loop waits until all of the assets are processed.
There are three stop conditions: allAssetProcessed, assetProcessorFailed and a timeout.
After all asset are processed the allAssetProcessed will be set to true.
assetProcessorFailed will be set to true if the asset processor does not respond.
The time out will break the loop if assetLoopTimeout is exceed. */
while (!allAssetProcessed && !assetProcessorFailed)
{
auto loopTime = AZStd::chrono::system_clock::now();

if (loopTime - loopStartTime > assetJobTimeout)
{
AZ_Warning("RobotImporterUtils", false, "Loop waiting for assets timed out");
break;
}

allAssetProcessed = true;
for (const auto& [AssetFileName, AssetFilePath] : sourceAssetsPaths)
{
if (AssetFilePath.empty())
{
AZ_Warning("RobotImporterUtils", false, "WaitForAssetsToProcess got an empty filepath ");
continue;
}
using namespace AzToolsFramework;
using namespace AzToolsFramework::AssetSystem;
AZ::Outcome<AssetSystem::JobInfoContainer> result = AZ::Failure();
AssetSystemJobRequestBus::BroadcastResult(
result, &AssetSystemJobRequestBus::Events::GetAssetJobsInfo, AssetFilePath.String(), true);

if (!result.IsSuccess())
{
assetProcessorFailed = true;
AZ_Error("RobotImporterUtils", false, "Asset System failed to reply with jobs infos");
break;
}

JobInfoContainer& allJobs = result.GetValue();
for (const JobInfo& job : allJobs)
{
if (job.m_status == JobStatus::Queued || job.m_status == JobStatus::InProgress)
{
allAssetProcessed = false;
}
}
}

if (allAssetProcessed && !assetProcessorFailed)
{
AZ_Printf("RobotImporterUtils", "All assets processed");
return true;
}
}

return false;
}

bool Utils::IsWheelURDFHeuristics(const urdf::LinkConstSharedPtr& link)
{
const AZStd::regex wheel_regex("wheel[_]||[_]wheel");
Expand Down
7 changes: 7 additions & 0 deletions Gems/ROS2/Code/Source/RobotImporter/Utils/RobotImporterUtils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,5 +70,12 @@ namespace ROS2
const AZStd::string& amentPrefixPath,
const AZStd::function<bool(const AZStd::string&)>& fileExists = FileExistsCall);

//! Waits for asset processor to process provided assets.
//! This function will timeout after the time specified in /O3DE/ROS2/RobotImporter/AssetProcessorTimeoutInSeconds
//! settings registry.
//! @param sourceAssetsPaths - set of all non relative paths to assets for which we want to wait for processing
//! @returns false if a timeout or error occurs, otherwise true
bool WaitForAssetsToProcess(const AZStd::unordered_map<AZStd::string, AZ::IO::Path>& sourceAssetsPaths);

} // namespace Utils
} // namespace ROS2
79 changes: 25 additions & 54 deletions Gems/ROS2/Code/Source/RobotImporter/Utils/SourceAssetsStorage.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,29 +127,15 @@ namespace ROS2::Utils
using AssetSysReqBus = AzToolsFramework::AssetSystemRequestBus;
AvailableAsset foundAsset;

// get relativePath
bool relativePathFound{ false };
AZStd::string relativeSourcePath;
AZStd::string rootFilePath;
AssetSysReqBus::BroadcastResult(
relativePathFound,
&AssetSysReqBus::Events::GenerateRelativeSourcePath,
globalSourceAssetPath,
relativeSourcePath,
rootFilePath);
if (!relativePathFound)
{
return foundAsset;
}

// get source asset info
bool sourceAssetFound{ false };
AZ::Data::AssetInfo assetInfo;
AZStd::string watchFolder;
AssetSysReqBus::BroadcastResult(
sourceAssetFound, &AssetSysReqBus::Events::GetSourceInfoBySourcePath, relativeSourcePath.c_str(), assetInfo, watchFolder);
sourceAssetFound, &AssetSysReqBus::Events::GetSourceInfoBySourcePath, globalSourceAssetPath.c_str(), assetInfo, watchFolder);
if (!sourceAssetFound)
{
AZ_Trace("GetAvailableAssetInfo", "Source asset info not found for: %s", globalSourceAssetPath.c_str());
return foundAsset;
}

Expand Down Expand Up @@ -274,20 +260,16 @@ namespace ROS2::Utils
AZ::Crc32 urdfFileCrc;
urdfFileCrc.Add(urdfFilename);
const AZ::IO::Path urdfPath(urdfFilename);
const AZStd::string directoryNameTmp = AZStd::string::format("$tmp_%u.tmp", AZ::u32(urdfFileCrc));

const auto directoryNameDst = AZ::IO::FixedMaxPathString::format(
"%u_%.*s%.*s", AZ::u32(urdfFileCrc), AZ_PATH_ARG(urdfPath.Stem()), AZ_STRING_ARG(outputDirSuffix));

const AZ::IO::Path importDirectoryTmp = AZ::IO::Path(AZ::Utils::GetProjectPath()) / "Assets" / "UrdfImporter" / directoryNameTmp;
const AZ::IO::Path importDirectoryDst = AZ::IO::Path(AZ::Utils::GetProjectPath()) / "Assets" / "UrdfImporter" / directoryNameDst;

fileIO->DestroyPath(importDirectoryTmp.c_str());
const auto outcomeCreateDstDir = fileIO->CreatePath(importDirectoryDst.c_str());
const auto outcomeCreateTmpDir = fileIO->CreatePath(importDirectoryTmp.c_str());
AZ_Error("CopyAssetForURDF", outcomeCreateDstDir, "Cannot create destination directory : %s", importDirectoryDst.c_str());
AZ_Error("CopyAssetForURDF", outcomeCreateTmpDir, "Cannot create temporary directory : %s", importDirectoryTmp.c_str());

if (!outcomeCreateDstDir || !outcomeCreateTmpDir)
if (!outcomeCreateDstDir)
{
return urdfAssetMap;
}
Expand All @@ -304,46 +286,26 @@ namespace ROS2::Utils
}

AZ::IO::Path resolvedPath(resolved);
const bool needsVisual = visuals.contains(unresolvedUrfFileName);
const bool needsCollider = colliders.contains(unresolvedUrfFileName);

const AZStd::string pathSuffix = needsCollider ? "_collider" : "";
const AZStd::string newFilename = resolvedPath.Stem().String() + pathSuffix + resolvedPath.Extension().String();
AZ::IO::Path targetPathAssetDst(importDirectoryDst / newFilename);
AZ::IO::Path targetPathAssetTmp(importDirectoryTmp / newFilename);

AZ::IO::Path targetPathAssetInfo(targetPathAssetDst.Native() + ".assetinfo");

if (!fileIO->Exists(targetPathAssetDst.c_str()))
{
// Copy from ROS2 install to temporary
const auto outcomeCopyTmp = fileIO->Copy(resolvedPath.c_str(), targetPathAssetTmp.c_str());
const auto outcomeMoveDst = fileIO->Copy(resolvedPath.c_str(), targetPathAssetDst.c_str());
AZ_Printf(
"CopyAssetForURDF",
"Copy %s to %s, result: %d",
resolvedPath.c_str(),
targetPathAssetTmp.c_str(),
outcomeCopyTmp.GetResultCode());
if (outcomeCopyTmp)
targetPathAssetDst.c_str(),
outcomeMoveDst.GetResultCode());
if (outcomeMoveDst)
{
// create asset info at destination
const bool assetInfoOk =
CreateSceneManifest(targetPathAssetTmp.String(), targetPathAssetInfo.String(), needsCollider, needsVisual);

if (assetInfoOk)
{
const auto outcomeMoveDst = fileIO->Copy(targetPathAssetTmp.c_str(), targetPathAssetDst.c_str());
AZ_Printf(
"CopyAssetForURDF",
"Move %s to %s, result: %d",
targetPathAssetTmp.c_str(),
targetPathAssetDst.c_str(),
outcomeMoveDst.GetResultCode());
if (outcomeMoveDst)
{
copiedFiles[unresolvedUrfFileName] = targetPathAssetDst.String();
}
};
copiedFiles[unresolvedUrfFileName] = targetPathAssetDst.String();
}
}
else
Expand All @@ -359,20 +321,29 @@ namespace ROS2::Utils
urdfAssetMap.emplace(unresolvedUrfFileName, AZStd::move(asset));
}

fileIO->DestroyPath(importDirectoryTmp.c_str());
for (const auto& copied : copiedFiles)
{
AZ_Printf("CopyAssetForURDF", " %s is copied to %s", copied.first.c_str(), copied.second.c_str());
}

// add available asset info
// Wait for assets to be scanned by asset processor
Utils::WaitForAssetsToProcess(copiedFiles);
// Add available asset info
for (const auto& [unresolvedUrfFileName, sourceAssetGlobalPath] : copiedFiles)
{
AZ_Printf("CopyAssetForURDF", " %s is copied to %s", unresolvedUrfFileName.c_str(), sourceAssetGlobalPath.c_str());
AZ_Assert(
urdfAssetMap.contains(unresolvedUrfFileName), "urdfAssetMap should contain urdf path %s", unresolvedUrfFileName.c_str());
urdfAssetMap[unresolvedUrfFileName].m_availableAssetInfo = Utils::GetAvailableAssetInfo(sourceAssetGlobalPath.String());
}

// Create assetinfo
for (const auto& [unresolvedUrfFileName, sourceAssetGlobalPath] : copiedFiles)
{
AZ::IO::Path targetPathAssetInfo(sourceAssetGlobalPath.Native() + ".assetinfo");
if (!fileIO->Exists(targetPathAssetInfo.c_str()))
{
const bool needsVisual = visuals.contains(unresolvedUrfFileName);
const bool needsCollider = colliders.contains(unresolvedUrfFileName);
CreateSceneManifest(sourceAssetGlobalPath.String(), targetPathAssetInfo.String(), needsCollider, needsVisual);
}
}

return urdfAssetMap;
}

Expand Down

0 comments on commit 8f0973d

Please sign in to comment.