Skip to content

Commit

Permalink
VRT serialization: emit a warning if RAM usage of XML serialization r…
Browse files Browse the repository at this point in the history 
…eaches 80% of RAM (fixes #9212)
  • Loading branch information
@rouault
rouault committed Feb 8, 2024
1 parent ba0e386 commit 4874fd7
Show file tree
Hide file tree
Showing 8 changed files with 92 additions and 28 deletions.
11 changes: 8 additions & 3 deletions frmts/vrt/vrtdataset.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -328,10 +328,14 @@ CPLXMLNode *VRTDataset::SerializeToXML(const char *pszVRTPathIn)
{
}
CPLAssert(psLastChild); // we have at least rasterXSize
bool bHasWarnedAboutRAMUsage = false;
size_t nAccRAMUsage = 0;
for (int iBand = 0; iBand < nBands; iBand++)
{
CPLXMLNode *psBandTree = static_cast<VRTRasterBand *>(papoBands[iBand])
->SerializeToXML(pszVRTPathIn);
CPLXMLNode *psBandTree =
static_cast<VRTRasterBand *>(papoBands[iBand])
->SerializeToXML(pszVRTPathIn, bHasWarnedAboutRAMUsage,
nAccRAMUsage);

if (psBandTree != nullptr)
{
Expand All @@ -345,7 +349,8 @@ CPLXMLNode *VRTDataset::SerializeToXML(const char *pszVRTPathIn)
/* -------------------------------------------------------------------- */
if (m_poMaskBand)
{
CPLXMLNode *psBandTree = m_poMaskBand->SerializeToXML(pszVRTPathIn);
CPLXMLNode *psBandTree = m_poMaskBand->SerializeToXML(
pszVRTPathIn, bHasWarnedAboutRAMUsage, nAccRAMUsage);

if (psBandTree != nullptr)
{
Expand Down
24 changes: 18 additions & 6 deletions frmts/vrt/vrtdataset.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -565,7 +565,9 @@ class CPL_DLL VRTRasterBand CPL_NON_FINAL : public GDALRasterBand

virtual CPLErr XMLInit(CPLXMLNode *, const char *,
std::map<CPLString, GDALDataset *> &);
virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath);
virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage);

CPLErr SetNoDataValue(double) override;
CPLErr SetNoDataValueAsInt64(int64_t nNoData) override;
Expand Down Expand Up @@ -705,7 +707,9 @@ class CPL_DLL VRTSourcedRasterBand CPL_NON_FINAL : public VRTRasterBand

virtual CPLErr XMLInit(CPLXMLNode *, const char *,
std::map<CPLString, GDALDataset *> &) override;
virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override;
virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage) override;

virtual double GetMinimum(int *pbSuccess = nullptr) override;
virtual double GetMaximum(int *pbSuccess = nullptr) override;
Expand Down Expand Up @@ -804,7 +808,9 @@ class CPL_DLL VRTWarpedRasterBand final : public VRTRasterBand
GDALDataType eType = GDT_Unknown);
virtual ~VRTWarpedRasterBand();

virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override;
virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage) override;

virtual CPLErr IReadBlock(int, int, void *) override;
virtual CPLErr IWriteBlock(int, int, void *) override;
Expand All @@ -825,7 +831,9 @@ class VRTPansharpenedRasterBand final : public VRTRasterBand
GDALDataType eDataType = GDT_Unknown);
virtual ~VRTPansharpenedRasterBand();

virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override;
virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage) override;

virtual CPLErr IReadBlock(int, int, void *) override;

Expand Down Expand Up @@ -906,7 +914,9 @@ class CPL_DLL VRTDerivedRasterBand CPL_NON_FINAL : public VRTSourcedRasterBand

virtual CPLErr XMLInit(CPLXMLNode *, const char *,
std::map<CPLString, GDALDataset *> &) override;
virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override;
virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage) override;

virtual double GetMinimum(int *pbSuccess = nullptr) override;
virtual double GetMaximum(int *pbSuccess = nullptr) override;
Expand Down Expand Up @@ -947,7 +957,9 @@ class CPL_DLL VRTRawRasterBand CPL_NON_FINAL : public VRTRasterBand

virtual CPLErr XMLInit(CPLXMLNode *, const char *,
std::map<CPLString, GDALDataset *> &) override;
virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath) override;
virtual CPLXMLNode *SerializeToXML(const char *pszVRTPath,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage) override;

virtual CPLErr IRasterIO(GDALRWFlag, int, int, int, int, void *, int, int,
GDALDataType, GSpacing nPixelSpace,
Expand Down
7 changes: 5 additions & 2 deletions frmts/vrt/vrtderivedrasterband.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1487,9 +1487,12 @@ CPLErr VRTDerivedRasterBand::XMLInit(
/* SerializeToXML() */
/************************************************************************/

CPLXMLNode *VRTDerivedRasterBand::SerializeToXML(const char *pszVRTPath)
CPLXMLNode *VRTDerivedRasterBand::SerializeToXML(const char *pszVRTPath,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage)
{
CPLXMLNode *psTree = VRTSourcedRasterBand::SerializeToXML(pszVRTPath);
CPLXMLNode *psTree = VRTSourcedRasterBand::SerializeToXML(
pszVRTPath, bHasWarnedAboutRAMUsage, nAccRAMUsage);

/* -------------------------------------------------------------------- */
/* Set subclass. */
Expand Down
8 changes: 6 additions & 2 deletions frmts/vrt/vrtpansharpened.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1747,10 +1747,14 @@ CPLErr VRTPansharpenedRasterBand::IRasterIO(
/* SerializeToXML() */
/************************************************************************/

CPLXMLNode *VRTPansharpenedRasterBand::SerializeToXML(const char *pszVRTPathIn)
CPLXMLNode *
VRTPansharpenedRasterBand::SerializeToXML(const char *pszVRTPathIn,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage)

{
CPLXMLNode *psTree = VRTRasterBand::SerializeToXML(pszVRTPathIn);
CPLXMLNode *psTree = VRTRasterBand::SerializeToXML(
pszVRTPathIn, bHasWarnedAboutRAMUsage, nAccRAMUsage);

/* -------------------------------------------------------------------- */
/* Set subclass. */
Expand Down
9 changes: 7 additions & 2 deletions frmts/vrt/vrtrasterband.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -646,7 +646,9 @@ CPLString VRTSerializeNoData(double dfVal, GDALDataType eDataType,
/* SerializeToXML() */
/************************************************************************/

CPLXMLNode *VRTRasterBand::SerializeToXML(const char *pszVRTPath)
CPLXMLNode *VRTRasterBand::SerializeToXML(const char *pszVRTPath,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage)

{
CPLXMLNode *psTree =
Expand Down Expand Up @@ -830,9 +832,12 @@ CPLXMLNode *VRTRasterBand::SerializeToXML(const char *pszVRTPath)
/* Mask band (specific to that raster band) */
/* ==================================================================== */

nAccRAMUsage += CPLXMLNodeGetRAMUsageEstimate(psTree);

if (m_poMaskBand != nullptr)
{
CPLXMLNode *psBandTree = m_poMaskBand->SerializeToXML(pszVRTPath);
CPLXMLNode *psBandTree = m_poMaskBand->SerializeToXML(
pszVRTPath, bHasWarnedAboutRAMUsage, nAccRAMUsage);

if (psBandTree != nullptr)
{
Expand Down
7 changes: 5 additions & 2 deletions frmts/vrt/vrtrawrasterband.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -429,7 +429,9 @@ VRTRawRasterBand::XMLInit(CPLXMLNode *psTree, const char *pszVRTPath,
/* SerializeToXML() */
/************************************************************************/

CPLXMLNode *VRTRawRasterBand::SerializeToXML(const char *pszVRTPath)
CPLXMLNode *VRTRawRasterBand::SerializeToXML(const char *pszVRTPath,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage)

{

Expand All @@ -444,7 +446,8 @@ CPLXMLNode *VRTRawRasterBand::SerializeToXML(const char *pszVRTPath)
return nullptr;
}

CPLXMLNode *psTree = VRTRasterBand::SerializeToXML(pszVRTPath);
CPLXMLNode *psTree = VRTRasterBand::SerializeToXML(
pszVRTPath, bHasWarnedAboutRAMUsage, nAccRAMUsage);

/* -------------------------------------------------------------------- */
/* Set subclass. */
Expand Down
47 changes: 38 additions & 9 deletions frmts/vrt/vrtsourcedrasterband.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1861,30 +1861,59 @@ CPLErr VRTSourcedRasterBand::XMLInit(
/* SerializeToXML() */
/************************************************************************/

CPLXMLNode *VRTSourcedRasterBand::SerializeToXML(const char *pszVRTPath)
CPLXMLNode *VRTSourcedRasterBand::SerializeToXML(const char *pszVRTPath,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage)

{
CPLXMLNode *psTree = VRTRasterBand::SerializeToXML(pszVRTPath);
CPLXMLNode *psTree = VRTRasterBand::SerializeToXML(
pszVRTPath, bHasWarnedAboutRAMUsage, nAccRAMUsage);
CPLXMLNode *psLastChild = psTree->psChild;
while (psLastChild != nullptr && psLastChild->psNext != nullptr)
psLastChild = psLastChild->psNext;

/* -------------------------------------------------------------------- */
/* Process Sources. */
/* -------------------------------------------------------------------- */

GIntBig nUsableRAM = -1;

for (int iSource = 0; iSource < nSources; iSource++)
{
CPLXMLNode *const psXMLSrc =
papoSources[iSource]->SerializeToXML(pszVRTPath);

if (psXMLSrc != nullptr)
{
if (psLastChild == nullptr)
psTree->psChild = psXMLSrc;
else
psLastChild->psNext = psXMLSrc;
psLastChild = psXMLSrc;
// Creating the CPLXMLNode tree representation of a VRT can easily
// take several times RAM usage than its string serialization, or its
// internal representation in the driver.
// We multiply the estimate by a factor of 2 because it is found that
// the raw estimate if too conservative
nAccRAMUsage += 2 * CPLXMLNodeGetRAMUsageEstimate(psXMLSrc);
if (!bHasWarnedAboutRAMUsage && nAccRAMUsage > 512 * 1024 * 1024)
{
if (nUsableRAM < 0)
nUsableRAM = CPLGetUsablePhysicalRAM();
if (nUsableRAM > 0 &&
nAccRAMUsage > static_cast<uint64_t>(nUsableRAM) / 10 * 8)
{
bHasWarnedAboutRAMUsage = true;
CPLError(CE_Warning, CPLE_AppDefined,
"Serialization of this VRT file has already consumed "
"at least %.02f GB of RAM over a total of %.02f. This "
"process may abort",
double(nAccRAMUsage) / (1024 * 1024 * 1024),
double(nUsableRAM) / (1024 * 1024 * 1024));
}
}

if (psXMLSrc == nullptr)
break;

if (psLastChild == nullptr)
psTree->psChild = psXMLSrc;
else
psLastChild->psNext = psXMLSrc;
psLastChild = psXMLSrc;
}

return psTree;
Expand Down
7 changes: 5 additions & 2 deletions frmts/vrt/vrtwarped.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1929,10 +1929,13 @@ CPLErr VRTWarpedRasterBand::IWriteBlock(int nBlockXOff, int nBlockYOff,
/* SerializeToXML() */
/************************************************************************/

CPLXMLNode *VRTWarpedRasterBand::SerializeToXML(const char *pszVRTPathIn)
CPLXMLNode *VRTWarpedRasterBand::SerializeToXML(const char *pszVRTPathIn,
bool &bHasWarnedAboutRAMUsage,
size_t &nAccRAMUsage)

{
CPLXMLNode *const psTree = VRTRasterBand::SerializeToXML(pszVRTPathIn);
CPLXMLNode *const psTree = VRTRasterBand::SerializeToXML(
pszVRTPathIn, bHasWarnedAboutRAMUsage, nAccRAMUsage);

/* -------------------------------------------------------------------- */
/* Set subclass. */
Expand Down

0 comments on commit 4874fd7

Please sign in to comment.