Skip to content

Commit

Permalink
MRF: Allow open of MRF-in-TAR as MRF (OSGeo#10331)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lucianpls
lucianpls authored Jun 30, 2024
1 parent 3e41319 commit 6445593
Show file tree
Hide file tree
Showing 4 changed files with 63 additions and 14 deletions.
24 changes: 24 additions & 0 deletions autotest/gdrivers/mrf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,30 @@ def test_mrf_zen_test():
gdal.Unlink(f)


def test_mrf_in_tar(tmp_path):
import tarfile

files = tuple("plain." + ext for ext in ("mrf", "idx", "pzp", "mrf.aux.xml"))
gdal.Translate(
tmp_path / "plain.mrf",
"data/byte.tif",
format="MRF",
creationOptions=["COMPRESS=DEFLATE"],
)
tarname = tmp_path / "plain.mrf.tar"
# the .mrf has to be the first file in the tar, with no path
with tarfile.TarFile(tarname, "w", format=tarfile.GNU_FORMAT) as tar:
for fn in files:
tar.add(tmp_path / fn, arcname=fn)
for fn in files:
gdal.Unlink(tmp_path / fn)
ds = gdal.Open(tarname)
cs = ds.GetRasterBand(1).Checksum()
ds = None
assert cs == 4672
gdal.Unlink(tarname)


def test_mrf_overview_nnb_fact_2():

expected_cs = 1087
Expand Down
5 changes: 2 additions & 3 deletions frmts/mrf/marfa.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,7 @@ ILOrder OrderToken(const char *, ILOrder def = IL_ERR_ORD);
CPLString getFname(CPLXMLNode *, const char *, const CPLString &, const char *);
CPLString getFname(const CPLString &, const char *);
double getXMLNum(CPLXMLNode *, const char *, double);
// Offset of index, pos is in pages
GIntBig IdxOffset(const ILSize &, const ILImage &);
double logbase(double val, double base);
int IsPower(double value, double base);
Expand Down Expand Up @@ -344,9 +345,6 @@ typedef struct
GDALRWFlag acc;
} VF;

// Offset of index, pos is in pages
GIntBig IdxOffset(const ILSize &pos, const ILImage &img);

enum
{
SAMPLING_ERR,
Expand Down Expand Up @@ -556,6 +554,7 @@ class MRFDataset final : public GDALPamDataset

// MRF file name
CPLString fname;
CPLString publicname;

// The source to be cached in this MRF
CPLString source;
Expand Down
40 changes: 29 additions & 11 deletions frmts/mrf/marfa_dataset.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ MRFDataset::MRFDataset()
pzscctx(nullptr), pzsdctx(nullptr), read_timer(), write_timer(0)
{
m_oSRS.SetAxisMappingStrategy(OAMS_TRADITIONAL_GIS_ORDER);
// X0 Xx Xy Y0 Yx Yy
// X0 Xx Xy Y0 Yx Yy
double gt[6] = {0.0, 1.0, 0.0, 0.0, 0.0, 1.0};

memcpy(GeoTransform, gt, sizeof(gt));
Expand Down Expand Up @@ -603,15 +603,26 @@ GDALDataset *MRFDataset::Open(GDALOpenInfo *poOpenInfo)
int level = -1; // All levels
int version = 0; // Current
int zslice = 0;
string fn; // Used to parse and adjust the file name
string fn; // Used to parse and adjust the file name
string insidefn; // inside tar file name

// Different ways to open an MRF
if (poOpenInfo->nHeaderBytes >= 10)
{
const char *pszHeader =
reinterpret_cast<char *>(poOpenInfo->pabyHeader);
fn.assign(pszHeader, poOpenInfo->nHeaderBytes);
if (STARTS_WITH(pszHeader, "<MRF_META>")) // Regular file name
config = CPLParseXMLFile(pszFileName);
else if (poOpenInfo->eAccess == GA_ReadOnly && fn.size() > 600 &&
(fn[262] == 0 || fn[262] == 32) &&
STARTS_WITH(fn.c_str() + 257, "ustar") &&
strlen(CPLGetPath(fn.c_str())) == 0 &&
STARTS_WITH(fn.c_str() + 512, "<MRF_META>"))
{ // An MRF inside a tar
insidefn = string("/vsitar/") + pszFileName + "/" + pszHeader;
config = CPLParseXMLFile(insidefn.c_str());
}
#if defined(LERC)
else
config = LERC_Band::GetMRFConfig(poOpenInfo);
Expand Down Expand Up @@ -644,6 +655,11 @@ GDALDataset *MRFDataset::Open(GDALOpenInfo *poOpenInfo)

MRFDataset *ds = new MRFDataset();
ds->fname = pszFileName;
if (!insidefn.empty())
{
ds->publicname = pszFileName;
ds->fname = insidefn;
}
ds->eAccess = poOpenInfo->eAccess;
ds->level = level;
ds->zslice = zslice;
Expand All @@ -657,7 +673,7 @@ GDALDataset *MRFDataset::Open(GDALOpenInfo *poOpenInfo)
{
// Open the whole dataset, then pick one level
ds->cds = new MRFDataset();
ds->cds->fname = pszFileName;
ds->cds->fname = ds->fname;
ds->cds->eAccess = ds->eAccess;
ds->zslice = zslice;
ret = ds->cds->Initialize(config);
Expand All @@ -683,15 +699,15 @@ GDALDataset *MRFDataset::Open(GDALOpenInfo *poOpenInfo)
}

// Tell PAM what our real file name is, to help it find the aux.xml
ds->SetPhysicalFilename(pszFileName);
ds->SetPhysicalFilename(ds->fname);
// Don't mess with metadata after this, otherwise PAM will re-write the
// aux.xml
ds->TryLoadXML();

/* -------------------------------------------------------------------- */
/* Open external overviews. */
/* -------------------------------------------------------------------- */
ds->oOvManager.Initialize(ds, pszFileName);
ds->oOvManager.Initialize(ds, ds->fname);

return ds;
}
Expand Down Expand Up @@ -1012,10 +1028,13 @@ char **MRFDataset::GetFileList()
{
char **papszFileList = nullptr;

string usename = fname;
if (!publicname.empty())
usename = publicname;
// Add the header file name if it is real
VSIStatBufL sStat;
if (VSIStatExL(fname, &sStat, VSI_STAT_EXISTS_FLAG) == 0)
papszFileList = CSLAddString(papszFileList, fname);
if (VSIStatExL(usename.c_str(), &sStat, VSI_STAT_EXISTS_FLAG) == 0)
papszFileList = CSLAddString(papszFileList, usename.c_str());

// These two should be real
// We don't really want to add these files, since they will be erased when
Expand Down Expand Up @@ -1676,15 +1695,14 @@ GIntBig MRFDataset::AddOverviews(int scaleIn)

// And adjust the offset again, within next level
img.idxoffset += sizeof(ILIdx) * img.pagecount.l / img.size.z * zslice;

int l = static_cast<int>(img.size.l);
// Create and register the overviews for each band
for (int i = 1; i <= nBands; i++)
{
MRFRasterBand *b =
reinterpret_cast<MRFRasterBand *>(GetRasterBand(i));
if (!(b->GetOverview(static_cast<int>(img.size.l) - 1)))
b->AddOverview(newMRFRasterBand(this, img, i,
static_cast<int>(img.size.l)));
if (!(b->GetOverview(l - 1)))
b->AddOverview(newMRFRasterBand(this, img, i, l));
}
}

Expand Down
8 changes: 8 additions & 0 deletions frmts/mrf/mrfdrivercore.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,14 @@ int MRFDriverIdentify(GDALOpenInfo *poOpenInfo)
return TRUE;
#endif

// accept a tar file if the first file has no folder look like an MRF
if (poOpenInfo->eAccess == GA_ReadOnly && fn.size() > 600 &&
(fn[262] == 0 || fn[262] == 32) && STARTS_WITH(fn + 257, "ustar") &&
strlen(CPLGetPath(fn)) == 0 && STARTS_WITH(fn + 512, "<MRF_META>"))
{
return TRUE;
}

return FALSE;
}

Expand Down

0 comments on commit 6445593

Please sign in to comment.