feat(cogify): Update the cogify create cog to support topo raster. BM-1116

packages/cogify/src/cogify/__test__/covering.test.ts

@@ -4,7 +4,7 @@ import { describe, it } from 'node:test';
 import { GoogleTms, QuadKey } from '@basemaps/geo';
 
 import { gsdToMeter } from '../cli/cli.cover.js';
-import { addChildren, addSurrounding } from '../covering.js';
+import { addChildren, addSurrounding } from '../covering/covering.js';
 
 describe('getChildren', () => {
   it('should get children', () => {

packages/cogify/src/cogify/__test__/extract.test.ts

@@ -0,0 +1,37 @@
+import { strictEqual, throws } from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { extractMapCodeAndVersion } from '../topo/extract.js';
+
+describe('extractMapCodeAndVersion', () => {
+  const FakeDomain = 's3://topographic/fake-domain';
+  const validFiles = [
+    { input: `${FakeDomain}/MB07_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_GRIDLESS_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_TIFFv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_TIFF_600v1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    {
+      input: `${FakeDomain}/AX32ptsAX31AY31AY32_GeoTifv1-00.tif`,
+      expected: { mapCode: 'AX32ptsAX31AY31AY32', version: 'v1-00' },
+    },
+    {
+      input: `${FakeDomain}/AZ36ptsAZ35BA35BA36_GeoTifv1-00.tif`,
+      expected: { mapCode: 'AZ36ptsAZ35BA35BA36', version: 'v1-00' },
+    },
+  ];
+  const invalidFiles = [`${FakeDomain}/MB07_GeoTif1-00.tif`, `${FakeDomain}/MB07_TIFF_600v1.tif`];
+
+  it('should parse the correct MapSheet Names', () => {
+    for (const file of validFiles) {
+      const output = extractMapCodeAndVersion(file.input);
+      strictEqual(output.mapCode, file.expected.mapCode, 'Map code does not match');
+      strictEqual(output.version, file.expected.version, 'Version does not match');
+    }
+  });
+
+  it('should not able to parse a version from file', () => {
+    for (const file of invalidFiles) {
+      throws(() => extractMapCodeAndVersion(file), new Error('Version not found in the file name'));
+    }
+  });
+});

packages/cogify/src/cogify/cli.ts

@@ -3,6 +3,7 @@ import { subcommands } from 'cmd-ts';
 import { BasemapsCogifyCreateCommand } from './cli/cli.cog.js';
 import { BasemapsCogifyConfigCommand } from './cli/cli.config.js';
 import { BasemapsCogifyCoverCommand } from './cli/cli.cover.js';
+import { TopoStacCreationCommand } from './cli/cli.topo.js';
 import { BasemapsCogifyValidateCommand } from './cli/cli.validate.js';
 
 export const CogifyCli = subcommands({
@@ -12,5 +13,6 @@ export const CogifyCli = subcommands({
     create: BasemapsCogifyCreateCommand,
     config: BasemapsCogifyConfigCommand,
     validate: BasemapsCogifyValidateCommand,
+    topo: TopoStacCreationCommand,
   },
 });

packages/cogify/src/cogify/cli/cli.cog.ts

@@ -11,12 +11,18 @@ import path from 'path';
 import { StacAsset, StacCollection } from 'stac-ts';
 import { pathToFileURL } from 'url';
 
-import { CutlineOptimizer } from '../../cutline.js';
 import { SourceDownloader } from '../../download.js';
 import { HashTransform } from '../../hash.stream.js';
 import { getLogger, logArguments } from '../../log.js';
-import { gdalBuildCog, gdalBuildVrt, gdalBuildVrtWarp, gdalCreate } from '../gdal.command.js';
-import { GdalRunner } from '../gdal.runner.js';
+import { CutlineOptimizer } from '../covering/cutline.js';
+import {
+  gdalBuildCog,
+  gdalBuildTopoRasterCommands,
+  gdalBuildVrt,
+  gdalBuildVrtWarp,
+  gdalCreate,
+} from '../gdal/gdal.command.js';
+import { GdalRunner } from '../gdal/gdal.runner.js';
 import { Url, UrlArrayJsonFile } from '../parsers.js';
 import { CogifyCreationOptions, CogifyStacItem, getCutline, getSources } from '../stac.js';
 
@@ -65,6 +71,7 @@ export const BasemapsCogifyCreateCommand = command({
     ...logArguments,
     path: restPositionals({ type: Url, displayName: 'path', description: 'Path to covering configuration' }),
     force: flag({ long: 'force', description: 'Overwrite existing tiff files' }),
+    topo: flag({ long: 'topo', description: 'Standardize topo raster map' }),
     concurrency: option({
       type: number,
       long: 'concurrency',
@@ -155,7 +162,7 @@ export const BasemapsCogifyCreateCommand = command({
         const { item, url } = f;
         const cutlineLink = getCutline(item.links);
         const options = item.properties['linz_basemaps:options'];
-        const tileId = TileId.fromTile(options.tile);
+        const tileId = args.topo ? item.id : TileId.fromTile(options.tile);
 
         // Location to where the tiff should be stored
         const tiffPath = new URL(tileId + '.tiff', url);
@@ -173,11 +180,24 @@ export const BasemapsCogifyCreateCommand = command({
         // Create the tiff concurrently
         const outputTiffPath = await Q(async () => {
           metrics.start(tileId); // Only start the timer when the cog is actually being processed
-
-          const cutline = await CutlineOptimizer.loadFromLink(cutlineLink, tileMatrix);
           const sourceLocations = await Promise.all(sourceFiles.map((f) => sources.get(f, logger)));
+          const cutline = await CutlineOptimizer.loadFromLink(cutlineLink, tileMatrix);
+          if (args.topo) {
+            const width = item.properties['source:width'] as number;
+            const height = item.properties['source:height'] as number;
+            return createTopoCog({
+              tileId,
+              options,
+              tempFolder: tmpFolder,
+              sourceFiles: sourceLocations,
+              cutline,
+              size: { width, height },
+              logger,
+            });
+          }
 
           return createCog({
+            tileId,
             options,
             tempFolder: tmpFolder,
             sourceFiles: sourceLocations,
@@ -268,14 +288,18 @@ export const BasemapsCogifyCreateCommand = command({
       {
         count: toCreate.length,
         created: filtered.length,
-        files: filtered.map((f) => TileId.fromTile(f.item.properties['linz_basemaps:options'].tile)),
+        files: filtered.map((f) => {
+          return args.topo ? f.item.id : TileId.fromTile(f.item.properties['linz_basemaps:options'].tile);
+        }),
       },
       'Cog:Done',
     );
   },
 });
 
 export interface CogCreationContext {
+  /** TileId for the file name */
+  tileId: string;
   /** COG Creation options */
   options: CogifyCreationOptions;
   /** Location to store all the temporary files */
@@ -284,6 +308,8 @@ export interface CogCreationContext {
   sourceFiles: URL[];
   /** Optional cutline to cut the imagery too */
   cutline: CutlineOptimizer;
+  /** Optional Source imagery size for topo raster trim pixel */
+  size?: { width: number; height: number };
   /** Optional logger */
   logger?: LogType;
 }
@@ -292,7 +318,7 @@ export interface CogCreationContext {
 async function createCog(ctx: CogCreationContext): Promise<URL> {
   const options = ctx.options;
   await ProjectionLoader.load(options.sourceEpsg);
-  const tileId = TileId.fromTile(options.tile);
+  const tileId = ctx.tileId;
 
   const logger = ctx.logger?.child({ tileId });
 
@@ -309,7 +335,7 @@ async function createCog(ctx: CogCreationContext): Promise<URL> {
   logger?.debug({ tileId }, 'Cog:Create:VrtWarp');
 
   const cutlineProperties: { url: URL | null; blend: number } = { url: null, blend: ctx.cutline.blend };
-  if (ctx.cutline.path) {
+  if (ctx.cutline) {
     logger?.debug('Cog:Cutline');
     const optimizedCutline = ctx.cutline.optimize(options.tile);
     if (optimizedCutline) {
@@ -355,6 +381,32 @@ async function createCog(ctx: CogCreationContext): Promise<URL> {
   return cogCreateCommand.output;
 }
 
+/** Create a cog from the creation options */
+async function createTopoCog(ctx: CogCreationContext): Promise<URL> {
+  const options = ctx.options;
+  await ProjectionLoader.load(options.sourceEpsg);
+  const tileId = ctx.tileId;
+
+  const logger = ctx.logger?.child({ tileId });
+
+  logger?.debug({ tileId }, 'TopoCog:Create:VrtSource');
+  // Create the vrt of all the source files
+  const vrtSourceCommand = gdalBuildVrt(new URL(`${tileId}-source.vrt`, ctx.tempFolder), ctx.sourceFiles, true);
+  await new GdalRunner(vrtSourceCommand).run(logger);
+
+  // Create the COG from the vrt file
+  if (ctx.size == null) throw new Error('TopoCog: Source image size is required for pixel trim');
+  const cogCreateCommand = gdalBuildTopoRasterCommands(
+    new URL(`${tileId}.tiff`, ctx.tempFolder),
+    vrtSourceCommand.output,
+    options,
+    ctx.size?.width,
+    ctx.size?.height,
+  );
+  await new GdalRunner(cogCreateCommand).run(logger);
+  return cogCreateCommand.output;
+}
+
 /**
  * Very basic checking for the output tiff to ensure it was uploaded ok
  * Just open it as a COG and ensure the metadata looks about  right

packages/cogify/src/cogify/cli/cli.cover.ts

@@ -7,10 +7,10 @@ import { Metrics } from '@linzjs/metrics';
 import { command, flag, number, oneOf, option, optional, restPositionals, string } from 'cmd-ts';
 
 import { isArgo } from '../../argo.js';
-import { CutlineOptimizer } from '../../cutline.js';
 import { getLogger, logArguments } from '../../log.js';
 import { Presets } from '../../preset.js';
-import { createTileCover, TileCoverContext } from '../../tile.cover.js';
+import { CutlineOptimizer } from '../covering/cutline.js';
+import { createTileCover, TileCoverContext } from '../covering/tile.cover.js';
 import { RgbaType, Url, UrlFolder } from '../parsers.js';
 import { createFileStats } from '../stac.js';
 
@@ -84,6 +84,9 @@ export const BasemapsCogifyCoverCommand = command({
       throw new Error(`No collection.json found with imagery: ${im.url.href}`);
     }
 
+    const slug = im.collection?.['linz:slug'];
+    if (slug != null) im.name = slug as string;
+
     const tms = SupportedTileMatrix.find((f) => f.identifier.toLowerCase() === args.tileMatrix.toLowerCase());
     if (tms == null) throw new Error('--tile-matrix: ' + args.tileMatrix + ' not found');
 
@@ -144,11 +147,13 @@ export const BasemapsCogifyCoverCommand = command({
     const items = [];
     const tilesByZoom: number[] = [];
     for (const item of res.items) {
-      const tileId = TileId.fromTile(item.properties['linz_basemaps:options'].tile);
+      const tile = item.properties['linz_basemaps:options'].tile;
+      if (tile == null) throw new Error('Tile not found in item');
+      const tileId = TileId.fromTile(tile);
       const itemPath = new URL(`${tileId}.json`, targetPath);
       items.push({ path: itemPath });
       await fsa.write(itemPath, JSON.stringify(item, null, 2));
-      const z = item.properties['linz_basemaps:options'].tile.z;
+      const z = tile.z;
       tilesByZoom[z] = (tilesByZoom[z] ?? 0) + 1;
       ctx.logger?.trace({ path: itemPath }, 'Imagery:Stac:Item:Write');
     }