feat(cli): compare config bundles using --target config.json (#3016)

#### Motivation

It is useful to compare configuration bundles, but they contain too many
ids and values that could rotate. The import command can compare two
configurations while ignoring the changes in ids.

#### Modification

compare config bundles using `--target config.json`


#### Checklist

_If not applicable, provide explanation of why._

- [ ] Tests updated
- [ ] Docs updated
- [ ] Issue linked in Title

---------

Co-authored-by: Wentao Kuang <[email protected]>

packages/cli/src/cli/config/action.bundle.ts

@@ -41,9 +41,8 @@ export class CommandBundle extends CommandLineAction {
     const config = this.config.value ?? DefaultConfig;
     const bundle = this.output.value ?? DefaultOutput;
     const mem = await ConfigJson.fromPath(config, logger);
+    if (this.assets.value) mem.assets = this.assets.value;
     const configJson = mem.toJson();
-    const assets = this.assets.value;
-    if (assets) configJson.assets = assets;
     await fsa.writeJson(bundle, configJson);
     logger.info({ path: bundle }, 'ConfigBundled');
     return;

packages/cli/src/cli/config/action.import.ts

@@ -11,7 +11,7 @@ import {
   standardizeLayerName,
 } from '@basemaps/config';
 import { GoogleTms, Nztm2000QuadTms, Projection, TileMatrixSet } from '@basemaps/geo';
-import { Env, fsa, getDefaultConfig, LogConfig } from '@basemaps/shared';
+import { Env, fsa, getDefaultConfig, LogConfig, LogType, setDefaultConfig } from '@basemaps/shared';
 import { CogJobJson } from '@basemaps/shared';
 import { CommandLineAction, CommandLineFlagParameter, CommandLineStringParameter } from '@rushstack/ts-command-line';
 import { FeatureCollection } from 'geojson';
@@ -26,9 +26,9 @@ const VectorStyles = ['topographic', 'topolite', 'aerialhybrid']; // Vector styl
 
 export class CommandImport extends CommandLineAction {
   private config!: CommandLineStringParameter;
-  private backup!: CommandLineStringParameter;
   private output!: CommandLineStringParameter;
   private commit!: CommandLineFlagParameter;
+  private target!: CommandLineStringParameter;
 
   promises: Promise<boolean>[] = [];
   /** List of paths to invalidate at the end of the request */
@@ -55,34 +55,46 @@ export class CommandImport extends CommandLineAction {
       description: 'Path of config json, this can be both a local path or s3 location',
       required: true,
     });
-    this.backup = this.defineStringParameter({
-      argumentName: 'BACKUP',
-      parameterLongName: '--backup',
-      description: 'Backup the old config into a config bundle json',
-    });
     this.output = this.defineStringParameter({
       argumentName: 'OUTPUT',
       parameterLongName: '--output',
       description: 'Output a markdown file with the config changes',
     });
+    this.target = this.defineStringParameter({
+      argumentName: 'TARGET',
+      parameterLongName: '--target',
+      description: 'Target config file to compare',
+    });
     this.commit = this.defineFlagParameter({
       parameterLongName: '--commit',
       description: 'Actually start the import',
       required: false,
     });
   }
 
+  async getConfig(logger: LogType): Promise<BasemapsConfigProvider> {
+    if (this.target.value) {
+      logger.info({ config: this.target.value }, 'Import:Target:Load');
+      const configJson = await fsa.readJson<ConfigBundled>(this.target.value);
+      const mem = ConfigProviderMemory.fromJson(configJson);
+      setDefaultConfig(mem);
+      return mem;
+    }
+    return getDefaultConfig();
+  }
+
   async onExecute(): Promise<void> {
     const logger = LogConfig.get();
     const commit = this.commit.value ?? false;
     const config = this.config.value;
-    const backup = this.backup.value;
-    const cfg = getDefaultConfig();
+
     if (config == null) throw new Error('Please provide a config json');
     if (commit && !config.startsWith('s3://') && Env.isProduction()) {
       throw new Error('To actually import into dynamo has to use the config file from s3.');
     }
 
+    const cfg = await this.getConfig(logger);
+
     const HostPrefix = Env.isProduction() ? '' : 'dev.';
     const healthEndpoint = `https://${HostPrefix}basemaps.linz.govt.nz/v1/health`;
 
@@ -95,12 +107,20 @@ export class CommandImport extends CommandLineAction {
     logger.info({ config }, 'Import:Load');
     const configJson = await fsa.readJson<ConfigBundled>(config);
     const mem = ConfigProviderMemory.fromJson(configJson);
-    mem.createVirtualTileSets();
 
     logger.info({ config }, 'Import:Start');
-    for (const config of mem.objects.values()) this.update(config, commit);
+    const objectTypes: Partial<Record<ConfigPrefix, number>> = {};
+    for (const config of mem.objects.values()) {
+      const objectType = ConfigId.getPrefix(config.id);
+      if (objectType) {
+        objectTypes[objectType] = (objectTypes[objectType] ?? 0) + 1;
+      }
+      this.update(config, cfg, commit);
+    }
     await Promise.all(this.promises);
 
+    logger.info({ objects: mem.objects.size, types: objectTypes }, 'Import:Compare:Done');
+
     if (commit) {
       const configBundle: ConfigBundle = {
         id: cfg.ConfigBundle.id(configJson.hash),
@@ -117,6 +137,8 @@ export class CommandImport extends CommandLineAction {
         // Update the cb_latest record
         configBundle.id = cfg.ConfigBundle.id('latest');
         await cfg.ConfigBundle.put(configBundle);
+      } else {
+        logger.error('Import:NotWriteable');
       }
     }
 
@@ -134,19 +156,15 @@ export class CommandImport extends CommandLineAction {
       if (!res.ok) throw new Error('Basemaps is unhealthy');
     }
 
-    if (backup) {
-      await fsa.writeJson(backup, this.backupConfig.toJson());
-    }
-
     const output = this.output.value;
     if (output) await this.outputChange(output, mem, cfg);
 
     if (commit !== true) logger.info('DryRun:Done');
   }
 
-  update(config: BaseConfig, commit: boolean): void {
+  update(config: BaseConfig, oldConfig: BasemapsConfigProvider, commit: boolean): void {
     const promise = Q(async (): Promise<boolean> => {
-      const updater = new Updater(config, commit);
+      const updater = new Updater(config, oldConfig, commit);
 
       const hasChanges = await updater.reconcile();
       if (hasChanges) {

packages/cli/src/cli/config/config.diff.ts

@@ -2,7 +2,7 @@ import { LogType } from '@basemaps/shared';
 import c from 'ansi-colors';
 import diff from 'deep-diff';
 
-export const IgnoredProperties = ['id', 'createdAt', 'updatedAt', 'year', 'resolution'];
+export const IgnoredProperties = new Set(['id', 'createdAt', 'updatedAt', 'year', 'resolution']);
 
 export class ConfigDiff {
   static getDiff<T>(changes: diff.Diff<T, T>[]): string {
@@ -32,7 +32,7 @@ export class ConfigDiff {
   }
 
   static showDiff<T extends { id: string }>(type: string, oldData: T, newData: T, logger: LogType): boolean {
-    const changes = diff.diff(oldData, newData, (_path: string[], key: string) => IgnoredProperties.indexOf(key) >= 0);
+    const changes = diff.diff(oldData, newData, (_path: string[], key: string) => IgnoredProperties.has(key));
     if (changes) {
       const changeDif = ConfigDiff.getDiff(changes);
       logger.info({ type, record: newData.id }, 'Changes');

packages/cli/src/cli/config/config.update.ts

@@ -1,14 +1,15 @@
 import {
   BaseConfig,
+  BasemapsConfigObject,
   BasemapsConfigProvider,
+  ConfigId,
   ConfigImagery,
   ConfigPrefix,
   ConfigProvider,
   ConfigTileSet,
   ConfigVectorStyle,
 } from '@basemaps/config';
-import { BasemapsConfigObject, ConfigId } from '@basemaps/config';
-import { ConfigDynamoBase, getDefaultConfig, LogConfig, LogType } from '@basemaps/shared';
+import { ConfigDynamoBase, LogConfig, LogType } from '@basemaps/shared';
 import PLimit from 'p-limit';
 
 import { ConfigDiff } from './config.diff.js';
@@ -27,9 +28,9 @@ export class Updater<S extends BaseConfig = BaseConfig> {
    * Class to apply an TileSetConfig source to the tile metadata db
    * @param config a string or TileSetConfig to use
    */
-  constructor(config: S, isCommit: boolean) {
+  constructor(config: S, oldConfig: BasemapsConfigProvider, isCommit: boolean) {
     this.config = config;
-    this.cfg = getDefaultConfig();
+    this.cfg = oldConfig;
     const prefix = ConfigId.getPrefix(config.id);
     if (prefix == null) throw new Error(`Incorrect Config Id ${config.id}`);
     this.prefix = prefix;
@@ -72,6 +73,7 @@ export class Updater<S extends BaseConfig = BaseConfig> {
     const newData = this.getConfig();
     const db = this.getDB();
     const oldData = await this.getOldData();
+
     if (oldData == null || ConfigDiff.showDiff(db.prefix, oldData, newData, this.logger)) {
       const operation = oldData == null ? 'Insert' : 'Update';
       this.logger.info({ type: db.prefix, record: newData.id, commit: this.isCommit }, `Change:${operation}`);

packages/config/src/json/json.config.ts

@@ -24,7 +24,7 @@ import { ConfigPrefix } from '../config/prefix.js';
 import { ConfigProvider } from '../config/provider.js';
 import { ConfigLayer, ConfigTileSet, TileSetType } from '../config/tile.set.js';
 import { ConfigVectorStyle, StyleJson } from '../config/vector.style.js';
-import { ConfigProviderMemory } from '../memory/memory.config.js';
+import { ConfigBundled, ConfigProviderMemory } from '../memory/memory.config.js';
 import { LogType } from './log.js';
 import { zProviderConfig } from './parse.provider.js';
 import { zStyleJson } from './parse.style.js';
@@ -93,6 +93,14 @@ export class ConfigJson {
 
   /** Import configuration from a base path */
   static async fromPath(basePath: string, log: LogType): Promise<ConfigProviderMemory> {
+    if (basePath.endsWith('.json') || basePath.endsWith('.json.gz')) {
+      const config = await fsa.readJson<BaseConfig>(basePath);
+      if (config.id && config.id.startsWith('cb_')) {
+        // We have been given a config bundle just load that instead!
+        return ConfigProviderMemory.fromJson(config as unknown as ConfigBundled);
+      }
+    }
+
     const cfg = new ConfigJson(basePath, log);
 
     const files = await fsa.toArray(fsa.list(basePath));

packages/config/src/memory/memory.config.ts

@@ -68,6 +68,9 @@ function removeUndefined(obj: unknown): void {
 export class ConfigProviderMemory extends BasemapsConfigProvider {
   override type = 'memory' as const;
 
+  /** Optional id of the configuration */
+  id?: string;
+
   Imagery = new MemoryConfigObject<ConfigImagery>(this, ConfigPrefix.Imagery);
   Style = new MemoryConfigObject<ConfigVectorStyle>(this, ConfigPrefix.Style);
   TileSet = new MemoryConfigObject<ConfigTileSet>(this, ConfigPrefix.TileSet);
@@ -118,7 +121,8 @@ export class ConfigProviderMemory extends BasemapsConfigProvider {
     }
 
     cfg.hash = sha256base58(JSON.stringify(cfg));
-    cfg.id = ConfigId.prefix(ConfigPrefix.ConfigBundle, ulid());
+    this.id = this.id ?? ConfigId.prefix(ConfigPrefix.ConfigBundle, ulid());
+    cfg.id = this.id;
 
     return cfg;
   }
@@ -140,7 +144,8 @@ export class ConfigProviderMemory extends BasemapsConfigProvider {
     const layerByName = new Map<string, ConfigLayer>();
     // Set all layers as minZoom:32
     for (const l of layers) {
-      const newLayer = { ...l, maxZoom: undefined, minZoom: 32 };
+      const newLayer = { ...l, minZoom: 32 };
+      delete newLayer.maxZoom; // max zoom not needed when minzoom is 32
       layerByName.set(newLayer.name, { ...layerByName.get(l.name), ...newLayer });
     }
     const allTileset: ConfigTileSet = {
@@ -209,17 +214,20 @@ export class ConfigProviderMemory extends BasemapsConfigProvider {
     if (cfg.id == null || ConfigId.getPrefix(cfg.id) !== ConfigPrefix.ConfigBundle) {
       throw new Error('Provided configuration file is not a basemaps config bundle.');
     }
-    // Load the time the bundle was created from the ULID
-    const updatedAt = decodeTime(ConfigId.unprefix(ConfigPrefix.ConfigBundle, cfg.id));
     // TODO this should validate the config
     const mem = new ConfigProviderMemory();
+
     for (const ts of cfg.tileSet) mem.put(ts);
     for (const st of cfg.style) mem.put(st);
     for (const pv of cfg.provider) mem.put(pv);
     for (const img of cfg.imagery) mem.put(img);
 
+    // Load the time the bundle was created from the ULID
+    const updatedAt = decodeTime(ConfigId.unprefix(ConfigPrefix.ConfigBundle, cfg.id));
     for (const obj of mem.objects.values()) obj.updatedAt = updatedAt;
+
     mem.assets = cfg.assets;
+    mem.id = cfg.id;
 
     return mem;
   }