From bb802c56882c9fc1e8864369c866f2ed67fd78d1 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Fri, 15 Dec 2023 23:13:57 -0500 Subject: [PATCH 1/5] Feature: `mbtiles copy --bbox ...` to copy tiles within a bbox only (#1060) Allow users to copy tiles between mbtiles files that are only within a bounding box. Multiple `--bbox` params are allowed. --- Cargo.lock | 1 + justfile | 4 + martin-tile-utils/src/lib.rs | 4 +- martin/src/fonts/mod.rs | 3 +- martin/src/pg/configurator.rs | 2 +- martin/src/pg/function_source.rs | 2 +- martin/src/pg/utils.rs | 3 +- martin/src/utils/error.rs | 2 +- martin/src/utils/id_resolver.rs | 2 +- mbtiles/Cargo.toml | 3 +- mbtiles/src/bin/mbtiles.rs | 4 +- mbtiles/src/copier.rs | 88 +++++++++++-------- mbtiles/tests/copy.rs | 28 +++++- .../copy__convert@v1__bbox__flat-flat.snap | 42 +++++++++ .../copy__convert@v1__bbox__flat-hash.snap | 50 +++++++++++ .../copy__convert@v1__bbox__flat-norm.snap | 85 ++++++++++++++++++ .../copy__convert@v1__bbox__hash-flat.snap | 42 +++++++++ .../copy__convert@v1__bbox__hash-hash.snap | 50 +++++++++++ .../copy__convert@v1__bbox__hash-norm.snap | 85 ++++++++++++++++++ .../copy__convert@v1__bbox__norm-flat.snap | 42 +++++++++ .../copy__convert@v1__bbox__norm-hash.snap | 50 +++++++++++ .../copy__convert@v1__bbox__norm-norm.snap | 85 ++++++++++++++++++ mbtiles/tests/validate.rs | 54 +++++++++++- 23 files changed, 678 insertions(+), 53 deletions(-) create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-flat.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-hash.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-norm.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-flat.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-hash.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-norm.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-flat.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-hash.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-norm.snap diff --git a/Cargo.lock b/Cargo.lock index 2218bb5a0..676ee6095 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1919,6 +1919,7 @@ dependencies = [ "env_logger", "futures", "insta", + "itertools 0.12.0", "log", "martin-tile-utils", "pretty_assertions", diff --git a/justfile b/justfile index 755b091c8..2a13a4dce 100644 --- a/justfile +++ b/justfile @@ -264,6 +264,10 @@ fmt-md: fmt2: cargo +nightly fmt -- --config imports_granularity=Module,group_imports=StdExternalCrate +# Run cargo check +check: + cargo check --workspace --all-targets --bins --tests --lib --benches + # Run cargo clippy clippy: cargo clippy --workspace --all-targets --bins --tests --lib --benches -- -D warnings diff --git a/martin-tile-utils/src/lib.rs b/martin-tile-utils/src/lib.rs index efbbfa9c0..2b187e7e2 100644 --- a/martin-tile-utils/src/lib.rs +++ b/martin-tile-utils/src/lib.rs @@ -11,7 +11,7 @@ use tile_grid::{tms, Tms, Xyz}; pub const EARTH_CIRCUMFERENCE: f64 = 40_075_016.685_578_5; pub const EARTH_RADIUS: f64 = EARTH_CIRCUMFERENCE / 2.0 / PI; -pub const MAX_ZOOM: u8 = 30; +pub const MAX_ZOOM: u8 = 24; use std::sync::OnceLock; fn web_merc() -> &'static Tms { @@ -209,6 +209,8 @@ pub fn tile_index(lon: f64, lat: f64, zoom: u8) -> (u32, u32) { #[must_use] pub fn xyz_to_bbox(zoom: u8, min_x: u32, min_y: u32, max_x: u32, max_y: u32) -> [f64; 4] { assert!(zoom <= MAX_ZOOM, "zoom {zoom} must be <= {MAX_ZOOM}"); + assert!(min_x <= max_x, "min_x {min_x} must be <= max_x {max_x}"); + assert!(min_y <= max_y, "min_y {min_y} must be <= max_y {max_y}"); let left_top_bounds = web_merc().xy_bounds(&Xyz::new(u64::from(min_x), u64::from(min_y), zoom)); let right_bottom_bounds = web_merc().xy_bounds(&Xyz::new(u64::from(max_x), u64::from(max_y), zoom)); diff --git a/martin/src/fonts/mod.rs b/martin/src/fonts/mod.rs index c60c4a06a..c776f1ca6 100644 --- a/martin/src/fonts/mod.rs +++ b/martin/src/fonts/mod.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; use std::sync::OnceLock; use bit_set::BitSet; -use itertools::Itertools; +use itertools::Itertools as _; use log::{debug, info, warn}; use pbf_font_tools::freetype::{Face, Library}; use pbf_font_tools::protobuf::Message; @@ -335,7 +335,6 @@ fn parse_font( } else { format!("{s:02X}-{e:02X}") }) - .collect::>() .join(", "), ); diff --git a/martin/src/pg/configurator.rs b/martin/src/pg/configurator.rs index 60b7afe93..9e0bde7a5 100644 --- a/martin/src/pg/configurator.rs +++ b/martin/src/pg/configurator.rs @@ -2,7 +2,7 @@ use std::cmp::Ordering; use std::collections::HashSet; use futures::future::join_all; -use itertools::Itertools; +use itertools::Itertools as _; use log::{debug, error, info, warn}; use crate::args::BoundsCalcType; diff --git a/martin/src/pg/function_source.rs b/martin/src/pg/function_source.rs index 6b3fc852e..0fa921e13 100644 --- a/martin/src/pg/function_source.rs +++ b/martin/src/pg/function_source.rs @@ -1,4 +1,4 @@ -use std::fmt::Write; +use std::fmt::Write as _; use std::iter::zip; use log::{debug, warn}; diff --git a/martin/src/pg/utils.rs b/martin/src/pg/utils.rs index 800367084..8d79c214a 100755 --- a/martin/src/pg/utils.rs +++ b/martin/src/pg/utils.rs @@ -1,6 +1,7 @@ use std::collections::{BTreeMap, HashMap}; use deadpool_postgres::tokio_postgres::types::Json; +use itertools::Itertools as _; use log::{error, info, warn}; use postgis::{ewkb, LineString, Point, Polygon}; use tilejson::{Bounds, TileJSON}; @@ -104,7 +105,7 @@ fn find_info_kv<'a, T>( match find_kv_ignore_case(map, key) { Ok(None) => { warn!("Unable to configure source {id} because {info} '{key}' was not found. Possible values are: {}", - map.keys().map(String::as_str).collect::>().join(", ")); + map.keys().map(String::as_str).join(", ")); None } Ok(Some(result)) => { diff --git a/martin/src/utils/error.rs b/martin/src/utils/error.rs index 701d332da..3167e8a6d 100644 --- a/martin/src/utils/error.rs +++ b/martin/src/utils/error.rs @@ -1,5 +1,5 @@ use std::error::Error; -use std::fmt::Write; +use std::fmt::Write as _; use std::io; use std::path::PathBuf; diff --git a/martin/src/utils/id_resolver.rs b/martin/src/utils/id_resolver.rs index 7c91d14b4..3942717e2 100644 --- a/martin/src/utils/id_resolver.rs +++ b/martin/src/utils/id_resolver.rs @@ -1,6 +1,6 @@ use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; -use std::fmt::Write; +use std::fmt::Write as _; use std::sync::{Arc, Mutex}; use log::warn; diff --git a/mbtiles/Cargo.toml b/mbtiles/Cargo.toml index cb5c4f761..f87f0fd3c 100644 --- a/mbtiles/Cargo.toml +++ b/mbtiles/Cargo.toml @@ -19,10 +19,11 @@ cli = ["dep:anyhow", "dep:clap", "dep:env_logger", "dep:serde_yaml", "dep:tokio" [dependencies] enum-display.workspace = true futures.workspace = true +itertools.workspace = true log.workspace = true martin-tile-utils.workspace = true -serde_json.workspace = true serde.workspace = true +serde_json.workspace = true serde_with.workspace = true size_format.workspace = true sqlite-hashes.workspace = true diff --git a/mbtiles/src/bin/mbtiles.rs b/mbtiles/src/bin/mbtiles.rs index 75e6f5eba..76c11048a 100644 --- a/mbtiles/src/bin/mbtiles.rs +++ b/mbtiles/src/bin/mbtiles.rs @@ -4,7 +4,7 @@ use clap::{Parser, Subcommand}; use log::error; use mbtiles::{apply_patch, AggHashType, IntegrityCheckType, MbtResult, Mbtiles, MbtilesCopier}; -#[derive(Parser, PartialEq, Eq, Debug)] +#[derive(Parser, PartialEq, Debug)] #[command( version, name = "mbtiles", @@ -19,7 +19,7 @@ pub struct Args { command: Commands, } -#[derive(Subcommand, PartialEq, Eq, Debug)] +#[derive(Subcommand, PartialEq, Debug)] enum Commands { /// Show MBTiles file summary statistics #[command(name = "summary", alias = "info")] diff --git a/mbtiles/src/copier.rs b/mbtiles/src/copier.rs index 7c4ba4cfa..a5a576f98 100644 --- a/mbtiles/src/copier.rs +++ b/mbtiles/src/copier.rs @@ -1,13 +1,16 @@ -use std::collections::HashSet; +use std::fmt::Write as _; use std::path::PathBuf; #[cfg(feature = "cli")] use clap::{Args, ValueEnum}; use enum_display::EnumDisplay; -use log::{debug, info}; +use itertools::Itertools as _; +use log::{debug, info, trace}; +use martin_tile_utils::{bbox_to_xyz, MAX_ZOOM}; use serde::{Deserialize, Serialize}; -use sqlite_hashes::rusqlite::{params_from_iter, Connection}; +use sqlite_hashes::rusqlite::Connection; use sqlx::{query, Executor as _, Row, SqliteConnection}; +use tilejson::Bounds; use crate::errors::MbtResult; use crate::queries::{ @@ -15,7 +18,7 @@ use crate::queries::{ }; use crate::MbtType::{Flat, FlatWithHash, Normalized}; use crate::{ - reset_db_settings, MbtError, MbtType, MbtTypeCli, Mbtiles, AGG_TILES_HASH, + invert_y_value, reset_db_settings, MbtError, MbtType, MbtTypeCli, Mbtiles, AGG_TILES_HASH, AGG_TILES_HASH_IN_DIFF, }; @@ -39,7 +42,7 @@ impl CopyDuplicateMode { } } -#[derive(Clone, Default, PartialEq, Eq, Debug)] +#[derive(Clone, Default, PartialEq, Debug)] #[cfg_attr(feature = "cli", derive(Args))] pub struct MbtilesCopier { /// MBTiles file to read from @@ -73,6 +76,9 @@ pub struct MbtilesCopier { /// List of zoom levels to copy #[cfg_attr(feature = "cli", arg(long, value_delimiter = ','))] pub zoom_levels: Vec, + /// Bounding box to copy, in the format `min_lon,min_lat,max_lon,max_lat`. Can be used multiple times. + #[cfg_attr(feature = "cli", arg(long))] + pub bbox: Vec, /// Compare source file with this file, and only copy non-identical tiles to destination. /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. #[cfg_attr(feature = "cli", arg(long, conflicts_with("apply_patch")))] @@ -105,6 +111,7 @@ impl MbtilesCopier { min_zoom: None, max_zoom: None, zoom_levels: Vec::default(), + bbox: vec![], diff_with_file: None, apply_patch: None, skip_agg_tiles_hash: false, @@ -216,7 +223,7 @@ impl MbtileCopierInt { Self::get_select_from(src_type, dst_type).to_string() }; - let (where_clause, query_args) = self.get_where_clause(); + let where_clause = self.get_where_clause(); let select_from = format!("{select_from} {where_clause}"); let on_dupl = on_duplicate.to_sql(); let sql_cond = Self::get_on_duplicate_sql_cond(on_duplicate, dst_type); @@ -232,14 +239,7 @@ impl MbtileCopierInt { // SAFETY: this is safe as long as handle_lock is valid. We will drop the lock. let rusqlite_conn = unsafe { Connection::from_handle(handle) }?; - Self::copy_tiles( - &rusqlite_conn, - dst_type, - &query_args, - on_dupl, - &select_from, - &sql_cond, - )?; + Self::copy_tiles(&rusqlite_conn, dst_type, on_dupl, &select_from, &sql_cond)?; self.copy_metadata(&rusqlite_conn, &dif, on_dupl)?; } @@ -316,7 +316,6 @@ impl MbtileCopierInt { fn copy_tiles( rusqlite_conn: &Connection, dst_type: MbtType, - query_args: &Vec, on_dupl: &str, select_from: &str, sql_cond: &str, @@ -346,8 +345,8 @@ impl MbtileCopierInt { SELECT tile_hash as tile_id, tile_data FROM ({select_from})" ); - debug!("Copying to {dst_type} with {sql} {query_args:?}"); - rusqlite_conn.execute(&sql, params_from_iter(query_args))?; + debug!("Copying to {dst_type} with {sql}"); + rusqlite_conn.execute(&sql, [])?; format!( " @@ -359,8 +358,8 @@ impl MbtileCopierInt { } }; - debug!("Copying to {dst_type} with {sql} {query_args:?}"); - rusqlite_conn.execute(&sql, params_from_iter(query_args))?; + debug!("Copying to {dst_type} with {sql}"); + rusqlite_conn.execute(&sql, [])?; Ok(()) } @@ -585,32 +584,49 @@ impl MbtileCopierInt { } } - fn get_where_clause(&self) -> (String, Vec) { - let mut query_args = vec![]; - - let sql = if !&self.options.zoom_levels.is_empty() { - let zooms: HashSet = self.options.zoom_levels.iter().copied().collect(); - for z in &zooms { - query_args.push(*z); - } - format!(" AND zoom_level IN ({})", vec!["?"; zooms.len()].join(",")) + /// Format SQL WHERE clause and return it along with the query arguments. + /// Note that there is no risk of SQL injection here, as the arguments are integers. + fn get_where_clause(&self) -> String { + let mut sql = if !&self.options.zoom_levels.is_empty() { + let zooms = self.options.zoom_levels.iter().join(","); + format!(" AND zoom_level IN ({zooms})") } else if let Some(min_zoom) = self.options.min_zoom { if let Some(max_zoom) = self.options.max_zoom { - query_args.push(min_zoom); - query_args.push(max_zoom); - " AND zoom_level BETWEEN ? AND ?".to_string() + format!(" AND zoom_level BETWEEN {min_zoom} AND {max_zoom}") } else { - query_args.push(min_zoom); - " AND zoom_level >= ?".to_string() + format!(" AND zoom_level >= {min_zoom}") } } else if let Some(max_zoom) = self.options.max_zoom { - query_args.push(max_zoom); - " AND zoom_level <= ?".to_string() + format!(" AND zoom_level <= {max_zoom}") } else { String::new() }; - (sql, query_args) + if !self.options.bbox.is_empty() { + sql.push_str(" AND (\n"); + for (idx, bbox) in self.options.bbox.iter().enumerate() { + // Use maximum zoom value for easy filtering, + // converting it on the fly to the actual zoom level + let (min_x, min_y, max_x, max_y) = + bbox_to_xyz(bbox.left, bbox.bottom, bbox.right, bbox.top, MAX_ZOOM); + trace!("Bounding box {bbox} converted to {min_x},{min_y},{max_x},{max_y} at zoom {MAX_ZOOM}"); + let (min_y, max_y) = ( + invert_y_value(MAX_ZOOM, max_y), + invert_y_value(MAX_ZOOM, min_y), + ); + + if idx > 0 { + sql.push_str(" OR\n"); + } + writeln!( + sql, + "((tile_column * (1 << ({MAX_ZOOM} - zoom_level))) BETWEEN {min_x} AND {max_x} AND (tile_row * (1 << ({MAX_ZOOM} - zoom_level))) BETWEEN {min_y} AND {max_y})", + ).unwrap(); + } + sql.push(')'); + } + + sql } } diff --git a/mbtiles/tests/copy.rs b/mbtiles/tests/copy.rs index 407c1571d..b5d59a16b 100644 --- a/mbtiles/tests/copy.rs +++ b/mbtiles/tests/copy.rs @@ -4,11 +4,15 @@ use std::str::from_utf8; use ctor::ctor; use insta::{allow_duplicates, assert_display_snapshot}; +use itertools::Itertools as _; use log::info; +use martin_tile_utils::xyz_to_bbox; use mbtiles::AggHashType::Verify; use mbtiles::IntegrityCheckType::Off; use mbtiles::MbtTypeCli::{Flat, FlatWithHash, Normalized}; -use mbtiles::{apply_patch, init_mbtiles_schema, MbtResult, MbtTypeCli, Mbtiles, MbtilesCopier}; +use mbtiles::{ + apply_patch, init_mbtiles_schema, invert_y_value, MbtResult, MbtTypeCli, Mbtiles, MbtilesCopier, +}; use pretty_assertions::assert_eq as pretty_assert_eq; use rstest::{fixture, rstest}; use serde::Serialize; @@ -255,6 +259,20 @@ async fn convert( let z6only = dump(&mut opt.run().await?).await?; assert_snapshot!(z6only, "v1__z6__{frm}-{to}"); + let mut opt = copier(&frm_mbt, &mem); + opt.dst_type_cli = Some(dst_type); + + // Filter (0, 0, 2, 2) in mbtiles coordinates, which is (0, 2^5-1-2, 2, 2^5-1-0) = (0, 29, 2, 31) in XYZ coordinates, and slightly decrease it + let mut bbox = xyz_to_bbox(5, 0, invert_y_value(5, 2), 2, invert_y_value(5, 0)); + bbox[0] += 180.0 * 0.1 / f64::from(1 << 5); + bbox[1] += 90.0 * 0.1 / f64::from(1 << 5); + bbox[2] -= 180.0 * 0.1 / f64::from(1 << 5); + bbox[3] -= 90.0 * 0.1 / f64::from(1 << 5); + opt.bbox.push(bbox.into()); + + let dmp = dump(&mut opt.run().await?).await?; + assert_snapshot!(dmp, "v1__bbox__{frm}-{to}"); + let mut opt = copier(&frm_mbt, &mem); opt.dst_type_cli = Some(dst_type); opt.min_zoom = Some(6); @@ -370,11 +388,16 @@ async fn patch_on_copy( #[actix_rt::test] #[ignore] async fn test_one() { + let db = Databases::default(); + + // Test convert + convert(Flat, Flat, &db).await.unwrap(); + + // Test diff patch copy let src_type = FlatWithHash; let dif_type = FlatWithHash; // let dst_type = Some(FlatWithHash); let dst_type = None; - let db = databases(); diff_and_patch(src_type, dif_type, dst_type, &db) .await @@ -465,7 +488,6 @@ async fn dump(conn: &mut SqliteConnection) -> MbtResult> { }) .unwrap_or("NULL".to_string()) }) - .collect::>() .join(", "); format!("( {val} )") }) diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-flat.snap new file mode 100644 index 000000000..510f57545 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-flat.snap @@ -0,0 +1,42 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1) )', + '( 5, 1, 2, blob() )', + '( 5, 2, 2, blob(remove) )', + '( 6, 1, 4, blob(edit-v1) )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-hash.snap new file mode 100644 index 000000000..d7da64239 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-hash.snap @@ -0,0 +1,50 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-norm.snap new file mode 100644 index 000000000..ae37bc9bf --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-norm.snap @@ -0,0 +1,85 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [ + '( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )', + '( "D41D8CD98F00B204E9800998ECF8427E", blob() )', + '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-flat.snap new file mode 100644 index 000000000..510f57545 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-flat.snap @@ -0,0 +1,42 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1) )', + '( 5, 1, 2, blob() )', + '( 5, 2, 2, blob(remove) )', + '( 6, 1, 4, blob(edit-v1) )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-hash.snap new file mode 100644 index 000000000..d7da64239 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-hash.snap @@ -0,0 +1,50 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-norm.snap new file mode 100644 index 000000000..ae37bc9bf --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-norm.snap @@ -0,0 +1,85 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [ + '( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )', + '( "D41D8CD98F00B204E9800998ECF8427E", blob() )', + '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-flat.snap new file mode 100644 index 000000000..510f57545 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-flat.snap @@ -0,0 +1,42 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1) )', + '( 5, 1, 2, blob() )', + '( 5, 2, 2, blob(remove) )', + '( 6, 1, 4, blob(edit-v1) )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-hash.snap new file mode 100644 index 000000000..d7da64239 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-hash.snap @@ -0,0 +1,50 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-norm.snap new file mode 100644 index 000000000..ae37bc9bf --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-norm.snap @@ -0,0 +1,85 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [ + '( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )', + '( "D41D8CD98F00B204E9800998ECF8427E", blob() )', + '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/validate.rs b/mbtiles/tests/validate.rs index 7192d4538..d682f1111 100644 --- a/mbtiles/tests/validate.rs +++ b/mbtiles/tests/validate.rs @@ -1,4 +1,7 @@ -use martin_tile_utils::MAX_ZOOM; +#![allow(clippy::unreadable_literal)] + +use insta::assert_snapshot; +use martin_tile_utils::{bbox_to_xyz, MAX_ZOOM}; use mbtiles::MbtError::InvalidTileIndex; use mbtiles::{create_metadata_table, Mbtiles}; use rstest::rstest; @@ -92,13 +95,16 @@ async fn tile_coordinate(#[case] prefix: &str, #[case] suffix: &str) { ok!("1, {prefix} 1 {suffix}"); ok!("2, {prefix} 3 {suffix}"); ok!("3, {prefix} 7 {suffix}"); - ok!("30, {prefix} 0 {suffix}"); - ok!("30, {prefix} 1073741823 {suffix}"); + ok!("24, {prefix} 0 {suffix}"); + ok!("24, {prefix} 16777215 {suffix}"); + // ok!("30, {prefix} 0 {suffix}"); + // ok!("30, {prefix} 1073741823 {suffix}"); err!("0, {prefix} 1 {suffix}"); err!("1, {prefix} 2 {suffix}"); err!("2, {prefix} 4 {suffix}"); err!("3, {prefix} 8 {suffix}"); + err!("24, {prefix} 16777216 {suffix}"); err!("30, {prefix} 1073741824 {suffix}"); err!("{MAX_ZOOM}, {prefix} 1073741824 {suffix}"); err!("{}, {prefix} 0 {suffix}", MAX_ZOOM + 1); // unsupported zoom @@ -117,3 +123,45 @@ async fn tile_data() { err!("0, 0, 0, CAST('abc' AS TEXT)"); err!("0, 0, 0, CAST(123 AS TEXT)"); } + +#[test] +fn test_box() { + fn tst(left: f64, bottom: f64, right: f64, top: f64, zoom: u8) -> String { + let (x0, y0, x1, y1) = bbox_to_xyz(left, bottom, right, top, zoom); + format!("({x0}, {y0}, {x1}, {y1})") + } + + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 0), @"(0, 0, 0, 0)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 1), @"(0, 1, 0, 1)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 2), @"(0, 3, 0, 3)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 3), @"(0, 7, 0, 7)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 4), @"(0, 14, 1, 15)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 5), @"(0, 29, 2, 31)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 6), @"(0, 58, 5, 63)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 7), @"(0, 116, 11, 126)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 8), @"(0, 233, 23, 253)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 9), @"(0, 466, 47, 507)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 10), @"(1, 933, 94, 1014)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 11), @"(3, 1866, 188, 2029)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 12), @"(6, 3732, 377, 4059)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 13), @"(12, 7465, 755, 8119)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 14), @"(25, 14931, 1510, 16239)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 15), @"(51, 29863, 3020, 32479)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 16), @"(102, 59727, 6041, 64958)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 17), @"(204, 119455, 12083, 129917)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 18), @"(409, 238911, 24166, 259834)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 19), @"(819, 477823, 48332, 519669)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 20), @"(1638, 955647, 96665, 1039339)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 21), @"(3276, 1911295, 193331, 2078678)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 22), @"(6553, 3822590, 386662, 4157356)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 23), @"(13107, 7645181, 773324, 8314713)"); + assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 24), @"(26214, 15290363, 1546649, 16629427)"); + + // All these are incorrect + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 25), @"(33554431, 33554431, 33554431, 33554431)"); + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 26), @"(67108863, 67108863, 67108863, 67108863)"); + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 27), @"(134217727, 134217727, 134217727, 134217727)"); + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 28), @"(268435455, 268435455, 268435455, 268435455)"); + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 29), @"(536870911, 536870911, 536870911, 536870911)"); + // assert_snapshot!(tst(-179.43749999999955,-84.76987877980656,-146.8124999999996,-81.37446385260833, 30), @"(1073741823, 1073741823, 1073741823, 1073741823)"); +} From 16fbfa58ff1a5e486c3060d192a76211c4ed7926 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 16 Dec 2023 02:25:50 -0500 Subject: [PATCH 2/5] Split Clap code from `MbtilesCopier` into `CopyArgs` (#1072) This allows easier reuse of the copy functionality without affecting CLI commands. Closes #1071 --- .github/files/markdown.links.config.json | 3 + mbtiles/src/bin/mbtiles.rs | 153 +++++++++++++---------- mbtiles/src/copier.rs | 24 +--- mbtiles/src/mbtiles.rs | 4 +- mbtiles/src/validation.rs | 6 +- 5 files changed, 97 insertions(+), 93 deletions(-) diff --git a/.github/files/markdown.links.config.json b/.github/files/markdown.links.config.json index fad1deb32..eaff6f07c 100644 --- a/.github/files/markdown.links.config.json +++ b/.github/files/markdown.links.config.json @@ -15,6 +15,9 @@ { "pattern": "^http://localhost" }, + { + "pattern": "^https://ghcr.io/maplibre/martin($|/|\\?)" + }, { "pattern": "^http://opensource.org" } diff --git a/mbtiles/src/bin/mbtiles.rs b/mbtiles/src/bin/mbtiles.rs index 76c11048a..ebfb5e631 100644 --- a/mbtiles/src/bin/mbtiles.rs +++ b/mbtiles/src/bin/mbtiles.rs @@ -2,7 +2,11 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; use log::error; -use mbtiles::{apply_patch, AggHashType, IntegrityCheckType, MbtResult, Mbtiles, MbtilesCopier}; +use mbtiles::{ + apply_patch, AggHashType, CopyDuplicateMode, IntegrityCheckType, MbtResult, MbtTypeCli, + Mbtiles, MbtilesCopier, +}; +use tilejson::Bounds; #[derive(Parser, PartialEq, Debug)] #[command( @@ -31,7 +35,7 @@ enum Commands { file: PathBuf, }, /// Gets a single value from the MBTiles metadata table. - #[command(name = "meta-get")] + #[command(name = "meta-get", alias = "get-meta")] MetaGetValue { /// MBTiles file to read a value from file: PathBuf, @@ -39,7 +43,7 @@ enum Commands { key: String, }, /// Sets a single value in the MBTiles metadata table or deletes it if no value. - #[command(name = "meta-set")] + #[command(name = "meta-set", alias = "set-meta")] MetaSetValue { /// MBTiles file to modify file: PathBuf, @@ -50,7 +54,7 @@ enum Commands { }, /// Copy tiles from one mbtiles file to another. #[command(name = "copy", alias = "cp")] - Copy(MbtilesCopier), + Copy(CopyArgs), /// Apply diff file generated from 'copy' command #[command(name = "apply-patch", alias = "apply-diff")] ApplyPatch { @@ -76,6 +80,43 @@ enum Commands { }, } +#[derive(Clone, Default, PartialEq, Debug, clap::Args)] +pub struct CopyArgs { + /// MBTiles file to read from + pub src_file: PathBuf, + /// MBTiles file to write to + pub dst_file: PathBuf, + /// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source + #[arg(long, alias = "dst-type", alias = "dst_type", value_name = "SCHEMA")] + pub mbtiles_type: Option, + /// Allow copying to existing files, and indicate what to do if a tile with the same Z/X/Y already exists + #[arg(long, value_enum)] + pub on_duplicate: Option, + /// Minimum zoom level to copy + #[arg(long, conflicts_with("zoom_levels"))] + pub min_zoom: Option, + /// Maximum zoom level to copy + #[arg(long, conflicts_with("zoom_levels"))] + pub max_zoom: Option, + /// List of zoom levels to copy + #[arg(long, value_delimiter = ',')] + pub zoom_levels: Vec, + /// Bounding box to copy, in the format `min_lon,min_lat,max_lon,max_lat`. Can be used multiple times. + #[arg(long)] + pub bbox: Vec, + /// Compare source file with this file, and only copy non-identical tiles to destination. + /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. + #[arg(long, conflicts_with("apply_patch"))] + pub diff_with_file: Option, + /// Compare source file with this file, and only copy non-identical tiles to destination. + /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. + #[arg(long, conflicts_with("diff_with_file"))] + pub apply_patch: Option, + /// Skip generating a global hash for mbtiles validation. By default, `mbtiles` will compute `agg_tiles_hash` metadata value. + #[arg(long)] + pub skip_agg_tiles_hash: bool, +} + #[tokio::main] async fn main() { let env = env_logger::Env::default().default_filter_or("mbtiles=info"); @@ -105,6 +146,20 @@ async fn main_int() -> anyhow::Result<()> { meta_set_value(file.as_path(), &key, value.as_deref()).await?; } Commands::Copy(opts) => { + let opts = MbtilesCopier { + src_file: opts.src_file, + dst_file: opts.dst_file, + dst_type_cli: opts.mbtiles_type, + dst_type: None, + on_duplicate: opts.on_duplicate, + min_zoom: opts.min_zoom, + max_zoom: opts.max_zoom, + zoom_levels: opts.zoom_levels, + bbox: opts.bbox, + diff_with_file: opts.diff_with_file, + apply_patch: opts.apply_patch, + skip_agg_tiles_hash: opts.skip_agg_tiles_hash, + }; opts.run().await?; } Commands::ApplyPatch { @@ -176,7 +231,7 @@ mod tests { use clap::error::ErrorKind; use clap::Parser; - use mbtiles::{CopyDuplicateMode, MbtilesCopier}; + use mbtiles::CopyDuplicateMode; use super::*; use crate::Commands::{ApplyPatch, Copy, MetaGetValue, MetaSetValue, Validate}; @@ -198,20 +253,17 @@ mod tests { Args::parse_from(["mbtiles", "copy", "src_file", "dst_file"]), Args { verbose: false, - command: Copy(MbtilesCopier::new( - PathBuf::from("src_file"), - PathBuf::from("dst_file") - )) + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + ..Default::default() + }) } ); } #[test] fn test_copy_min_max_zoom_arguments() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.min_zoom = Some(1); - opt.max_zoom = Some(100); - let args = Args::parse_from([ "mbtiles", "copy", @@ -226,7 +278,13 @@ mod tests { args, Args { verbose: false, - command: Copy(opt) + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + min_zoom: Some(1), + max_zoom: Some(100), + ..Default::default() + }) } ); } @@ -271,8 +329,6 @@ mod tests { #[test] fn test_copy_zoom_levels_arguments() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.zoom_levels.extend(&[3, 7, 1]); assert_eq!( Args::parse_from([ "mbtiles", @@ -284,15 +340,18 @@ mod tests { ]), Args { verbose: false, - command: Copy(opt) + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + zoom_levels: vec![3, 7, 1], + ..Default::default() + }) } ); } #[test] fn test_copy_diff_with_file_arguments() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.diff_with_file = Some(PathBuf::from("no_file")); assert_eq!( Args::parse_from([ "mbtiles", @@ -304,15 +363,18 @@ mod tests { ]), Args { verbose: false, - command: Copy(opt) + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + diff_with_file: Some(PathBuf::from("no_file")), + ..Default::default() + }) } ); } #[test] fn test_copy_diff_with_override_copy_duplicate_mode() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.on_duplicate = Some(CopyDuplicateMode::Override); assert_eq!( Args::parse_from([ "mbtiles", @@ -324,47 +386,12 @@ mod tests { ]), Args { verbose: false, - command: Copy(opt) - } - ); - } - - #[test] - fn test_copy_diff_with_ignore_copy_duplicate_mode() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.on_duplicate = Some(CopyDuplicateMode::Ignore); - assert_eq!( - Args::parse_from([ - "mbtiles", - "copy", - "src_file", - "dst_file", - "--on-duplicate", - "ignore" - ]), - Args { - verbose: false, - command: Copy(opt) - } - ); - } - - #[test] - fn test_copy_diff_with_abort_copy_duplicate_mode() { - let mut opt = MbtilesCopier::new(PathBuf::from("src_file"), PathBuf::from("dst_file")); - opt.on_duplicate = Some(CopyDuplicateMode::Abort); - assert_eq!( - Args::parse_from([ - "mbtiles", - "copy", - "src_file", - "dst_file", - "--on-duplicate", - "abort" - ]), - Args { - verbose: false, - command: Copy(opt) + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + on_duplicate: Some(CopyDuplicateMode::Override), + ..Default::default() + }) } ); } diff --git a/mbtiles/src/copier.rs b/mbtiles/src/copier.rs index a5a576f98..e59763ab5 100644 --- a/mbtiles/src/copier.rs +++ b/mbtiles/src/copier.rs @@ -1,8 +1,6 @@ use std::fmt::Write as _; use std::path::PathBuf; -#[cfg(feature = "cli")] -use clap::{Args, ValueEnum}; use enum_display::EnumDisplay; use itertools::Itertools as _; use log::{debug, info, trace}; @@ -24,7 +22,7 @@ use crate::{ #[derive(PartialEq, Eq, Debug, Clone, Copy, EnumDisplay, Serialize, Deserialize)] #[enum_display(case = "Kebab")] -#[cfg_attr(feature = "cli", derive(ValueEnum))] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum CopyDuplicateMode { Override, Ignore, @@ -43,52 +41,32 @@ impl CopyDuplicateMode { } #[derive(Clone, Default, PartialEq, Debug)] -#[cfg_attr(feature = "cli", derive(Args))] pub struct MbtilesCopier { /// MBTiles file to read from pub src_file: PathBuf, /// MBTiles file to write to pub dst_file: PathBuf, /// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source - #[cfg_attr( - feature = "cli", - arg( - long = "mbtiles-type", - alias = "dst-type", - alias = "dst_type", - value_name = "SCHEMA", - value_enum - ) - )] pub dst_type_cli: Option, /// Destination type with options - #[cfg_attr(feature = "cli", arg(skip))] pub dst_type: Option, /// Allow copying to existing files, and indicate what to do if a tile with the same Z/X/Y already exists - #[cfg_attr(feature = "cli", arg(long, value_enum))] pub on_duplicate: Option, /// Minimum zoom level to copy - #[cfg_attr(feature = "cli", arg(long, conflicts_with("zoom_levels")))] pub min_zoom: Option, /// Maximum zoom level to copy - #[cfg_attr(feature = "cli", arg(long, conflicts_with("zoom_levels")))] pub max_zoom: Option, /// List of zoom levels to copy - #[cfg_attr(feature = "cli", arg(long, value_delimiter = ','))] pub zoom_levels: Vec, /// Bounding box to copy, in the format `min_lon,min_lat,max_lon,max_lat`. Can be used multiple times. - #[cfg_attr(feature = "cli", arg(long))] pub bbox: Vec, /// Compare source file with this file, and only copy non-identical tiles to destination. /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. - #[cfg_attr(feature = "cli", arg(long, conflicts_with("apply_patch")))] pub diff_with_file: Option, /// Compare source file with this file, and only copy non-identical tiles to destination. /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. - #[cfg_attr(feature = "cli", arg(long, conflicts_with("diff_with_file")))] pub apply_patch: Option, /// Skip generating a global hash for mbtiles validation. By default, `mbtiles` will compute `agg_tiles_hash` metadata value. - #[cfg_attr(feature = "cli", arg(long))] pub skip_agg_tiles_hash: bool, } diff --git a/mbtiles/src/mbtiles.rs b/mbtiles/src/mbtiles.rs index de9bfe030..64dcb16e5 100644 --- a/mbtiles/src/mbtiles.rs +++ b/mbtiles/src/mbtiles.rs @@ -2,8 +2,6 @@ use std::ffi::OsStr; use std::fmt::{Display, Formatter}; use std::path::Path; -#[cfg(feature = "cli")] -use clap::ValueEnum; use enum_display::EnumDisplay; use log::debug; use serde::{Deserialize, Serialize}; @@ -16,7 +14,7 @@ use crate::{invert_y_value, CopyDuplicateMode, MbtType}; #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)] #[enum_display(case = "Kebab")] -#[cfg_attr(feature = "cli", derive(ValueEnum))] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum MbtTypeCli { Flat, FlatWithHash, diff --git a/mbtiles/src/validation.rs b/mbtiles/src/validation.rs index d222c7df9..57e9ebbc0 100644 --- a/mbtiles/src/validation.rs +++ b/mbtiles/src/validation.rs @@ -1,8 +1,6 @@ use std::collections::HashSet; use std::str::from_utf8; -#[cfg(feature = "cli")] -use clap::ValueEnum; use enum_display::EnumDisplay; use log::{debug, info, warn}; use martin_tile_utils::{Format, TileInfo, MAX_ZOOM}; @@ -52,7 +50,7 @@ impl MbtType { #[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)] #[enum_display(case = "Kebab")] -#[cfg_attr(feature = "cli", derive(ValueEnum))] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum IntegrityCheckType { #[default] Quick, @@ -62,7 +60,7 @@ pub enum IntegrityCheckType { #[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)] #[enum_display(case = "Kebab")] -#[cfg_attr(feature = "cli", derive(ValueEnum))] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum AggHashType { /// Verify that the aggregate tiles hash value in the metadata table matches the computed value. Used by default. #[default] From b77bc892bff4817ec90104a41167427fbacfa6d0 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 16 Dec 2023 11:49:21 -0500 Subject: [PATCH 3/5] Minor test snapshot cleanup (#1074) --- mbtiles/tests/copy.rs | 8 +- ...snap => copy__convert@v1__bbox__flat.snap} | 0 .../copy__convert@v1__bbox__hash-flat.snap | 42 --------- .../copy__convert@v1__bbox__hash-hash.snap | 50 ----------- .../copy__convert@v1__bbox__hash-norm.snap | 85 ------------------- ...snap => copy__convert@v1__bbox__hash.snap} | 0 .../copy__convert@v1__bbox__norm-flat.snap | 42 --------- .../copy__convert@v1__bbox__norm-hash.snap | 50 ----------- .../copy__convert@v1__bbox__norm-norm.snap | 85 ------------------- ...snap => copy__convert@v1__bbox__norm.snap} | 0 ...t.snap => copy__convert@v1__z6__flat.snap} | 0 .../copy__convert@v1__z6__hash-flat.snap | 42 --------- .../copy__convert@v1__z6__hash-hash.snap | 50 ----------- .../copy__convert@v1__z6__hash-norm.snap | 85 ------------------- ...h.snap => copy__convert@v1__z6__hash.snap} | 0 .../copy__convert@v1__z6__norm-flat.snap | 42 --------- .../copy__convert@v1__z6__norm-hash.snap | 50 ----------- .../copy__convert@v1__z6__norm-norm.snap | 85 ------------------- ...m.snap => copy__convert@v1__z6__norm.snap} | 0 19 files changed, 6 insertions(+), 710 deletions(-) rename mbtiles/tests/snapshots/{copy__convert@v1__bbox__flat-flat.snap => copy__convert@v1__bbox__flat.snap} (100%) delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-flat.snap delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-hash.snap delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-norm.snap rename mbtiles/tests/snapshots/{copy__convert@v1__bbox__flat-hash.snap => copy__convert@v1__bbox__hash.snap} (100%) delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-flat.snap delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-hash.snap delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-norm.snap rename mbtiles/tests/snapshots/{copy__convert@v1__bbox__flat-norm.snap => copy__convert@v1__bbox__norm.snap} (100%) rename mbtiles/tests/snapshots/{copy__convert@v1__z6__flat-flat.snap => copy__convert@v1__z6__flat.snap} (100%) delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__z6__hash-flat.snap delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__z6__hash-hash.snap delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__z6__hash-norm.snap rename mbtiles/tests/snapshots/{copy__convert@v1__z6__flat-hash.snap => copy__convert@v1__z6__hash.snap} (100%) delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__z6__norm-flat.snap delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__z6__norm-hash.snap delete mode 100644 mbtiles/tests/snapshots/copy__convert@v1__z6__norm-norm.snap rename mbtiles/tests/snapshots/{copy__convert@v1__z6__flat-norm.snap => copy__convert@v1__z6__norm.snap} (100%) diff --git a/mbtiles/tests/copy.rs b/mbtiles/tests/copy.rs index b5d59a16b..e99125a43 100644 --- a/mbtiles/tests/copy.rs +++ b/mbtiles/tests/copy.rs @@ -257,7 +257,9 @@ async fn convert( opt.dst_type_cli = Some(dst_type); opt.zoom_levels.push(6); let z6only = dump(&mut opt.run().await?).await?; - assert_snapshot!(z6only, "v1__z6__{frm}-{to}"); + allow_duplicates! { + assert_snapshot!(z6only, "v1__z6__{to}"); + } let mut opt = copier(&frm_mbt, &mem); opt.dst_type_cli = Some(dst_type); @@ -271,7 +273,9 @@ async fn convert( opt.bbox.push(bbox.into()); let dmp = dump(&mut opt.run().await?).await?; - assert_snapshot!(dmp, "v1__bbox__{frm}-{to}"); + allow_duplicates! { + assert_snapshot!(dmp, "v1__bbox__{to}"); + } let mut opt = copier(&frm_mbt, &mem); opt.dst_type_cli = Some(dst_type); diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat.snap similarity index 100% rename from mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-flat.snap rename to mbtiles/tests/snapshots/copy__convert@v1__bbox__flat.snap diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-flat.snap deleted file mode 100644 index 510f57545..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-flat.snap +++ /dev/null @@ -1,42 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'table' -tbl_name = 'tiles' -sql = ''' -CREATE TABLE tiles ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_data blob, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 5, 1, 1, blob(edit-v1) )', - '( 5, 1, 2, blob() )', - '( 5, 2, 2, blob(remove) )', - '( 6, 1, 4, blob(edit-v1) )', -] - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'index' -tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-hash.snap deleted file mode 100644 index d7da64239..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-hash.snap +++ /dev/null @@ -1,50 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'table' -tbl_name = 'tiles_with_hash' -sql = ''' -CREATE TABLE tiles_with_hash ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_data blob, - tile_hash text, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', - '( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', - '( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )', - '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', -] - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'index' -tbl_name = 'tiles_with_hash' - -[[]] -type = 'view' -tbl_name = 'tiles' -sql = ''' -CREATE VIEW tiles AS - SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-norm.snap deleted file mode 100644 index ae37bc9bf..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash-norm.snap +++ /dev/null @@ -1,85 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'images' -sql = ''' -CREATE TABLE images ( - tile_id text NOT NULL PRIMARY KEY, - tile_data blob)''' -values = [ - '( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )', - '( "D41D8CD98F00B204E9800998ECF8427E", blob() )', - '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', -] - -[[]] -type = 'table' -tbl_name = 'map' -sql = ''' -CREATE TABLE map ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_id text, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', - '( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )', - '( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )', - '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', -] - -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'index' -tbl_name = 'images' - -[[]] -type = 'index' -tbl_name = 'map' - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'view' -tbl_name = 'tiles' -sql = ''' -CREATE VIEW tiles AS - SELECT map.zoom_level AS zoom_level, - map.tile_column AS tile_column, - map.tile_row AS tile_row, - images.tile_data AS tile_data - FROM map - JOIN images ON images.tile_id = map.tile_id''' - -[[]] -type = 'view' -tbl_name = 'tiles_with_hash' -sql = ''' -CREATE VIEW tiles_with_hash AS - SELECT - map.zoom_level AS zoom_level, - map.tile_column AS tile_column, - map.tile_row AS tile_row, - images.tile_data AS tile_data, - images.tile_id AS tile_hash - FROM map - JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__hash.snap similarity index 100% rename from mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-hash.snap rename to mbtiles/tests/snapshots/copy__convert@v1__bbox__hash.snap diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-flat.snap deleted file mode 100644 index 510f57545..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-flat.snap +++ /dev/null @@ -1,42 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'table' -tbl_name = 'tiles' -sql = ''' -CREATE TABLE tiles ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_data blob, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 5, 1, 1, blob(edit-v1) )', - '( 5, 1, 2, blob() )', - '( 5, 2, 2, blob(remove) )', - '( 6, 1, 4, blob(edit-v1) )', -] - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'index' -tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-hash.snap deleted file mode 100644 index d7da64239..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-hash.snap +++ /dev/null @@ -1,50 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'table' -tbl_name = 'tiles_with_hash' -sql = ''' -CREATE TABLE tiles_with_hash ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_data blob, - tile_hash text, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', - '( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', - '( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )', - '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', -] - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'index' -tbl_name = 'tiles_with_hash' - -[[]] -type = 'view' -tbl_name = 'tiles' -sql = ''' -CREATE VIEW tiles AS - SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-norm.snap deleted file mode 100644 index ae37bc9bf..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm-norm.snap +++ /dev/null @@ -1,85 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'images' -sql = ''' -CREATE TABLE images ( - tile_id text NOT NULL PRIMARY KEY, - tile_data blob)''' -values = [ - '( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )', - '( "D41D8CD98F00B204E9800998ECF8427E", blob() )', - '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', -] - -[[]] -type = 'table' -tbl_name = 'map' -sql = ''' -CREATE TABLE map ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_id text, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', - '( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )', - '( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )', - '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', -] - -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "012434681F0EBF296906D6608C54D632" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'index' -tbl_name = 'images' - -[[]] -type = 'index' -tbl_name = 'map' - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'view' -tbl_name = 'tiles' -sql = ''' -CREATE VIEW tiles AS - SELECT map.zoom_level AS zoom_level, - map.tile_column AS tile_column, - map.tile_row AS tile_row, - images.tile_data AS tile_data - FROM map - JOIN images ON images.tile_id = map.tile_id''' - -[[]] -type = 'view' -tbl_name = 'tiles_with_hash' -sql = ''' -CREATE VIEW tiles_with_hash AS - SELECT - map.zoom_level AS zoom_level, - map.tile_column AS tile_column, - map.tile_row AS tile_row, - images.tile_data AS tile_data, - images.tile_id AS tile_hash - FROM map - JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__bbox__norm.snap similarity index 100% rename from mbtiles/tests/snapshots/copy__convert@v1__bbox__flat-norm.snap rename to mbtiles/tests/snapshots/copy__convert@v1__bbox__norm.snap diff --git a/mbtiles/tests/snapshots/copy__convert@v1__z6__flat-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__z6__flat.snap similarity index 100% rename from mbtiles/tests/snapshots/copy__convert@v1__z6__flat-flat.snap rename to mbtiles/tests/snapshots/copy__convert@v1__z6__flat.snap diff --git a/mbtiles/tests/snapshots/copy__convert@v1__z6__hash-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__z6__hash-flat.snap deleted file mode 100644 index 07c6751ee..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__z6__hash-flat.snap +++ /dev/null @@ -1,42 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "675349A4153AEC0679BE9C0637AEEBCC" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'table' -tbl_name = 'tiles' -sql = ''' -CREATE TABLE tiles ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_data blob, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 6, 0, 3, blob(same) )', - '( 6, 0, 5, blob(1-keep-1-rm) )', - '( 6, 1, 4, blob(edit-v1) )', - '( 6, 2, 6, blob(1-keep-1-rm) )', -] - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'index' -tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__z6__hash-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__z6__hash-hash.snap deleted file mode 100644 index 2c5e7f773..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__z6__hash-hash.snap +++ /dev/null @@ -1,50 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "675349A4153AEC0679BE9C0637AEEBCC" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'table' -tbl_name = 'tiles_with_hash' -sql = ''' -CREATE TABLE tiles_with_hash ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_data blob, - tile_hash text, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 6, 0, 3, blob(same), "51037A4A37730F52C8732586D3AAA316" )', - '( 6, 0, 5, blob(1-keep-1-rm), "535A5575B48444EDEB926815AB26EC9B" )', - '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', - '( 6, 2, 6, blob(1-keep-1-rm), "535A5575B48444EDEB926815AB26EC9B" )', -] - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'index' -tbl_name = 'tiles_with_hash' - -[[]] -type = 'view' -tbl_name = 'tiles' -sql = ''' -CREATE VIEW tiles AS - SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__z6__hash-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__z6__hash-norm.snap deleted file mode 100644 index 7dd593b82..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__z6__hash-norm.snap +++ /dev/null @@ -1,85 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'images' -sql = ''' -CREATE TABLE images ( - tile_id text NOT NULL PRIMARY KEY, - tile_data blob)''' -values = [ - '( "51037A4A37730F52C8732586D3AAA316", blob(same) )', - '( "535A5575B48444EDEB926815AB26EC9B", blob(1-keep-1-rm) )', - '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', -] - -[[]] -type = 'table' -tbl_name = 'map' -sql = ''' -CREATE TABLE map ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_id text, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 6, 0, 3, "51037A4A37730F52C8732586D3AAA316" )', - '( 6, 0, 5, "535A5575B48444EDEB926815AB26EC9B" )', - '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', - '( 6, 2, 6, "535A5575B48444EDEB926815AB26EC9B" )', -] - -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "675349A4153AEC0679BE9C0637AEEBCC" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'index' -tbl_name = 'images' - -[[]] -type = 'index' -tbl_name = 'map' - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'view' -tbl_name = 'tiles' -sql = ''' -CREATE VIEW tiles AS - SELECT map.zoom_level AS zoom_level, - map.tile_column AS tile_column, - map.tile_row AS tile_row, - images.tile_data AS tile_data - FROM map - JOIN images ON images.tile_id = map.tile_id''' - -[[]] -type = 'view' -tbl_name = 'tiles_with_hash' -sql = ''' -CREATE VIEW tiles_with_hash AS - SELECT - map.zoom_level AS zoom_level, - map.tile_column AS tile_column, - map.tile_row AS tile_row, - images.tile_data AS tile_data, - images.tile_id AS tile_hash - FROM map - JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__z6__flat-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__z6__hash.snap similarity index 100% rename from mbtiles/tests/snapshots/copy__convert@v1__z6__flat-hash.snap rename to mbtiles/tests/snapshots/copy__convert@v1__z6__hash.snap diff --git a/mbtiles/tests/snapshots/copy__convert@v1__z6__norm-flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__z6__norm-flat.snap deleted file mode 100644 index 07c6751ee..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__z6__norm-flat.snap +++ /dev/null @@ -1,42 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "675349A4153AEC0679BE9C0637AEEBCC" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'table' -tbl_name = 'tiles' -sql = ''' -CREATE TABLE tiles ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_data blob, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 6, 0, 3, blob(same) )', - '( 6, 0, 5, blob(1-keep-1-rm) )', - '( 6, 1, 4, blob(edit-v1) )', - '( 6, 2, 6, blob(1-keep-1-rm) )', -] - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'index' -tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__z6__norm-hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__z6__norm-hash.snap deleted file mode 100644 index 2c5e7f773..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__z6__norm-hash.snap +++ /dev/null @@ -1,50 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "675349A4153AEC0679BE9C0637AEEBCC" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'table' -tbl_name = 'tiles_with_hash' -sql = ''' -CREATE TABLE tiles_with_hash ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_data blob, - tile_hash text, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 6, 0, 3, blob(same), "51037A4A37730F52C8732586D3AAA316" )', - '( 6, 0, 5, blob(1-keep-1-rm), "535A5575B48444EDEB926815AB26EC9B" )', - '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', - '( 6, 2, 6, blob(1-keep-1-rm), "535A5575B48444EDEB926815AB26EC9B" )', -] - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'index' -tbl_name = 'tiles_with_hash' - -[[]] -type = 'view' -tbl_name = 'tiles' -sql = ''' -CREATE VIEW tiles AS - SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__z6__norm-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__z6__norm-norm.snap deleted file mode 100644 index 7dd593b82..000000000 --- a/mbtiles/tests/snapshots/copy__convert@v1__z6__norm-norm.snap +++ /dev/null @@ -1,85 +0,0 @@ ---- -source: mbtiles/tests/copy.rs -expression: actual_value ---- -[[]] -type = 'table' -tbl_name = 'images' -sql = ''' -CREATE TABLE images ( - tile_id text NOT NULL PRIMARY KEY, - tile_data blob)''' -values = [ - '( "51037A4A37730F52C8732586D3AAA316", blob(same) )', - '( "535A5575B48444EDEB926815AB26EC9B", blob(1-keep-1-rm) )', - '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', -] - -[[]] -type = 'table' -tbl_name = 'map' -sql = ''' -CREATE TABLE map ( - zoom_level integer NOT NULL, - tile_column integer NOT NULL, - tile_row integer NOT NULL, - tile_id text, - PRIMARY KEY(zoom_level, tile_column, tile_row))''' -values = [ - '( 6, 0, 3, "51037A4A37730F52C8732586D3AAA316" )', - '( 6, 0, 5, "535A5575B48444EDEB926815AB26EC9B" )', - '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', - '( 6, 2, 6, "535A5575B48444EDEB926815AB26EC9B" )', -] - -[[]] -type = 'table' -tbl_name = 'metadata' -sql = ''' -CREATE TABLE metadata ( - name text NOT NULL PRIMARY KEY, - value text)''' -values = [ - '( "agg_tiles_hash", "675349A4153AEC0679BE9C0637AEEBCC" )', - '( "md-edit", "value - v1" )', - '( "md-remove", "value - remove" )', - '( "md-same", "value - same" )', -] - -[[]] -type = 'index' -tbl_name = 'images' - -[[]] -type = 'index' -tbl_name = 'map' - -[[]] -type = 'index' -tbl_name = 'metadata' - -[[]] -type = 'view' -tbl_name = 'tiles' -sql = ''' -CREATE VIEW tiles AS - SELECT map.zoom_level AS zoom_level, - map.tile_column AS tile_column, - map.tile_row AS tile_row, - images.tile_data AS tile_data - FROM map - JOIN images ON images.tile_id = map.tile_id''' - -[[]] -type = 'view' -tbl_name = 'tiles_with_hash' -sql = ''' -CREATE VIEW tiles_with_hash AS - SELECT - map.zoom_level AS zoom_level, - map.tile_column AS tile_column, - map.tile_row AS tile_row, - images.tile_data AS tile_data, - images.tile_id AS tile_hash - FROM map - JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__z6__flat-norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__z6__norm.snap similarity index 100% rename from mbtiles/tests/snapshots/copy__convert@v1__z6__flat-norm.snap rename to mbtiles/tests/snapshots/copy__convert@v1__z6__norm.snap From 15a521dcdc6918dd2d90dd2c2616872038a6b763 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 16 Dec 2023 13:04:16 -0500 Subject: [PATCH 4/5] Revert "update artifact actions (#1067)" Due to https://github.com/actions/download-artifact/issues/249 This reverts commit 780492c10b851b168437fea0e65762d4d3bddae7. --- .github/workflows/ci.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0a3fcce02..efba3140d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,7 +154,7 @@ jobs: done - name: Save build artifacts to build-${{ matrix.target }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: cross-build path: target_releases/* @@ -282,7 +282,7 @@ jobs: mv target/${{ matrix.target }}/release/martin-cp${{ matrix.ext }} target_releases/ mv target/${{ matrix.target }}/release/mbtiles${{ matrix.ext }} target_releases/ - name: Save build artifacts to build-${{ matrix.target }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: build-${{ matrix.target }} path: target_releases/* @@ -324,7 +324,7 @@ jobs: env: DATABASE_URL: ${{ steps.pg.outputs.connection-uri }} - name: Download build artifact build-${{ matrix.target }} - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: build-${{ matrix.target }} path: target/ @@ -345,7 +345,7 @@ jobs: run: diff --brief --recursive --new-file tests/output tests/expected - name: Download Debian package (Linux) if: matrix.target == 'x86_64-unknown-linux-gnu' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: build-debian-x86_64 path: target/ @@ -362,7 +362,7 @@ jobs: DATABASE_URL: ${{ steps.pg.outputs.connection-uri }} - name: Save test output on failure if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: failed-test-output-${{ runner.os }} path: | @@ -439,7 +439,7 @@ jobs: docker cp ${{ job.services.postgres.id }}:/etc/ssl/certs/ssl-cert-snakeoil.pem target/certs/server.crt docker cp ${{ job.services.postgres.id }}:/etc/ssl/private/ssl-cert-snakeoil.key target/certs/server.key - name: Download build artifact build-x86_64-unknown-linux-gnu - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: build-x86_64-unknown-linux-gnu path: target_releases/ @@ -458,7 +458,7 @@ jobs: env: DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }} - name: Download Debian package - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: build-debian-x86_64 path: target_releases/ @@ -490,7 +490,7 @@ jobs: DATABASE_URL: postgres://${{ env.PGUSER }}:${{ env.PGUSER }}@${{ env.PGHOST }}:${{ job.services.postgres.ports[5432] }}/${{ env.PGDATABASE }}?sslmode=${{ matrix.sslmode }} - name: On error, save test output if: failure() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: test-output path: | @@ -506,34 +506,34 @@ jobs: - name: Checkout sources uses: actions/checkout@v4 - name: Download build artifact build-aarch64-apple-darwin - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: build-aarch64-apple-darwin path: target/aarch64-apple-darwin - name: Download build artifact build-x86_64-apple-darwin - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: build-x86_64-apple-darwin path: target/x86_64-apple-darwin - name: Download build artifact build-x86_64-unknown-linux-gnu - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: build-x86_64-unknown-linux-gnu path: target/x86_64-unknown-linux-gnu - name: Download cross-build artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: cross-build path: target/cross - name: Download build artifact build-x86_64-pc-windows-msvc - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: build-x86_64-pc-windows-msvc path: target/x86_64-pc-windows-msvc - name: Download build artifact build-debian-x86_64 - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v3 with: name: build-debian-x86_64 path: target/debian-x86_64 @@ -604,7 +604,7 @@ jobs: EOF - name: Save Homebrew Config - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v3 with: name: homebrew-config path: target/homebrew_config.yaml From de6b681d74a7f4ea7ff61200fbc3ac1017aeae1f Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Sat, 16 Dec 2023 14:59:52 -0500 Subject: [PATCH 5/5] New `mbtiles copy --copy (all|tiles|metadata)` flag to limit what gets copied (#1073) Limit what gets copied from one mbtiles to another. Closes #1069 --- mbtiles/src/bin/mbtiles.rs | 47 +++-- mbtiles/src/copier.rs | 195 ++++++++++-------- mbtiles/src/lib.rs | 2 +- mbtiles/src/mbtiles.rs | 21 ++ mbtiles/src/patcher.rs | 20 +- mbtiles/src/validation.rs | 4 +- mbtiles/tests/copy.rs | 144 +++++++++---- .../copy__convert@v1__meta__flat.snap | 37 ++++ .../copy__convert@v1__meta__hash.snap | 45 ++++ .../copy__convert@v1__meta__norm.snap | 76 +++++++ .../copy__convert@v1__tiles__flat.snap | 45 ++++ .../copy__convert@v1__tiles__hash.snap | 53 +++++ .../copy__convert@v1__tiles__norm.snap | 92 +++++++++ 13 files changed, 638 insertions(+), 143 deletions(-) create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__meta__flat.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__meta__hash.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__meta__norm.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__tiles__flat.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__tiles__hash.snap create mode 100644 mbtiles/tests/snapshots/copy__convert@v1__tiles__norm.snap diff --git a/mbtiles/src/bin/mbtiles.rs b/mbtiles/src/bin/mbtiles.rs index ebfb5e631..4c7460166 100644 --- a/mbtiles/src/bin/mbtiles.rs +++ b/mbtiles/src/bin/mbtiles.rs @@ -3,8 +3,8 @@ use std::path::{Path, PathBuf}; use clap::{Parser, Subcommand}; use log::error; use mbtiles::{ - apply_patch, AggHashType, CopyDuplicateMode, IntegrityCheckType, MbtResult, MbtTypeCli, - Mbtiles, MbtilesCopier, + apply_patch, AggHashType, CopyDuplicateMode, CopyType, IntegrityCheckType, MbtResult, + MbtTypeCli, Mbtiles, MbtilesCopier, }; use tilejson::Bounds; @@ -83,38 +83,42 @@ enum Commands { #[derive(Clone, Default, PartialEq, Debug, clap::Args)] pub struct CopyArgs { /// MBTiles file to read from - pub src_file: PathBuf, + src_file: PathBuf, /// MBTiles file to write to - pub dst_file: PathBuf, + dst_file: PathBuf, + /// Limit what gets copied. + /// When copying tiles only, the agg_tiles_hash will still be updated unless --skip-agg-tiles-hash is set. + #[arg(long, value_name = "TYPE", default_value_t=CopyType::default())] + copy: CopyType, /// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source #[arg(long, alias = "dst-type", alias = "dst_type", value_name = "SCHEMA")] - pub mbtiles_type: Option, + mbtiles_type: Option, /// Allow copying to existing files, and indicate what to do if a tile with the same Z/X/Y already exists #[arg(long, value_enum)] - pub on_duplicate: Option, + on_duplicate: Option, /// Minimum zoom level to copy #[arg(long, conflicts_with("zoom_levels"))] - pub min_zoom: Option, + min_zoom: Option, /// Maximum zoom level to copy #[arg(long, conflicts_with("zoom_levels"))] - pub max_zoom: Option, + max_zoom: Option, /// List of zoom levels to copy #[arg(long, value_delimiter = ',')] - pub zoom_levels: Vec, + zoom_levels: Vec, /// Bounding box to copy, in the format `min_lon,min_lat,max_lon,max_lat`. Can be used multiple times. #[arg(long)] - pub bbox: Vec, + bbox: Vec, /// Compare source file with this file, and only copy non-identical tiles to destination. /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. #[arg(long, conflicts_with("apply_patch"))] - pub diff_with_file: Option, + diff_with_file: Option, /// Compare source file with this file, and only copy non-identical tiles to destination. /// It should be later possible to run `mbtiles apply-diff SRC_FILE DST_FILE` to get the same DIFF file. #[arg(long, conflicts_with("diff_with_file"))] - pub apply_patch: Option, + apply_patch: Option, /// Skip generating a global hash for mbtiles validation. By default, `mbtiles` will compute `agg_tiles_hash` metadata value. #[arg(long)] - pub skip_agg_tiles_hash: bool, + skip_agg_tiles_hash: bool, } #[tokio::main] @@ -149,6 +153,7 @@ async fn main_int() -> anyhow::Result<()> { let opts = MbtilesCopier { src_file: opts.src_file, dst_file: opts.dst_file, + copy: opts.copy, dst_type_cli: opts.mbtiles_type, dst_type: None, on_duplicate: opts.on_duplicate, @@ -396,6 +401,22 @@ mod tests { ); } + #[test] + fn test_copy_limit() { + assert_eq!( + Args::parse_from(["mbtiles", "copy", "src_file", "dst_file", "--copy", "metadata"]), + Args { + verbose: false, + command: Copy(CopyArgs { + src_file: PathBuf::from("src_file"), + dst_file: PathBuf::from("dst_file"), + copy: CopyType::Metadata, + ..Default::default() + }) + } + ); + } + #[test] fn test_meta_get_no_arguments() { assert_eq!( diff --git a/mbtiles/src/copier.rs b/mbtiles/src/copier.rs index e59763ab5..7270ef397 100644 --- a/mbtiles/src/copier.rs +++ b/mbtiles/src/copier.rs @@ -16,11 +16,11 @@ use crate::queries::{ }; use crate::MbtType::{Flat, FlatWithHash, Normalized}; use crate::{ - invert_y_value, reset_db_settings, MbtError, MbtType, MbtTypeCli, Mbtiles, AGG_TILES_HASH, - AGG_TILES_HASH_IN_DIFF, + invert_y_value, reset_db_settings, CopyType, MbtError, MbtType, MbtTypeCli, Mbtiles, + AGG_TILES_HASH, AGG_TILES_HASH_IN_DIFF, }; -#[derive(PartialEq, Eq, Debug, Clone, Copy, EnumDisplay, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)] #[enum_display(case = "Kebab")] #[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum CopyDuplicateMode { @@ -46,6 +46,8 @@ pub struct MbtilesCopier { pub src_file: PathBuf, /// MBTiles file to write to pub dst_file: PathBuf, + /// Limit what gets copied + pub copy: CopyType, /// Output format of the destination file, ignored if the file exists. If not specified, defaults to the type of source pub dst_type_cli: Option, /// Destination type with options @@ -78,24 +80,6 @@ struct MbtileCopierInt { } impl MbtilesCopier { - #[must_use] - pub fn new(src_filepath: PathBuf, dst_filepath: PathBuf) -> Self { - Self { - src_file: src_filepath, - dst_file: dst_filepath, - dst_type_cli: None, - dst_type: None, - on_duplicate: None, - min_zoom: None, - max_zoom: None, - zoom_levels: Vec::default(), - bbox: vec![], - diff_with_file: None, - apply_patch: None, - skip_agg_tiles_hash: false, - } - } - pub async fn run(self) -> MbtResult { MbtileCopierInt::new(self)?.run().await } @@ -166,6 +150,11 @@ impl MbtileCopierInt { src_mbt.attach_to(&mut conn, "sourceDb").await?; + let what = match self.options.copy { + CopyType::All => "", + CopyType::Tiles => "tiles data ", + CopyType::Metadata => "metadata ", + }; let dst_type: MbtType; if let Some((dif_mbt, dif_type, _)) = &dif { if !is_empty_db { @@ -175,39 +164,24 @@ impl MbtileCopierInt { dif_mbt.attach_to(&mut conn, "diffDb").await?; let dif_path = dif_mbt.filepath(); if self.options.diff_with_file.is_some() { - info!("Comparing {src_mbt} ({src_type}) and {dif_path} ({dif_type}) into a new file {dst_mbt} ({dst_type})"); + info!("Comparing {src_mbt} ({src_type}) and {dif_path} ({dif_type}) {what}into a new file {dst_mbt} ({dst_type})"); } else { - info!("Applying patch from {dif_path} ({dif_type}) to {src_mbt} ({src_type}) into a new file {dst_mbt} ({dst_type})"); + info!("Applying patch from {dif_path} ({dif_type}) to {src_mbt} ({src_type}) {what}into a new file {dst_mbt} ({dst_type})"); } } else if is_empty_db { dst_type = self.options.dst_type().unwrap_or(src_type); - info!("Copying {src_mbt} ({src_type}) to a new file {dst_mbt} ({dst_type})"); + info!("Copying {src_mbt} ({src_type}) {what}to a new file {dst_mbt} ({dst_type})"); } else { dst_type = self.validate_dst_type(dst_mbt.detect_type(&mut conn).await?)?; - info!("Copying {src_mbt} ({src_type}) to an existing file {dst_mbt} ({dst_type})"); + info!( + "Copying {src_mbt} ({src_type}) {what}to an existing file {dst_mbt} ({dst_type})" + ); } if is_empty_db { self.init_new_schema(&mut conn, src_type, dst_type).await?; } - let select_from = if let Some((_, dif_type, _)) = &dif { - if self.options.diff_with_file.is_some() { - Self::get_select_from_with_diff(*dif_type, dst_type) - } else { - Self::get_select_from_apply_patch(src_type, *dif_type, dst_type) - } - } else { - Self::get_select_from(src_type, dst_type).to_string() - }; - - let where_clause = self.get_where_clause(); - let select_from = format!("{select_from} {where_clause}"); - let on_dupl = on_duplicate.to_sql(); - let sql_cond = Self::get_on_duplicate_sql_cond(on_duplicate, dst_type); - - debug!("Copying tiles with 'INSERT {on_dupl}' {src_type} -> {dst_type} ({sql_cond})"); - { // SAFETY: This must be scoped to make sure the handle is dropped before we continue using conn // Make sure not to execute any other queries while the handle is locked @@ -217,12 +191,20 @@ impl MbtileCopierInt { // SAFETY: this is safe as long as handle_lock is valid. We will drop the lock. let rusqlite_conn = unsafe { Connection::from_handle(handle) }?; - Self::copy_tiles(&rusqlite_conn, dst_type, on_dupl, &select_from, &sql_cond)?; + if self.options.copy.copy_tiles() { + self.copy_tiles(&rusqlite_conn, &dif, src_type, dst_type, on_duplicate)?; + } else { + debug!("Skipping copying tiles"); + } - self.copy_metadata(&rusqlite_conn, &dif, on_dupl)?; + if self.options.copy.copy_metadata() { + self.copy_metadata(&rusqlite_conn, &dif, on_duplicate)?; + } else { + debug!("Skipping copying metadata"); + } } - if !self.options.skip_agg_tiles_hash { + if self.options.copy.copy_tiles() && !self.options.skip_agg_tiles_hash { dst_mbt.update_agg_tiles_hash(&mut conn).await?; } @@ -237,8 +219,9 @@ impl MbtileCopierInt { &self, rusqlite_conn: &Connection, dif: &Option<(Mbtiles, MbtType, MbtType)>, - on_dupl: &str, + on_duplicate: CopyDuplicateMode, ) -> Result<(), MbtError> { + let on_dupl = on_duplicate.to_sql(); let sql; if dif.is_some() { // Insert all rows from diffDb.metadata if they do not exist or are different in sourceDb.metadata. @@ -292,19 +275,35 @@ impl MbtileCopierInt { } fn copy_tiles( + &self, rusqlite_conn: &Connection, + dif: &Option<(Mbtiles, MbtType, MbtType)>, + src_type: MbtType, dst_type: MbtType, - on_dupl: &str, - select_from: &str, - sql_cond: &str, + on_duplicate: CopyDuplicateMode, ) -> Result<(), MbtError> { + let on_dupl = on_duplicate.to_sql(); + + let select_from = if let Some((_, dif_type, _)) = &dif { + if self.options.diff_with_file.is_some() { + Self::get_select_from_with_diff(*dif_type, dst_type) + } else { + Self::get_select_from_apply_patch(src_type, *dif_type, dst_type) + } + } else { + Self::get_select_from(src_type, dst_type).to_string() + }; + + let where_clause = self.get_where_clause(); + let sql_cond = Self::get_on_duplicate_sql_cond(on_duplicate, dst_type); + let sql = match dst_type { Flat => { format!( " INSERT {on_dupl} INTO tiles (zoom_level, tile_column, tile_row, tile_data) - {select_from} {sql_cond}" + {select_from} {where_clause} {sql_cond}" ) } FlatWithHash => { @@ -312,7 +311,7 @@ impl MbtileCopierInt { " INSERT {on_dupl} INTO tiles_with_hash (zoom_level, tile_column, tile_row, tile_data, tile_hash) - {select_from} {sql_cond}" + {select_from} {where_clause} {sql_cond}" ) } Normalized { .. } => { @@ -321,7 +320,7 @@ impl MbtileCopierInt { INSERT OR IGNORE INTO images (tile_id, tile_data) SELECT tile_hash as tile_id, tile_data - FROM ({select_from})" + FROM ({select_from} {where_clause})" ); debug!("Copying to {dst_type} with {sql}"); rusqlite_conn.execute(&sql, [])?; @@ -331,7 +330,7 @@ impl MbtileCopierInt { INSERT {on_dupl} INTO map (zoom_level, tile_column, tile_row, tile_id) SELECT zoom_level, tile_column, tile_row, tile_hash as tile_id - FROM ({select_from} {sql_cond})" + FROM ({select_from} {where_clause} {sql_cond})" ) } }; @@ -634,8 +633,12 @@ mod tests { dst_type_cli: Option, expected_dst_type: MbtType, ) -> MbtResult<()> { - let mut opt = MbtilesCopier::new(src_filepath.clone(), dst_filepath.clone()); - opt.dst_type_cli = dst_type_cli; + let opt = MbtilesCopier { + src_file: src_filepath.clone(), + dst_file: dst_filepath.clone(), + dst_type_cli, + ..Default::default() + }; let mut dst_conn = opt.run().await?; Mbtiles::new(src_filepath)? @@ -750,22 +753,26 @@ mod tests { #[actix_rt::test] async fn copy_with_min_max_zoom() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); - let dst = PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared"); - let mut opt = MbtilesCopier::new(src, dst); - opt.min_zoom = Some(2); - opt.max_zoom = Some(4); + let opt = MbtilesCopier { + src_file: PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"), + dst_file: PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared"), + min_zoom: Some(2), + max_zoom: Some(4), + ..Default::default() + }; verify_copy_with_zoom_filter(opt, 3).await } #[actix_rt::test] async fn copy_with_zoom_levels() -> MbtResult<()> { - let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); - let dst = PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared"); - let mut opt = MbtilesCopier::new(src, dst); - opt.min_zoom = Some(2); - opt.max_zoom = Some(4); - opt.zoom_levels.extend(&[1, 6]); + let opt = MbtilesCopier { + src_file: PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"), + dst_file: PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared"), + min_zoom: Some(2), + max_zoom: Some(4), + zoom_levels: vec![1, 6], + ..Default::default() + }; verify_copy_with_zoom_filter(opt, 2).await } @@ -777,8 +784,12 @@ mod tests { let diff_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg-modified.mbtiles"); - let mut opt = MbtilesCopier::new(src.clone(), dst.clone()); - opt.diff_with_file = Some(diff_file.clone()); + let opt = MbtilesCopier { + src_file: src.clone(), + dst_file: dst.clone(), + diff_with_file: Some(diff_file.clone()), + ..Default::default() + }; let mut dst_conn = opt.run().await?; assert!(dst_conn @@ -820,8 +831,12 @@ mod tests { let src = PathBuf::from("../tests/fixtures/mbtiles/world_cities_modified.mbtiles"); let dst = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); - let mut opt = MbtilesCopier::new(src.clone(), dst.clone()); - opt.on_duplicate = Some(CopyDuplicateMode::Abort); + let opt = MbtilesCopier { + src_file: src.clone(), + dst_file: dst.clone(), + on_duplicate: Some(CopyDuplicateMode::Abort), + ..Default::default() + }; assert!(matches!( opt.run().await.unwrap_err(), @@ -838,12 +853,20 @@ mod tests { let dst = PathBuf::from("file:copy_to_existing_override_mode_mem_db?mode=memory&cache=shared"); - let _dst_conn = MbtilesCopier::new(dst_file.clone(), dst.clone()) - .run() - .await?; - - let mut opt = MbtilesCopier::new(src_file.clone(), dst.clone()); - opt.on_duplicate = Some(CopyDuplicateMode::Override); + let _dst_conn = MbtilesCopier { + src_file: dst_file.clone(), + dst_file: dst.clone(), + ..Default::default() + } + .run() + .await?; + + let opt = MbtilesCopier { + src_file: src_file.clone(), + dst_file: dst.clone(), + on_duplicate: Some(CopyDuplicateMode::Override), + ..Default::default() + }; let mut dst_conn = opt.run().await?; // Verify the tiles in the destination file is a superset of the tiles in the source file @@ -867,12 +890,20 @@ mod tests { let dst = PathBuf::from("file:copy_to_existing_ignore_mode_mem_db?mode=memory&cache=shared"); - let _dst_conn = MbtilesCopier::new(dst_file.clone(), dst.clone()) - .run() - .await?; - - let mut opt = MbtilesCopier::new(src_file.clone(), dst.clone()); - opt.on_duplicate = Some(CopyDuplicateMode::Ignore); + let _dst_conn = MbtilesCopier { + src_file: dst_file.clone(), + dst_file: dst.clone(), + ..Default::default() + } + .run() + .await?; + + let opt = MbtilesCopier { + src_file: src_file.clone(), + dst_file: dst.clone(), + on_duplicate: Some(CopyDuplicateMode::Ignore), + ..Default::default() + }; let mut dst_conn = opt.run().await?; // Verify the tiles in the destination file are the same as those in the source file except for those with duplicate (zoom_level, tile_column, tile_row) diff --git a/mbtiles/src/lib.rs b/mbtiles/src/lib.rs index 7cdc1a8d0..30be6032f 100644 --- a/mbtiles/src/lib.rs +++ b/mbtiles/src/lib.rs @@ -10,7 +10,7 @@ mod errors; pub use errors::{MbtError, MbtResult}; mod mbtiles; -pub use mbtiles::{MbtTypeCli, Mbtiles}; +pub use mbtiles::{CopyType, MbtTypeCli, Mbtiles}; mod metadata; pub use metadata::Metadata; diff --git a/mbtiles/src/mbtiles.rs b/mbtiles/src/mbtiles.rs index 64dcb16e5..fe62c7e6b 100644 --- a/mbtiles/src/mbtiles.rs +++ b/mbtiles/src/mbtiles.rs @@ -21,6 +21,27 @@ pub enum MbtTypeCli { Normalized, } +#[derive(Default, Debug, Clone, Copy, Hash, PartialEq, Eq, Serialize, Deserialize, EnumDisplay)] +#[enum_display(case = "Kebab")] +#[cfg_attr(feature = "cli", derive(clap::ValueEnum))] +pub enum CopyType { + #[default] + All, + Metadata, + Tiles, +} + +impl CopyType { + #[must_use] + pub fn copy_tiles(&self) -> bool { + matches!(self, Self::All | Self::Tiles) + } + #[must_use] + pub fn copy_metadata(&self) -> bool { + matches!(self, Self::All | Self::Metadata) + } +} + #[derive(Clone, Debug)] pub struct Mbtiles { filepath: String, diff --git a/mbtiles/src/patcher.rs b/mbtiles/src/patcher.rs index c1b68b302..063333208 100644 --- a/mbtiles/src/patcher.rs +++ b/mbtiles/src/patcher.rs @@ -148,9 +148,13 @@ mod tests { let src_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities.mbtiles"); let src = PathBuf::from("file:apply_flat_diff_file_mem_db?mode=memory&cache=shared"); - let mut src_conn = MbtilesCopier::new(src_file.clone(), src.clone()) - .run() - .await?; + let mut src_conn = MbtilesCopier { + src_file: src_file.clone(), + dst_file: src.clone(), + ..Default::default() + } + .run() + .await?; // Apply patch to the src data in in-memory DB let patch_file = PathBuf::from("../tests/fixtures/mbtiles/world_cities_diff.mbtiles"); @@ -175,9 +179,13 @@ mod tests { let src_file = PathBuf::from("../tests/fixtures/mbtiles/geography-class-jpg.mbtiles"); let src = PathBuf::from("file:apply_normalized_diff_file_mem_db?mode=memory&cache=shared"); - let mut src_conn = MbtilesCopier::new(src_file.clone(), src.clone()) - .run() - .await?; + let mut src_conn = MbtilesCopier { + src_file: src_file.clone(), + dst_file: src.clone(), + ..Default::default() + } + .run() + .await?; // Apply patch to the src data in in-memory DB let patch_file = diff --git a/mbtiles/src/validation.rs b/mbtiles/src/validation.rs index 57e9ebbc0..34ce483a8 100644 --- a/mbtiles/src/validation.rs +++ b/mbtiles/src/validation.rs @@ -48,7 +48,7 @@ impl MbtType { } } -#[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)] +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, EnumDisplay)] #[enum_display(case = "Kebab")] #[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum IntegrityCheckType { @@ -58,7 +58,7 @@ pub enum IntegrityCheckType { Off, } -#[derive(PartialEq, Eq, Default, Debug, Clone, EnumDisplay)] +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, EnumDisplay)] #[enum_display(case = "Kebab")] #[cfg_attr(feature = "cli", derive(clap::ValueEnum))] pub enum AggHashType { diff --git a/mbtiles/tests/copy.rs b/mbtiles/tests/copy.rs index e99125a43..3166fa16a 100644 --- a/mbtiles/tests/copy.rs +++ b/mbtiles/tests/copy.rs @@ -11,7 +11,8 @@ use mbtiles::AggHashType::Verify; use mbtiles::IntegrityCheckType::Off; use mbtiles::MbtTypeCli::{Flat, FlatWithHash, Normalized}; use mbtiles::{ - apply_patch, init_mbtiles_schema, invert_y_value, MbtResult, MbtTypeCli, Mbtiles, MbtilesCopier, + apply_patch, init_mbtiles_schema, invert_y_value, CopyType, MbtResult, MbtTypeCli, Mbtiles, + MbtilesCopier, }; use pretty_assertions::assert_eq as pretty_assert_eq; use rstest::{fixture, rstest}; @@ -87,10 +88,6 @@ fn path(mbt: &Mbtiles) -> PathBuf { PathBuf::from(mbt.filepath()) } -fn copier(src: &Mbtiles, dst: &Mbtiles) -> MbtilesCopier { - MbtilesCopier::new(path(src), path(dst)) -} - fn shorten(v: MbtTypeCli) -> &'static str { match v { Flat => "flat", @@ -135,9 +132,13 @@ macro_rules! new_file { cn_tmp.execute($sql_meta).await.unwrap(); let (dst_mbt, cn_dst) = open!($function, $($arg)*); - let mut opt = copier(&tmp_mbt, &dst_mbt); - opt.dst_type_cli = Some($dst_type_cli); - opt.skip_agg_tiles_hash = $skip_agg; + let opt = MbtilesCopier { + src_file: path(&tmp_mbt), + dst_file: path(&dst_mbt), + dst_type_cli: Some($dst_type_cli), + skip_agg_tiles_hash: $skip_agg, + ..Default::default() + }; opt.run().await.unwrap(); (dst_mbt, cn_dst) @@ -199,7 +200,12 @@ fn databases() -> Databases { let (v1_mbt, mut v1_cn) = open!(databases, "{typ}__v1"); let raw_mbt = result.mbtiles("v1_no_hash", mbt_typ); - copier(raw_mbt, &v1_mbt).run().await.unwrap(); + let opt = MbtilesCopier { + src_file: path(raw_mbt), + dst_file: path(&v1_mbt), + ..Default::default() + }; + opt.run().await.unwrap(); let dmp = dump(&mut v1_cn).await.unwrap(); assert_snapshot!(&dmp, "{typ}__v1"); let hash = v1_mbt.validate(Off, Verify).await.unwrap(); @@ -220,9 +226,13 @@ fn databases() -> Databases { let (dif_mbt, mut dif_cn) = open!(databases, "{typ}__dif"); let v1_mbt = result.mbtiles("v1", mbt_typ); - let mut opt = copier(v1_mbt, &dif_mbt); let v2_mbt = result.mbtiles("v2", mbt_typ); - opt.diff_with_file = Some(path(v2_mbt)); + let opt = MbtilesCopier { + src_file: path(v1_mbt), + dst_file: path(&dif_mbt), + diff_with_file: Some(path(v2_mbt)), + ..Default::default() + }; opt.run().await.unwrap(); let dmp = dump(&mut dif_cn).await.unwrap(); assert_snapshot!(&dmp, "{typ}__dif"); @@ -248,44 +258,87 @@ async fn convert( let mem = Mbtiles::new(":memory:")?; let (frm_mbt, _frm_cn) = new_file!(convert, frm_type, METADATA_V1, TILES_V1, "{frm}-{to}"); - let mut opt = copier(&frm_mbt, &mem); - opt.dst_type_cli = Some(dst_type); + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + dst_type_cli: Some(dst_type), + ..Default::default() + }; let dmp = dump(&mut opt.run().await?).await?; pretty_assert_eq!(databases.dump("v1", dst_type), &dmp); - let mut opt = copier(&frm_mbt, &mem); - opt.dst_type_cli = Some(dst_type); - opt.zoom_levels.push(6); + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + copy: CopyType::Metadata, + dst_type_cli: Some(dst_type), + ..Default::default() + }; + let dmp = dump(&mut opt.run().await?).await?; + allow_duplicates! { + assert_snapshot!(dmp, "v1__meta__{to}"); + }; + + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + copy: CopyType::Tiles, + dst_type_cli: Some(dst_type), + ..Default::default() + }; + let dmp = dump(&mut opt.run().await?).await?; + allow_duplicates! { + assert_snapshot!(dmp, "v1__tiles__{to}"); + } + + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + dst_type_cli: Some(dst_type), + zoom_levels: vec![6], + ..Default::default() + }; let z6only = dump(&mut opt.run().await?).await?; allow_duplicates! { assert_snapshot!(z6only, "v1__z6__{to}"); } - let mut opt = copier(&frm_mbt, &mem); - opt.dst_type_cli = Some(dst_type); - // Filter (0, 0, 2, 2) in mbtiles coordinates, which is (0, 2^5-1-2, 2, 2^5-1-0) = (0, 29, 2, 31) in XYZ coordinates, and slightly decrease it let mut bbox = xyz_to_bbox(5, 0, invert_y_value(5, 2), 2, invert_y_value(5, 0)); - bbox[0] += 180.0 * 0.1 / f64::from(1 << 5); - bbox[1] += 90.0 * 0.1 / f64::from(1 << 5); - bbox[2] -= 180.0 * 0.1 / f64::from(1 << 5); - bbox[3] -= 90.0 * 0.1 / f64::from(1 << 5); - opt.bbox.push(bbox.into()); - + let adjust = 90.0 * 0.1 / f64::from(1 << 5); + bbox[0] += adjust; + bbox[1] += adjust; + bbox[2] -= adjust; + bbox[3] -= adjust; + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + dst_type_cli: Some(dst_type), + bbox: vec![bbox.into()], + ..Default::default() + }; let dmp = dump(&mut opt.run().await?).await?; allow_duplicates! { assert_snapshot!(dmp, "v1__bbox__{to}"); } - let mut opt = copier(&frm_mbt, &mem); - opt.dst_type_cli = Some(dst_type); - opt.min_zoom = Some(6); + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + dst_type_cli: Some(dst_type), + min_zoom: Some(6), + ..Default::default() + }; pretty_assert_eq!(&z6only, &dump(&mut opt.run().await?).await?); - let mut opt = copier(&frm_mbt, &mem); - opt.dst_type_cli = Some(dst_type); - opt.min_zoom = Some(6); - opt.max_zoom = Some(6); + let opt = MbtilesCopier { + src_file: path(&frm_mbt), + dst_file: path(&mem), + dst_type_cli: Some(dst_type), + min_zoom: Some(6), + max_zoom: Some(6), + ..Default::default() + }; pretty_assert_eq!(&z6only, &dump(&mut opt.run().await?).await?); Ok(()) @@ -309,8 +362,12 @@ async fn diff_and_patch( let (dif_mbt, mut dif_cn) = open!(diff_and_patchdiff_and_patch, "{prefix}__dif"); info!("TEST: Compare v1 with v2, and copy anything that's different (i.e. mathematically: v2-v1=diff)"); - let mut opt = copier(v1_mbt, &dif_mbt); - opt.diff_with_file = Some(path(v2_mbt)); + let mut opt = MbtilesCopier { + src_file: path(v1_mbt), + dst_file: path(&dif_mbt), + diff_with_file: Some(path(v2_mbt)), + ..Default::default() + }; if let Some(dif_type) = dif_type { opt.dst_type_cli = Some(dif_type); } @@ -374,8 +431,12 @@ async fn patch_on_copy( let (v2_mbt, mut v2_cn) = open!(patch_on_copy, "{prefix}__v2"); info!("TEST: Compare v1 with v2, and copy anything that's different (i.e. mathematically: v2-v1=diff)"); - let mut opt = copier(v1_mbt, &v2_mbt); - opt.apply_patch = Some(path(dif_mbt)); + let mut opt = MbtilesCopier { + src_file: path(v1_mbt), + dst_file: path(&v2_mbt), + apply_patch: Some(path(dif_mbt)), + ..Default::default() + }; if let Some(v2_type) = v2_type { opt.dst_type_cli = Some(v2_type); } @@ -506,9 +567,14 @@ async fn dump(conn: &mut SqliteConnection) -> MbtResult> { } #[allow(dead_code)] -async fn save_to_file(source_mbt: &Mbtiles, path: &str) -> MbtResult<()> { - let mut opt = copier(source_mbt, &Mbtiles::new(path)?); - opt.skip_agg_tiles_hash = true; +async fn save_to_file(source_mbt: &Mbtiles, path_mbt: &str) -> MbtResult<()> { + let dst = &Mbtiles::new(path_mbt)?; + let opt = MbtilesCopier { + src_file: path(source_mbt), + dst_file: path(dst), + skip_agg_tiles_hash: true, + ..Default::default() + }; opt.run().await?; Ok(()) } diff --git a/mbtiles/tests/snapshots/copy__convert@v1__meta__flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__meta__flat.snap new file mode 100644 index 000000000..910232f8a --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__meta__flat.snap @@ -0,0 +1,37 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__meta__hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__meta__hash.snap new file mode 100644 index 000000000..548e2328e --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__meta__hash.snap @@ -0,0 +1,45 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__meta__norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__meta__norm.snap new file mode 100644 index 000000000..3aa32c6cd --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__meta__norm.snap @@ -0,0 +1,76 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = [ + '( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )', + '( "md-edit", "value - v1" )', + '( "md-remove", "value - remove" )', + '( "md-same", "value - same" )', +] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__tiles__flat.snap b/mbtiles/tests/snapshots/copy__convert@v1__tiles__flat.snap new file mode 100644 index 000000000..865b95c66 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__tiles__flat.snap @@ -0,0 +1,45 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = ['( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )'] + +[[]] +type = 'table' +tbl_name = 'tiles' +sql = ''' +CREATE TABLE tiles ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, blob(root) )', + '( 5, 0, 0, blob(same) )', + '( 5, 0, 1, blob() )', + '( 5, 1, 1, blob(edit-v1) )', + '( 5, 1, 2, blob() )', + '( 5, 1, 3, blob(non-empty) )', + '( 5, 2, 2, blob(remove) )', + '( 5, 2, 3, blob() )', + '( 6, 0, 3, blob(same) )', + '( 6, 0, 5, blob(1-keep-1-rm) )', + '( 6, 1, 4, blob(edit-v1) )', + '( 6, 2, 6, blob(1-keep-1-rm) )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__tiles__hash.snap b/mbtiles/tests/snapshots/copy__convert@v1__tiles__hash.snap new file mode 100644 index 000000000..3c5f391e9 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__tiles__hash.snap @@ -0,0 +1,53 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = ['( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )'] + +[[]] +type = 'table' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE TABLE tiles_with_hash ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_data blob, + tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, blob(root), "63A9F0EA7BB98050796B649E85481845" )', + '( 5, 0, 0, blob(same), "51037A4A37730F52C8732586D3AAA316" )', + '( 5, 0, 1, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 1, 1, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 1, 3, blob(non-empty), "720C02778717818CC0A869955BA2AFB6" )', + '( 5, 2, 2, blob(remove), "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 5, 2, 3, blob(), "D41D8CD98F00B204E9800998ECF8427E" )', + '( 6, 0, 3, blob(same), "51037A4A37730F52C8732586D3AAA316" )', + '( 6, 0, 5, blob(1-keep-1-rm), "535A5575B48444EDEB926815AB26EC9B" )', + '( 6, 1, 4, blob(edit-v1), "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 6, 2, 6, blob(1-keep-1-rm), "535A5575B48444EDEB926815AB26EC9B" )', +] + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'index' +tbl_name = 'tiles_with_hash' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash''' diff --git a/mbtiles/tests/snapshots/copy__convert@v1__tiles__norm.snap b/mbtiles/tests/snapshots/copy__convert@v1__tiles__norm.snap new file mode 100644 index 000000000..02a44e398 --- /dev/null +++ b/mbtiles/tests/snapshots/copy__convert@v1__tiles__norm.snap @@ -0,0 +1,92 @@ +--- +source: mbtiles/tests/copy.rs +expression: actual_value +--- +[[]] +type = 'table' +tbl_name = 'images' +sql = ''' +CREATE TABLE images ( + tile_id text NOT NULL PRIMARY KEY, + tile_data blob)''' +values = [ + '( "0F6969D7052DA9261E31DDB6E88C136E", blob(remove) )', + '( "51037A4A37730F52C8732586D3AAA316", blob(same) )', + '( "535A5575B48444EDEB926815AB26EC9B", blob(1-keep-1-rm) )', + '( "63A9F0EA7BB98050796B649E85481845", blob(root) )', + '( "720C02778717818CC0A869955BA2AFB6", blob(non-empty) )', + '( "D41D8CD98F00B204E9800998ECF8427E", blob() )', + '( "EFE0AE5FD114DE99855BC2838BE97E1D", blob(edit-v1) )', +] + +[[]] +type = 'table' +tbl_name = 'map' +sql = ''' +CREATE TABLE map ( + zoom_level integer NOT NULL, + tile_column integer NOT NULL, + tile_row integer NOT NULL, + tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row))''' +values = [ + '( 3, 6, 7, "63A9F0EA7BB98050796B649E85481845" )', + '( 5, 0, 0, "51037A4A37730F52C8732586D3AAA316" )', + '( 5, 0, 1, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 1, 1, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 5, 1, 2, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 5, 1, 3, "720C02778717818CC0A869955BA2AFB6" )', + '( 5, 2, 2, "0F6969D7052DA9261E31DDB6E88C136E" )', + '( 5, 2, 3, "D41D8CD98F00B204E9800998ECF8427E" )', + '( 6, 0, 3, "51037A4A37730F52C8732586D3AAA316" )', + '( 6, 0, 5, "535A5575B48444EDEB926815AB26EC9B" )', + '( 6, 1, 4, "EFE0AE5FD114DE99855BC2838BE97E1D" )', + '( 6, 2, 6, "535A5575B48444EDEB926815AB26EC9B" )', +] + +[[]] +type = 'table' +tbl_name = 'metadata' +sql = ''' +CREATE TABLE metadata ( + name text NOT NULL PRIMARY KEY, + value text)''' +values = ['( "agg_tiles_hash", "9ED9178D7025276336C783C2B54D6258" )'] + +[[]] +type = 'index' +tbl_name = 'images' + +[[]] +type = 'index' +tbl_name = 'map' + +[[]] +type = 'index' +tbl_name = 'metadata' + +[[]] +type = 'view' +tbl_name = 'tiles' +sql = ''' +CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id''' + +[[]] +type = 'view' +tbl_name = 'tiles_with_hash' +sql = ''' +CREATE VIEW tiles_with_hash AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data, + images.tile_id AS tile_hash + FROM map + JOIN images ON images.tile_id = map.tile_id'''