Add ability to copy to existing mbtiles files (maplibre#778)

* Allow the `dst` in `mbtiles copy <src> <dst>` command to already
contain data, merging tilesets
* Add `mbtiles copy --on-duplicate (override|ignore|abort)`

---------

Co-authored-by: rstanciu <[email protected]>
Co-authored-by: Yuri Astrakhan <[email protected]>

justfile

@@ -207,7 +207,7 @@ git-pre-push: stop start
 # Update sqlite database schema.
 prepare-sqlite: install-sqlx
     mkdir -p martin-mbtiles/.sqlx
-    cd martin-mbtiles && cargo sqlx prepare --database-url sqlite://$PWD/../tests/fixtures/files/world_cities.mbtiles
+    cd martin-mbtiles && cargo sqlx prepare --database-url sqlite://$PWD/../tests/fixtures/files/world_cities.mbtiles -- --lib --tests
 
 # Install SQLX cli if not already installed.
 [private]

martin-mbtiles/src/bin/main.rs

@@ -93,7 +93,7 @@ mod tests {
 
     use clap::error::ErrorKind;
     use clap::Parser;
-    use martin_mbtiles::TileCopierOptions;
+    use martin_mbtiles::{CopyDuplicateMode, TileCopierOptions};
 
     use crate::Args;
     use crate::Commands::{ApplyDiff, Copy, MetaGetValue};
@@ -245,6 +245,69 @@ mod tests {
         );
     }
 
+    #[test]
+    fn test_copy_diff_with_override_copy_duplicate_mode() {
+        assert_eq!(
+            Args::parse_from([
+                "mbtiles",
+                "copy",
+                "src_file",
+                "dst_file",
+                "--on-duplicate",
+                "override"
+            ]),
+            Args {
+                verbose: false,
+                command: Copy(
+                    TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
+                        .on_duplicate(CopyDuplicateMode::Override)
+                )
+            }
+        );
+    }
+
+    #[test]
+    fn test_copy_diff_with_ignore_copy_duplicate_mode() {
+        assert_eq!(
+            Args::parse_from([
+                "mbtiles",
+                "copy",
+                "src_file",
+                "dst_file",
+                "--on-duplicate",
+                "ignore"
+            ]),
+            Args {
+                verbose: false,
+                command: Copy(
+                    TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
+                        .on_duplicate(CopyDuplicateMode::Ignore)
+                )
+            }
+        );
+    }
+
+    #[test]
+    fn test_copy_diff_with_abort_copy_duplicate_mode() {
+        assert_eq!(
+            Args::parse_from([
+                "mbtiles",
+                "copy",
+                "src_file",
+                "dst_file",
+                "--on-duplicate",
+                "abort"
+            ]),
+            Args {
+                verbose: false,
+                command: Copy(
+                    TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
+                        .on_duplicate(CopyDuplicateMode::Abort)
+                )
+            }
+        );
+    }
+
     #[test]
     fn test_meta_get_no_arguments() {
         assert_eq!(

martin-mbtiles/src/errors.rs

@@ -28,6 +28,15 @@ pub enum MbtError {
 
     #[error("The destination file {0} is non-empty")]
     NonEmptyTargetFile(PathBuf),
+
+    #[error("The file {0} does not have the required uniqueness constraint")]
+    NoUniquenessConstraint(String),
+
+    #[error("Could not copy MBTiles file: {reason}")]
+    UnsupportedCopyOperation { reason: String },
+
+    #[error("Unexpected duplicate tiles found when copying")]
+    DuplicateValues,
 }
 
 pub type MbtResult<T> = Result<T, MbtError>;

martin-mbtiles/src/lib.rs

@@ -9,4 +9,6 @@ mod tile_copier;
 pub use errors::MbtError;
 pub use mbtiles::{Mbtiles, Metadata};
 pub use mbtiles_pool::MbtilesPool;
-pub use tile_copier::{apply_mbtiles_diff, copy_mbtiles_file, TileCopierOptions};
+pub use tile_copier::{
+    apply_mbtiles_diff, copy_mbtiles_file, CopyDuplicateMode, TileCopierOptions,
+};

martin-mbtiles/src/mbtiles.rs

@@ -2,6 +2,7 @@
 
 extern crate core;
 
+use std::collections::HashSet;
 use std::ffi::OsStr;
 use std::fmt::Display;
 use std::path::Path;
@@ -11,7 +12,7 @@ use futures::TryStreamExt;
 use log::{debug, info, warn};
 use martin_tile_utils::{Format, TileInfo};
 use serde_json::{Value as JSONValue, Value};
-use sqlx::{query, SqliteExecutor};
+use sqlx::{query, Row, SqliteExecutor};
 use tilejson::{tilejson, Bounds, Center, TileJSON};
 
 use crate::errors::{MbtError, MbtResult};
@@ -26,7 +27,7 @@ pub struct Metadata {
     pub json: Option<JSONValue>,
 }
 
-#[derive(Debug, PartialEq)]
+#[derive(Debug, Clone, PartialEq)]
 pub enum MbtType {
     TileTables,
     DeDuplicated,
@@ -280,13 +281,64 @@ impl Mbtiles {
     where
         for<'e> &'e mut T: SqliteExecutor<'e>,
     {
-        if is_deduplicated_type(&mut *conn).await? {
-            Ok(MbtType::DeDuplicated)
+        let mbt_type = if is_deduplicated_type(&mut *conn).await? {
+            MbtType::DeDuplicated
         } else if is_tile_tables_type(&mut *conn).await? {
-            Ok(MbtType::TileTables)
+            MbtType::TileTables
         } else {
-            Err(MbtError::InvalidDataFormat(self.filepath.clone()))
+            return Err(MbtError::InvalidDataFormat(self.filepath.clone()));
+        };
+
+        self.check_for_uniqueness_constraint(&mut *conn, &mbt_type)
+            .await?;
+
+        Ok(mbt_type)
+    }
+
+    async fn check_for_uniqueness_constraint<T>(
+        &self,
+        conn: &mut T,
+        mbt_type: &MbtType,
+    ) -> MbtResult<()>
+    where
+        for<'e> &'e mut T: SqliteExecutor<'e>,
+    {
+        let table_name = match mbt_type {
+            MbtType::TileTables => "tiles",
+            MbtType::DeDuplicated => "map",
+        };
+
+        let indexes = query("SELECT name FROM pragma_index_list(?) WHERE [unique] = 1")
+            .bind(table_name)
+            .fetch_all(&mut *conn)
+            .await?;
+
+        // Ensure there is some index on tiles that has a unique constraint on (zoom_level, tile_row, tile_column)
+        for index in indexes {
+            let mut unique_idx_cols = HashSet::new();
+            let rows = query("SELECT DISTINCT name FROM pragma_index_info(?)")
+                .bind(index.get::<String, _>("name"))
+                .fetch_all(&mut *conn)
+                .await?;
+
+            for row in rows {
+                unique_idx_cols.insert(row.get("name"));
+            }
+
+            if unique_idx_cols
+                .symmetric_difference(&HashSet::from([
+                    "zoom_level".to_string(),
+                    "tile_column".to_string(),
+                    "tile_row".to_string(),
+                ]))
+                .collect::<Vec<_>>()
+                .is_empty()
+            {
+                return Ok(());
+            }
         }
+
+        Err(MbtError::NoUniquenessConstraint(self.filepath.clone()))
     }
 }