diff --git a/Cargo.lock b/Cargo.lock index 64e434f14..757330e26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1077,6 +1077,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.0.0" @@ -1818,6 +1824,7 @@ dependencies = [ "log", "martin-tile-utils", "serde_json", + "sqlite-hashes", "sqlx", "thiserror", "tilejson", @@ -2628,6 +2635,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rusqlite" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" +dependencies = [ + "bitflags 2.3.3", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -3030,6 +3051,19 @@ dependencies = [ "unicode_categories", ] +[[package]] +name = "sqlite-hashes" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f600356eab3d8c80c0f7abd6e931d8a2b333fe917a5ead2dfb87a858cdfad9a" +dependencies = [ + "digest", + "md-5", + "rusqlite", + "sha1", + "sha2", +] + [[package]] name = "sqlx" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index 4423bbfe5..a8aaef752 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" spreet = { version = "0.8", default-features = false } +sqlite-hashes = "0.1" sqlx = { version = "0.7", features = ["sqlite"] } subst = { version = "0.2", features = ["yaml"] } thiserror = "1" diff --git a/docs/src/tools.md b/docs/src/tools.md index 5416cce1d..0978316b8 100644 --- a/docs/src/tools.md +++ b/docs/src/tools.md @@ -15,19 +15,24 @@ mbtiles meta-get my_file.mbtiles description ``` ### copy -Copy an mbtiles file, optionally filtering its content by zoom levels. Can also flatten mbtiles file from de-duplicated tiles to a simple table structure. +Copy an mbtiles file, optionally filtering its content by zoom levels. ```shell mbtiles copy src_file.mbtiles dst_file.mbtiles \ - --min-zoom 0 --max-zoom 10 --force-simple + --min-zoom 0 --max-zoom 10 ``` Copy command can also be used to compare two mbtiles files and generate a diff. ```shell mbtiles copy src_file.mbtiles diff_file.mbtiles \ - --force-simple --diff-with-file modified_file.mbtiles + --diff-with-file modified_file.mbtiles ``` +This command can also be used to generate files of different [supported schema](##supported-schema). +```shell +mbtiles copy normalized.mbtiles dst.mbtiles \ + --dst-mbttype flat-with-hash +``` ### apply-diff Apply the diff file generated from `copy` command above to an mbtiles file. The diff file can be applied to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset. ```shell @@ -44,4 +49,42 @@ sqlite3 src_file.mbtiles \ "INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) SELECT * FROM diffDb.tiles WHERE tile_data NOTNULL;" ``` -**_NOTE:_** Both of these methods for applying a diff _only_ work for mbtiles files in the simple tables format; they do _not_ work for mbtiles files in deduplicated format. +**_NOTE:_** Both of these methods for applying a diff _only_ work for mbtiles files in the simple tables format; they do _not_ work for mbtiles files in normalized_tables format. + +### validate +If the `.mbtiles` file is of `flat_with_hash` or `normalized` type, then verify that the data stored in columns `tile_hash` and `tile_id` respectively are MD5 hashes of the `tile_data` column. +```shell +mbtiles validate src_file.mbtiles +``` + +## Supported Schema +The `mbtiles` tool supports three different kinds of schema for `tiles` data in `.mbtiles` files: + +- `flat`: + ``` + CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob); + CREATE UNIQUE INDEX tile_index on tiles (zoom_level, tile_column, tile_row); + ``` +- `flat-with-hash`: + ``` + CREATE TABLE tiles_with_hash (zoom_level integer NOT NULL, tile_column integer NOT NULL, tile_row integer NOT NULL, tile_data blob, tile_hash text); + CREATE UNIQUE INDEX tiles_with_hash_index on tiles_with_hash (zoom_level, tile_column, tile_row); + CREATE VIEW tiles AS SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash; + ``` +- `normalized`: + ``` + CREATE TABLE map (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_id TEXT); + CREATE UNIQUE INDEX map_index ON map (zoom_level, tile_column, tile_row); + CREATE TABLE images (tile_data blob, tile_id text); + CREATE UNIQUE INDEX images_id ON images (tile_id); + CREATE VIEW tiles AS + SELECT + map.zoom_level AS zoom_level, + map.tile_column AS tile_column, + map.tile_row AS tile_row, + images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id; + ``` + +For more general spec information, see [here](https://github.com/mapbox/mbtiles-spec#readme). \ No newline at end of file diff --git a/martin-mbtiles/.sqlx/query-09e15d4479a96829f8dcd93e6f40f7e5f487f6c33614aa82ae3716e3bb932dfa.json b/martin-mbtiles/.sqlx/query-09e15d4479a96829f8dcd93e6f40f7e5f487f6c33614aa82ae3716e3bb932dfa.json deleted file mode 100644 index c4e454244..000000000 --- a/martin-mbtiles/.sqlx/query-09e15d4479a96829f8dcd93e6f40f7e5f487f6c33614aa82ae3716e3bb932dfa.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "SELECT (\n -- Has a \"map\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'map'\n AND type = 'table'\n --\n ) AND (\n -- \"map\" table's columns and their types are as expected:\n -- 4 non-null columns (zoom_level, tile_column, tile_row, tile_id).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('map')\n WHERE \"notnull\" = 0\n AND ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_id\" AND type = \"TEXT\"))\n --\n ) AND (\n -- Has a \"images\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'images'\n AND type = 'table'\n --\n ) AND (\n -- \"images\" table's columns and their types are as expected:\n -- 2 non-null columns (tile_id, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 2\n FROM pragma_table_info('images')\n WHERE \"notnull\" = 0\n AND ((name = \"tile_id\" AND type = \"TEXT\")\n OR (name = \"tile_data\" AND type = \"BLOB\"))\n --\n ) AS is_valid;\n", - "describe": { - "columns": [ - { - "name": "is_valid", - "ordinal": 0, - "type_info": "Int" - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - null - ] - }, - "hash": "09e15d4479a96829f8dcd93e6f40f7e5f487f6c33614aa82ae3716e3bb932dfa" -} diff --git a/martin-mbtiles/.sqlx/query-14f262aafedb8739ee403fe6fc67989d706ce91630c9332a600e8022c0d4b628.json b/martin-mbtiles/.sqlx/query-14f262aafedb8739ee403fe6fc67989d706ce91630c9332a600e8022c0d4b628.json new file mode 100644 index 000000000..0848bf78d --- /dev/null +++ b/martin-mbtiles/.sqlx/query-14f262aafedb8739ee403fe6fc67989d706ce91630c9332a600e8022c0d4b628.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "SELECT (\n -- Has a \"map\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'map'\n AND type = 'table'\n --\n ) AND (\n -- \"map\" table's columns and their types are as expected:\n -- 4 columns (zoom_level, tile_column, tile_row, tile_id).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('map')\n WHERE ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_id\" AND type = \"TEXT\"))\n --\n ) AND (\n -- Has a \"images\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'images'\n AND type = 'table'\n --\n ) AND (\n -- \"images\" table's columns and their types are as expected:\n -- 2 columns (tile_id, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 2\n FROM pragma_table_info('images')\n WHERE ((name = \"tile_id\" AND type = \"TEXT\")\n OR (name = \"tile_data\" AND type = \"BLOB\"))\n --\n ) AS is_valid;\n", + "describe": { + "columns": [ + { + "name": "is_valid", + "ordinal": 0, + "type_info": "Int" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + null + ] + }, + "hash": "14f262aafedb8739ee403fe6fc67989d706ce91630c9332a600e8022c0d4b628" +} diff --git a/martin-mbtiles/.sqlx/query-177aed5e4ee0e7a23eb708174a829e7f1af10037bdfb6543b029cc80c3ee60dd.json b/martin-mbtiles/.sqlx/query-177aed5e4ee0e7a23eb708174a829e7f1af10037bdfb6543b029cc80c3ee60dd.json new file mode 100644 index 000000000..6e141d9d8 --- /dev/null +++ b/martin-mbtiles/.sqlx/query-177aed5e4ee0e7a23eb708174a829e7f1af10037bdfb6543b029cc80c3ee60dd.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "SELECT (\n -- Has a \"tiles\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles'\n AND type = 'table'\n --\n ) AND (\n -- \"tiles\" table's columns and their types are as expected:\n -- 4 columns (zoom_level, tile_column, tile_row, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('tiles')\n WHERE ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_data\" AND type = \"BLOB\"))\n --\n ) as is_valid;\n", + "describe": { + "columns": [ + { + "name": "is_valid", + "ordinal": 0, + "type_info": "Int" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + null + ] + }, + "hash": "177aed5e4ee0e7a23eb708174a829e7f1af10037bdfb6543b029cc80c3ee60dd" +} diff --git a/martin-mbtiles/.sqlx/query-3a1e6e16157856190e061e1ade9b59995c337cfe7e4c54d4bbb2669a27682401.json b/martin-mbtiles/.sqlx/query-3a1e6e16157856190e061e1ade9b59995c337cfe7e4c54d4bbb2669a27682401.json new file mode 100644 index 000000000..6230f16d4 --- /dev/null +++ b/martin-mbtiles/.sqlx/query-3a1e6e16157856190e061e1ade9b59995c337cfe7e4c54d4bbb2669a27682401.json @@ -0,0 +1,20 @@ +{ + "db_name": "SQLite", + "query": "SELECT (\n -- Has a \"tiles_with_hash\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles_with_hash'\n AND type = 'table'\n --\n ) AND (\n -- \"tiles_with_hash\" table's columns and their types are as expected:\n -- 5 columns (zoom_level, tile_column, tile_row, tile_data, tile_hash).\n -- The order is not important\n SELECT COUNT(*) = 5\n FROM pragma_table_info('tiles_with_hash')\n WHERE ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_data\" AND type = \"BLOB\")\n OR (name = \"tile_hash\" AND type = \"TEXT\"))\n --\n ) as is_valid;\n", + "describe": { + "columns": [ + { + "name": "is_valid", + "ordinal": 0, + "type_info": "Int" + } + ], + "parameters": { + "Right": 0 + }, + "nullable": [ + null + ] + }, + "hash": "3a1e6e16157856190e061e1ade9b59995c337cfe7e4c54d4bbb2669a27682401" +} diff --git a/martin-mbtiles/.sqlx/query-78d1356063c080d9bcea05a5ad95ffb771de5adb62873d794be09062506451d3.json b/martin-mbtiles/.sqlx/query-78d1356063c080d9bcea05a5ad95ffb771de5adb62873d794be09062506451d3.json deleted file mode 100644 index 454e95343..000000000 --- a/martin-mbtiles/.sqlx/query-78d1356063c080d9bcea05a5ad95ffb771de5adb62873d794be09062506451d3.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "SELECT (\n -- Has a \"tiles\" table\n SELECT COUNT(*) = 1\n FROM sqlite_master\n WHERE name = 'tiles'\n AND type = 'table'\n --\n ) AND (\n -- \"tiles\" table's columns and their types are as expected:\n -- 4 non-null columns (zoom_level, tile_column, tile_row, tile_data).\n -- The order is not important\n SELECT COUNT(*) = 4\n FROM pragma_table_info('tiles')\n WHERE \"notnull\" = 0\n AND ((name = \"zoom_level\" AND type = \"INTEGER\")\n OR (name = \"tile_column\" AND type = \"INTEGER\")\n OR (name = \"tile_row\" AND type = \"INTEGER\")\n OR (name = \"tile_data\" AND type = \"BLOB\"))\n --\n ) as is_valid;\n", - "describe": { - "columns": [ - { - "name": "is_valid", - "ordinal": 0, - "type_info": "Int" - } - ], - "parameters": { - "Right": 0 - }, - "nullable": [ - null - ] - }, - "hash": "78d1356063c080d9bcea05a5ad95ffb771de5adb62873d794be09062506451d3" -} diff --git a/martin-mbtiles/.sqlx/query-a115609880b2c6ed3beeb5aaf8c7e779ecf324e1862945fbd18da4bf5baf565b.json b/martin-mbtiles/.sqlx/query-a115609880b2c6ed3beeb5aaf8c7e779ecf324e1862945fbd18da4bf5baf565b.json deleted file mode 100644 index fc0b3c0c2..000000000 --- a/martin-mbtiles/.sqlx/query-a115609880b2c6ed3beeb5aaf8c7e779ecf324e1862945fbd18da4bf5baf565b.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "ATTACH DATABASE ? AS newDb", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "a115609880b2c6ed3beeb5aaf8c7e779ecf324e1862945fbd18da4bf5baf565b" -} diff --git a/martin-mbtiles/.sqlx/query-e13e2e17d5bf56287bc0fd7c55a1f52ce710d8978e3b35b59b724fc5bee9f55c.json b/martin-mbtiles/.sqlx/query-e13e2e17d5bf56287bc0fd7c55a1f52ce710d8978e3b35b59b724fc5bee9f55c.json deleted file mode 100644 index 5d8f76197..000000000 --- a/martin-mbtiles/.sqlx/query-e13e2e17d5bf56287bc0fd7c55a1f52ce710d8978e3b35b59b724fc5bee9f55c.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "ATTACH DATABASE ? AS diffDb", - "describe": { - "columns": [], - "parameters": { - "Right": 1 - }, - "nullable": [] - }, - "hash": "e13e2e17d5bf56287bc0fd7c55a1f52ce710d8978e3b35b59b724fc5bee9f55c" -} diff --git a/martin-mbtiles/Cargo.toml b/martin-mbtiles/Cargo.toml index 8ab504ab5..5be2b10af 100644 --- a/martin-mbtiles/Cargo.toml +++ b/martin-mbtiles/Cargo.toml @@ -29,6 +29,7 @@ tilejson.workspace = true # Bin dependencies anyhow = { workspace = true, optional = true } clap = { workspace = true, optional = true } +sqlite-hashes.workspace = true tokio = { workspace = true, features = ["rt-multi-thread"], optional = true } [dev-dependencies] diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs index 1da5241a3..003262b7d 100644 --- a/martin-mbtiles/src/bin/main.rs +++ b/martin-mbtiles/src/bin/main.rs @@ -2,7 +2,9 @@ use std::path::{Path, PathBuf}; use anyhow::Result; use clap::{Parser, Subcommand}; -use martin_mbtiles::{apply_mbtiles_diff, copy_mbtiles_file, Mbtiles, TileCopierOptions}; +use martin_mbtiles::{ + apply_mbtiles_diff, copy_mbtiles_file, validate_mbtiles, Mbtiles, TileCopierOptions, +}; use sqlx::sqlite::SqliteConnectOptions; use sqlx::{Connection, SqliteConnection}; @@ -53,6 +55,12 @@ enum Commands { /// Diff file diff_file: PathBuf, }, + /// Validate tile data if hash of tile data exists in file + #[command(name = "validate")] + Validate { + /// MBTiles file to validate + file: PathBuf, + }, } #[tokio::main] @@ -72,6 +80,9 @@ async fn main() -> Result<()> { } => { apply_mbtiles_diff(src_file, diff_file).await?; } + Commands::Validate { file } => { + validate_mbtiles(file).await?; + } } Ok(()) @@ -96,7 +107,7 @@ mod tests { use martin_mbtiles::{CopyDuplicateMode, TileCopierOptions}; use crate::Args; - use crate::Commands::{ApplyDiff, Copy, MetaGetValue}; + use crate::Commands::{ApplyDiff, Copy, MetaGetValue, Validate}; #[test] fn test_copy_no_arguments() { @@ -205,23 +216,6 @@ mod tests { ); } - #[test] - fn test_copy_diff_with_file_no_force_simple_arguments() { - assert_eq!( - Args::try_parse_from([ - "mbtiles", - "copy", - "src_file", - "dst_file", - "--diff-with-file", - "no_file", - ]) - .unwrap_err() - .kind(), - ErrorKind::MissingRequiredArgument - ); - } - #[test] fn test_copy_diff_with_file_arguments() { assert_eq!( @@ -232,14 +226,12 @@ mod tests { "dst_file", "--diff-with-file", "no_file", - "--force-simple" ]), Args { verbose: false, command: Copy( TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file")) .diff_with_file(PathBuf::from("no_file")) - .force_simple(true) ) } ); @@ -345,4 +337,17 @@ mod tests { } ); } + + #[test] + fn test_validate() { + assert_eq!( + Args::parse_from(["mbtiles", "validate", "src_file"]), + Args { + verbose: false, + command: Validate { + file: PathBuf::from("src_file"), + } + } + ); + } } diff --git a/martin-mbtiles/src/errors.rs b/martin-mbtiles/src/errors.rs index c6245a265..c33df62a3 100644 --- a/martin-mbtiles/src/errors.rs +++ b/martin-mbtiles/src/errors.rs @@ -1,3 +1,4 @@ +use sqlite_hashes::rusqlite; use std::path::PathBuf; use martin_tile_utils::TileInfo; @@ -7,7 +8,10 @@ use crate::mbtiles::MbtType; #[derive(thiserror::Error, Debug)] pub enum MbtError { #[error("SQL Error {0}")] - SqlError(#[from] sqlx::Error), + SqlxError(#[from] sqlx::Error), + + #[error("SQL Error {0}")] + RusqliteError(#[from] rusqlite::Error), #[error("MBTile filepath contains unsupported characters: {}", .0.display())] UnsupportedCharsInFilepath(PathBuf), @@ -18,8 +22,11 @@ pub enum MbtError { #[error("Invalid data format for MBTile file {0}")] InvalidDataFormat(String), + #[error("Invalid tile data for MBTile file {0}")] + InvalidTileData(String), + #[error("Incorrect data format for MBTile file {0}; expected {1:?} and got {2:?}")] - IncorrectDataFormat(String, MbtType, MbtType), + IncorrectDataFormat(String, &'static [MbtType], MbtType), #[error(r#"Filename "{0}" passed to SQLite must be valid UTF-8"#)] InvalidFilenameType(PathBuf), diff --git a/martin-mbtiles/src/lib.rs b/martin-mbtiles/src/lib.rs index 59eaf2e3d..4540cb937 100644 --- a/martin-mbtiles/src/lib.rs +++ b/martin-mbtiles/src/lib.rs @@ -10,5 +10,5 @@ pub use errors::MbtError; pub use mbtiles::{Mbtiles, Metadata}; pub use mbtiles_pool::MbtilesPool; pub use tile_copier::{ - apply_mbtiles_diff, copy_mbtiles_file, CopyDuplicateMode, TileCopierOptions, + apply_mbtiles_diff, copy_mbtiles_file, validate_mbtiles, CopyDuplicateMode, TileCopierOptions, }; diff --git a/martin-mbtiles/src/mbtiles.rs b/martin-mbtiles/src/mbtiles.rs index b21426929..5690a6269 100644 --- a/martin-mbtiles/src/mbtiles.rs +++ b/martin-mbtiles/src/mbtiles.rs @@ -8,6 +8,8 @@ use std::fmt::Display; use std::path::Path; use std::str::FromStr; +#[cfg(feature = "cli")] +use clap::ValueEnum; use futures::TryStreamExt; use log::{debug, info, warn}; use martin_tile_utils::{Format, TileInfo}; @@ -16,7 +18,9 @@ use sqlx::{query, Row, SqliteExecutor}; use tilejson::{tilejson, Bounds, Center, TileJSON}; use crate::errors::{MbtError, MbtResult}; -use crate::mbtiles_queries::{is_deduplicated_type, is_tile_tables_type}; +use crate::mbtiles_queries::{ + is_flat_tables_type, is_flat_with_hash_tables_type, is_normalized_tables_type, +}; #[derive(Clone, Debug, PartialEq)] pub struct Metadata { @@ -27,10 +31,12 @@ pub struct Metadata { pub json: Option, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr(feature = "cli", derive(ValueEnum))] pub enum MbtType { - TileTables, - DeDuplicated, + Flat, + FlatWithHash, + Normalized, } #[derive(Clone, Debug)] @@ -281,10 +287,12 @@ impl Mbtiles { where for<'e> &'e mut T: SqliteExecutor<'e>, { - let mbt_type = if is_deduplicated_type(&mut *conn).await? { - MbtType::DeDuplicated - } else if is_tile_tables_type(&mut *conn).await? { - MbtType::TileTables + let mbt_type = if is_normalized_tables_type(&mut *conn).await? { + MbtType::Normalized + } else if is_flat_with_hash_tables_type(&mut *conn).await? { + MbtType::FlatWithHash + } else if is_flat_tables_type(&mut *conn).await? { + MbtType::Flat } else { return Err(MbtError::InvalidDataFormat(self.filepath.clone())); }; @@ -304,8 +312,9 @@ impl Mbtiles { for<'e> &'e mut T: SqliteExecutor<'e>, { let table_name = match mbt_type { - MbtType::TileTables => "tiles", - MbtType::DeDuplicated => "map", + MbtType::Flat => "tiles", + MbtType::FlatWithHash => "tiles_with_hash", + MbtType::Normalized => "map", }; let indexes = query("SELECT name FROM pragma_index_list(?) WHERE [unique] = 1") @@ -436,11 +445,15 @@ mod tests { async fn detect_type() { let (mut conn, mbt) = open("../tests/fixtures/files/world_cities.mbtiles").await; let res = mbt.detect_type(&mut conn).await.unwrap(); - assert_eq!(res, MbtType::TileTables); + assert_eq!(res, MbtType::Flat); + + let (mut conn, mbt) = open("../tests/fixtures/files/zoomed_world_cities.mbtiles").await; + let res = mbt.detect_type(&mut conn).await.unwrap(); + assert_eq!(res, MbtType::FlatWithHash); let (mut conn, mbt) = open("../tests/fixtures/files/geography-class-jpg.mbtiles").await; let res = mbt.detect_type(&mut conn).await.unwrap(); - assert_eq!(res, MbtType::DeDuplicated); + assert_eq!(res, MbtType::Normalized); let (mut conn, mbt) = open(":memory:").await; let res = mbt.detect_type(&mut conn).await; diff --git a/martin-mbtiles/src/mbtiles_queries.rs b/martin-mbtiles/src/mbtiles_queries.rs index e2fa62dec..0d04815da 100644 --- a/martin-mbtiles/src/mbtiles_queries.rs +++ b/martin-mbtiles/src/mbtiles_queries.rs @@ -2,7 +2,7 @@ use sqlx::{query, SqliteExecutor}; use crate::errors::MbtResult; -pub async fn is_deduplicated_type(conn: &mut T) -> MbtResult +pub async fn is_normalized_tables_type(conn: &mut T) -> MbtResult where for<'e> &'e mut T: SqliteExecutor<'e>, { @@ -16,12 +16,11 @@ where -- ) AND ( -- "map" table's columns and their types are as expected: - -- 4 non-null columns (zoom_level, tile_column, tile_row, tile_id). + -- 4 columns (zoom_level, tile_column, tile_row, tile_id). -- The order is not important SELECT COUNT(*) = 4 FROM pragma_table_info('map') - WHERE "notnull" = 0 - AND ((name = "zoom_level" AND type = "INTEGER") + WHERE ((name = "zoom_level" AND type = "INTEGER") OR (name = "tile_column" AND type = "INTEGER") OR (name = "tile_row" AND type = "INTEGER") OR (name = "tile_id" AND type = "TEXT")) @@ -35,12 +34,11 @@ where -- ) AND ( -- "images" table's columns and their types are as expected: - -- 2 non-null columns (tile_id, tile_data). + -- 2 columns (tile_id, tile_data). -- The order is not important SELECT COUNT(*) = 2 FROM pragma_table_info('images') - WHERE "notnull" = 0 - AND ((name = "tile_id" AND type = "TEXT") + WHERE ((name = "tile_id" AND type = "TEXT") OR (name = "tile_data" AND type = "BLOB")) -- ) AS is_valid; @@ -55,7 +53,43 @@ where == 1) } -pub async fn is_tile_tables_type(conn: &mut T) -> MbtResult +pub async fn is_flat_with_hash_tables_type(conn: &mut T) -> MbtResult +where + for<'e> &'e mut T: SqliteExecutor<'e>, +{ + let sql = query!( + r#"SELECT ( + -- Has a "tiles_with_hash" table + SELECT COUNT(*) = 1 + FROM sqlite_master + WHERE name = 'tiles_with_hash' + AND type = 'table' + -- + ) AND ( + -- "tiles_with_hash" table's columns and their types are as expected: + -- 5 columns (zoom_level, tile_column, tile_row, tile_data, tile_hash). + -- The order is not important + SELECT COUNT(*) = 5 + FROM pragma_table_info('tiles_with_hash') + WHERE ((name = "zoom_level" AND type = "INTEGER") + OR (name = "tile_column" AND type = "INTEGER") + OR (name = "tile_row" AND type = "INTEGER") + OR (name = "tile_data" AND type = "BLOB") + OR (name = "tile_hash" AND type = "TEXT")) + -- + ) as is_valid; +"# + ); + + Ok(sql + .fetch_one(&mut *conn) + .await? + .is_valid + .unwrap_or_default() + == 1) +} + +pub async fn is_flat_tables_type(conn: &mut T) -> MbtResult where for<'e> &'e mut T: SqliteExecutor<'e>, { @@ -69,12 +103,11 @@ where -- ) AND ( -- "tiles" table's columns and their types are as expected: - -- 4 non-null columns (zoom_level, tile_column, tile_row, tile_data). + -- 4 columns (zoom_level, tile_column, tile_row, tile_data). -- The order is not important SELECT COUNT(*) = 4 FROM pragma_table_info('tiles') - WHERE "notnull" = 0 - AND ((name = "zoom_level" AND type = "INTEGER") + WHERE ((name = "zoom_level" AND type = "INTEGER") OR (name = "tile_column" AND type = "INTEGER") OR (name = "tile_row" AND type = "INTEGER") OR (name = "tile_data" AND type = "BLOB")) diff --git a/martin-mbtiles/src/tile_copier.rs b/martin-mbtiles/src/tile_copier.rs index 8fcf6e986..f523f8851 100644 --- a/martin-mbtiles/src/tile_copier.rs +++ b/martin-mbtiles/src/tile_copier.rs @@ -1,16 +1,19 @@ extern crate core; use std::collections::HashSet; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; #[cfg(feature = "cli")] use clap::{builder::ValueParser, error::ErrorKind, Args, ValueEnum}; -use sqlx::sqlite::{SqliteArguments, SqliteConnectOptions}; -use sqlx::{query, query_with, Arguments, Connection, Row, SqliteConnection}; +use sqlite_hashes::rusqlite::params_from_iter; +use sqlite_hashes::{register_md5_function, rusqlite::Connection as RusqliteConnection}; +use sqlx::sqlite::SqliteConnectOptions; +use sqlx::{query, Connection, Row, SqliteConnection}; use crate::errors::MbtResult; use crate::mbtiles::MbtType; -use crate::mbtiles::MbtType::{DeDuplicated, TileTables}; +use crate::mbtiles::MbtType::{Flat, FlatWithHash, Normalized}; +use crate::MbtError::{IncorrectDataFormat, InvalidTileData}; use crate::{MbtError, Mbtiles}; #[derive(PartialEq, Eq, Default, Debug, Clone)] @@ -29,10 +32,9 @@ pub struct TileCopierOptions { src_file: PathBuf, /// MBTiles file to write to dst_file: PathBuf, - /// Force the output file to be in a simple MBTiles format with a `tiles` table - /// - #[cfg_attr(feature = "cli", arg(long))] - force_simple: bool, + /// TODO: add documentation Output format of the destination file, ignored if the file exists. if not specified, defaults to the type of source + #[cfg_attr(feature = "cli", arg(long, value_enum))] + dst_mbttype: Option, /// Specify copying behaviour when tiles with duplicate (zoom_level, tile_column, tile_row) values are found #[cfg_attr(feature = "cli", arg(long, value_enum, default_value_t = CopyDuplicateMode::Override))] on_duplicate: CopyDuplicateMode, @@ -46,7 +48,7 @@ pub struct TileCopierOptions { #[cfg_attr(feature = "cli", arg(long, value_parser(ValueParser::new(HashSetValueParser{})), default_value=""))] zoom_levels: HashSet, /// Compare source file with this file, and only copy non-identical tiles to destination - #[cfg_attr(feature = "cli", arg(long, requires("force_simple")))] + #[cfg_attr(feature = "cli", arg(long))] diff_with_file: Option, } @@ -95,7 +97,7 @@ impl TileCopierOptions { src_file: src_filepath, dst_file: dst_filepath, zoom_levels: HashSet::new(), - force_simple: false, + dst_mbttype: None, on_duplicate: CopyDuplicateMode::Override, min_zoom: None, max_zoom: None, @@ -103,8 +105,8 @@ impl TileCopierOptions { } } - pub fn force_simple(mut self, force_simple: bool) -> Self { - self.force_simple = force_simple; + pub fn dst_mbttype(mut self, dst_mbttype: Option) -> Self { + self.dst_mbttype = dst_mbttype; self } @@ -144,207 +146,268 @@ impl TileCopier { } pub async fn run(self) -> MbtResult { - let mut mbtiles_type = open_and_detect_type(&self.src_mbtiles).await?; - let force_simple = self.options.force_simple && mbtiles_type != TileTables; + let src_mbttype = open_and_detect_type(&self.src_mbtiles).await?; - let opt = SqliteConnectOptions::new() - .create_if_missing(true) - .filename(&self.options.dst_file); - let mut conn = SqliteConnection::connect_with(&opt).await?; + let mut conn = SqliteConnection::connect_with( + &SqliteConnectOptions::new() + .create_if_missing(true) + .filename(&self.options.dst_file), + ) + .await?; let is_empty = query!("SELECT 1 as has_rows FROM sqlite_schema LIMIT 1") .fetch_optional(&mut conn) .await? .is_none(); - if !is_empty && self.options.diff_with_file.is_some() { + let dst_mbttype = if is_empty { + let dst_mbttype = self + .options + .dst_mbttype + .clone() + .unwrap_or_else(|| src_mbttype.clone()); + + self.create_new_mbtiles(&mut conn, &dst_mbttype, &src_mbttype) + .await?; + + dst_mbttype + } else if self.options.diff_with_file.is_some() { return Err(MbtError::NonEmptyTargetFile(self.options.dst_file)); - } + } else { + open_and_detect_type(&self.dst_mbtiles).await? + }; + + let rusqlite_conn = RusqliteConnection::open(Path::new(&self.dst_mbtiles.filepath()))?; + register_md5_function(&rusqlite_conn)?; + rusqlite_conn.execute( + "ATTACH DATABASE ? AS sourceDb", + [self.src_mbtiles.filepath()], + )?; + + let (on_dupl, sql_cond) = self.get_on_duplicate_sql(&dst_mbttype); + + let (select_from, query_args) = { + let select_from = if let Some(diff_file) = &self.options.diff_with_file { + let diff_with_mbtiles = Mbtiles::new(diff_file)?; + let diff_mbttype = open_and_detect_type(&diff_with_mbtiles).await?; + + rusqlite_conn + .execute("ATTACH DATABASE ? AS newDb", [diff_with_mbtiles.filepath()])?; + self.get_select_from_with_diff(&dst_mbttype, &diff_mbttype) + } else { + self.get_select_from(&dst_mbttype, &src_mbttype) + }; + + let (options_sql, query_args) = self.get_options_sql()?; + + (format!("{select_from} {options_sql}"), query_args) + }; + + match dst_mbttype { + Flat => rusqlite_conn.execute( + &format!("INSERT {on_dupl} INTO tiles {select_from} {sql_cond}"), + params_from_iter(query_args), + )?, + FlatWithHash => rusqlite_conn.execute( + &format!("INSERT {on_dupl} INTO tiles_with_hash {select_from} {sql_cond}"), + params_from_iter(query_args), + )?, + Normalized => { + rusqlite_conn.execute( + &format!("INSERT {on_dupl} INTO map (zoom_level, tile_column, tile_row, tile_id) SELECT zoom_level, tile_column, tile_row, hash as tile_id FROM ({select_from} {sql_cond})"), + params_from_iter(&query_args), + )?; + rusqlite_conn.execute( + &format!( + "INSERT OR IGNORE INTO images SELECT tile_data, hash FROM ({select_from})" + ), + params_from_iter(query_args), + )? + } + }; + + Ok(conn) + } + + async fn create_new_mbtiles( + &self, + conn: &mut SqliteConnection, + dst_mbttype: &MbtType, + src_mbttype: &MbtType, + ) -> MbtResult<()> { let path = self.src_mbtiles.filepath(); query!("ATTACH DATABASE ? AS sourceDb", path) - .execute(&mut conn) + .execute(&mut *conn) .await?; - if is_empty { - query!("PRAGMA page_size = 512").execute(&mut conn).await?; - query!("VACUUM").execute(&mut conn).await?; + query!("PRAGMA page_size = 512").execute(&mut *conn).await?; + query!("VACUUM").execute(&mut *conn).await?; - if force_simple { - for statement in &["CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);", - "CREATE TABLE tiles (zoom_level integer NOT NULL, tile_column integer NOT NULL, tile_row integer NOT NULL, tile_data blob, - PRIMARY KEY(zoom_level, tile_column, tile_row));"] { - query(statement).execute(&mut conn).await?; - } - } else { - // DB objects must be created in a specific order: tables, views, triggers, indexes. + if dst_mbttype != src_mbttype { + match dst_mbttype { + Flat => self.create_flat_tables(&mut *conn).await?, + FlatWithHash => self.create_flat_with_hash_tables(&mut *conn).await?, + Normalized => self.create_normalized_tables(&mut *conn).await?, + }; + } else { + // DB objects must be created in a specific order: tables, views, triggers, indexes. - for row in query( - "SELECT sql + for row in query( + "SELECT sql FROM sourceDb.sqlite_schema - WHERE tbl_name IN ('metadata', 'tiles', 'map', 'images') + WHERE tbl_name IN ('metadata', 'tiles', 'map', 'images', 'tiles_with_hash') AND type IN ('table', 'view', 'trigger', 'index') ORDER BY CASE WHEN type = 'table' THEN 1 WHEN type = 'view' THEN 2 WHEN type = 'trigger' THEN 3 WHEN type = 'index' THEN 4 ELSE 5 END", - ) - .fetch_all(&mut conn) - .await? - { - query(row.get(0)).execute(&mut conn).await?; - } - }; + ) + .fetch_all(&mut *conn) + .await? + { + query(row.get(0)).execute(&mut *conn).await?; + } + }; - query("INSERT INTO metadata SELECT * FROM sourceDb.metadata") - .execute(&mut conn) - .await?; - } else { - let dst_type = open_and_detect_type(&self.dst_mbtiles).await?; + query("INSERT INTO metadata SELECT * FROM sourceDb.metadata") + .execute(&mut *conn) + .await?; - if mbtiles_type == TileTables && dst_type == DeDuplicated { - return Err(MbtError::UnsupportedCopyOperation{ reason: "\ - Attempted copying from a source file with simple format to a non-empty destination file with deduplicated format, which is not currently supported" - .to_string(), - }); - } + Ok(()) + } - mbtiles_type = dst_type; - - if self.options.on_duplicate == CopyDuplicateMode::Abort - && query( - "SELECT * FROM tiles t1 - JOIN sourceDb.tiles t2 - ON t1.zoom_level=t2.zoom_level AND t1.tile_column=t2.tile_column AND t1.tile_row=t2.tile_row AND t1.tile_data!=t2.tile_data - LIMIT 1", - ) - .fetch_optional(&mut conn) - .await? - .is_some() - { - return Err(MbtError::DuplicateValues); - } + async fn create_flat_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> { + for statement in &[ + "CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);", + "CREATE TABLE tiles (zoom_level integer NOT NULL, tile_column integer NOT NULL, tile_row integer NOT NULL, tile_data blob, + PRIMARY KEY(zoom_level, tile_column, tile_row));"] { + query(statement).execute(&mut *conn).await?; } + Ok(()) + } - if force_simple { - self.copy_tile_tables(&mut conn).await? - } else { - match mbtiles_type { - TileTables => self.copy_tile_tables(&mut conn).await?, - DeDuplicated => self.copy_deduplicated(&mut conn).await?, - } + async fn create_flat_with_hash_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> { + for statement in &[ + "CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);", + "CREATE TABLE tiles_with_hash (zoom_level integer NOT NULL, tile_column integer NOT NULL, tile_row integer NOT NULL, tile_data blob, tile_hash text, + PRIMARY KEY(zoom_level, tile_column, tile_row));", + "CREATE VIEW tiles AS SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash;"] { + query(statement).execute(&mut *conn).await?; } - - Ok(conn) + Ok(()) } - async fn copy_tile_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> { - if let Some(diff_with) = &self.options.diff_with_file { - let diff_with_mbtiles = Mbtiles::new(diff_with)?; - // Make sure file is of valid type; the specific type is irrelevant - // because all types will be used in the same way - open_and_detect_type(&diff_with_mbtiles).await?; - - let path = diff_with_mbtiles.filepath(); - query!("ATTACH DATABASE ? AS newDb", path) - .execute(&mut *conn) - .await?; + async fn create_normalized_tables(&self, conn: &mut SqliteConnection) -> MbtResult<()> { + for statement in &[ + "CREATE TABLE metadata (name text NOT NULL PRIMARY KEY, value text);", + "CREATE TABLE map (zoom_level integer NOT NULL, tile_column integer NOT NULL, tile_row integer NOT NULL, tile_id text, + PRIMARY KEY(zoom_level, tile_column, tile_row));", + "CREATE TABLE images (tile_data blob, tile_id text NOT NULL PRIMARY KEY);", + "CREATE VIEW tiles AS + SELECT map.zoom_level AS zoom_level, map.tile_column AS tile_column, map.tile_row AS tile_row, images.tile_data AS tile_data + FROM map + JOIN images ON images.tile_id = map.tile_id"] { + query(statement).execute(&mut *conn).await?; + } + Ok(()) + } - self.run_query_with_options( - &mut *conn, - "INSERT INTO tiles - SELECT COALESCE(sourceDb.tiles.zoom_level, newDb.tiles.zoom_level) as zoom_level, - COALESCE(sourceDb.tiles.tile_column, newDb.tiles.tile_column) as tile_column, - COALESCE(sourceDb.tiles.tile_row, newDb.tiles.tile_row) as tile_row, - newDb.tiles.tile_data as tile_data - FROM sourceDb.tiles FULL JOIN newDb.tiles - ON sourceDb.tiles.zoom_level = newDb.tiles.zoom_level - AND sourceDb.tiles.tile_column = newDb.tiles.tile_column - AND sourceDb.tiles.tile_row = newDb.tiles.tile_row - WHERE (sourceDb.tiles.tile_data != newDb.tiles.tile_data - OR sourceDb.tiles.tile_data ISNULL - OR newDb.tiles.tile_data ISNULL)", - ) - .await - } else { - self.run_query_with_options( - conn, - // Allows for adding clauses to query using "AND" - &format!( - "INSERT {} INTO tiles SELECT * FROM sourceDb.tiles WHERE TRUE", - match &self.options.on_duplicate { - CopyDuplicateMode::Override => "OR REPLACE", - CopyDuplicateMode::Ignore | CopyDuplicateMode::Abort => "OR IGNORE", - } - ), - ) - .await + fn get_on_duplicate_sql(&self, mbttype: &MbtType) -> (String, String) { + match &self.options.on_duplicate { + CopyDuplicateMode::Override => ("OR REPLACE".to_string(), "".to_string()), + CopyDuplicateMode::Ignore => ("OR IGNORE".to_string(), "".to_string()), + CopyDuplicateMode::Abort => ("OR ABORT".to_string(), { + let (main_table, tile_identifier) = match mbttype { + Flat => ("tiles", "tile_data"), + FlatWithHash => ("tiles_with_hash", "tile_data"), + Normalized => ("map", "tile_id"), + }; + + format!( + "AND NOT EXISTS (\ + SELECT 1 \ + FROM {main_table} \ + WHERE \ + {main_table}.zoom_level=sourceDb.{main_table}.zoom_level \ + AND {main_table}.tile_column=sourceDb.{main_table}.tile_column \ + AND {main_table}.tile_row=sourceDb.{main_table}.tile_row \ + AND {main_table}.{tile_identifier}!=sourceDb.{main_table}.{tile_identifier}\ + )" + ) + }), } } - async fn copy_deduplicated(&self, conn: &mut SqliteConnection) -> MbtResult<()> { - let on_duplicate_sql = match &self.options.on_duplicate { - CopyDuplicateMode::Override => "OR REPLACE", - CopyDuplicateMode::Ignore | CopyDuplicateMode::Abort => "OR IGNORE", + fn get_select_from_with_diff(&self, dst_mbttype: &MbtType, diff_mbttype: &MbtType) -> String { + let (hash_col_sql, new_tiles_with_hash) = if dst_mbttype == &Flat { + ("", "newDb.tiles") + } else { + match *diff_mbttype { + Flat => (", hex(md5(tile_data)) as hash", "newDb.tiles"), + FlatWithHash => (", new_tiles_with_hash.tile_hash as hash", "newDb.tiles_with_hash"), + Normalized => (", new_tiles_with_hash.hash", "(SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash + FROM newDb.map JOIN newDb.images ON newDb.map.tile_id=newDb.images.tile_id )") + } }; - query(&format!( - "INSERT {on_duplicate_sql} INTO images - SELECT images.tile_data, images.tile_id - FROM sourceDb.images" - )) - .execute(&mut *conn) - .await?; - - self.run_query_with_options( - conn, - // Allows for adding clauses to query using "AND" - &format!("INSERT {on_duplicate_sql} INTO map SELECT * FROM sourceDb.map WHERE TRUE"), - ) - .await?; + format!("SELECT COALESCE(sourceDb.tiles.zoom_level, new_tiles_with_hash.zoom_level) as zoom_level, + COALESCE(sourceDb.tiles.tile_column, new_tiles_with_hash.tile_column) as tile_column, + COALESCE(sourceDb.tiles.tile_row, new_tiles_with_hash.tile_row) as tile_row, + new_tiles_with_hash.tile_data as tile_data + {hash_col_sql} + FROM sourceDb.tiles FULL JOIN {new_tiles_with_hash} AS new_tiles_with_hash + ON sourceDb.tiles.zoom_level = new_tiles_with_hash.zoom_level + AND sourceDb.tiles.tile_column = new_tiles_with_hash.tile_column + AND sourceDb.tiles.tile_row = new_tiles_with_hash.tile_row + WHERE (sourceDb.tiles.tile_data != new_tiles_with_hash.tile_data + OR sourceDb.tiles.tile_data ISNULL + OR new_tiles_with_hash.tile_data ISNULL)") + } - query("DELETE FROM images WHERE tile_id NOT IN (SELECT DISTINCT tile_id FROM map)") - .execute(&mut *conn) - .await?; + fn get_select_from(&self, dst_mbttype: &MbtType, src_mbttype: &MbtType) -> String { + let select_from = if dst_mbttype == &Flat { + "SELECT * FROM sourceDb.tiles " + } else { + match *src_mbttype { + Flat => "SELECT *, hex(md5(tile_data)) as hash FROM sourceDb.tiles ", + FlatWithHash => "SELECT zoom_level, tile_column, tile_row, tile_data, tile_hash AS hash FROM sourceDb.tiles_with_hash", + Normalized => "SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash FROM sourceDb.map JOIN sourceDb.images ON sourceDb.map.tile_id=sourceDb.images.tile_id" + } + }.to_string(); - Ok(()) + format!("{select_from} WHERE TRUE ") } - async fn run_query_with_options( - &self, - conn: &mut SqliteConnection, - sql: &str, - ) -> MbtResult<()> { - let mut params = SqliteArguments::default(); + fn get_options_sql(&self) -> MbtResult<(String, Vec)> { + let mut query_args = vec![]; let sql = if !&self.options.zoom_levels.is_empty() { for z in &self.options.zoom_levels { - params.add(z); + query_args.push(*z); } format!( - "{sql} AND zoom_level IN ({})", + " AND zoom_level IN ({})", vec!["?"; self.options.zoom_levels.len()].join(",") ) - } else if let Some(min_zoom) = &self.options.min_zoom { - if let Some(max_zoom) = &self.options.max_zoom { - params.add(min_zoom); - params.add(max_zoom); - format!("{sql} AND zoom_level BETWEEN ? AND ?") + } else if let Some(min_zoom) = self.options.min_zoom { + if let Some(max_zoom) = self.options.max_zoom { + query_args.push(min_zoom); + query_args.push(max_zoom); + " AND zoom_level BETWEEN ? AND ?".to_string() } else { - params.add(min_zoom); - format!("{sql} AND zoom_level >= ?") + query_args.push(min_zoom); + " AND zoom_level >= ?".to_string() } - } else if let Some(max_zoom) = &self.options.max_zoom { - params.add(max_zoom); - format!("{sql} AND zoom_level <= ?") + } else if let Some(max_zoom) = self.options.max_zoom { + query_args.push(max_zoom); + " AND zoom_level <= ?".to_string() } else { - sql.to_string() + "".to_string() }; - query_with(sql.as_str(), params).execute(conn).await?; - - Ok(()) + Ok((sql, query_args)) } } @@ -356,49 +419,77 @@ async fn open_and_detect_type(mbtiles: &Mbtiles) -> MbtResult { mbtiles.detect_type(&mut conn).await } -pub async fn apply_mbtiles_diff( - src_file: PathBuf, - diff_file: PathBuf, -) -> MbtResult { +pub async fn apply_mbtiles_diff(src_file: PathBuf, diff_file: PathBuf) -> MbtResult<()> { let src_mbtiles = Mbtiles::new(src_file)?; let diff_mbtiles = Mbtiles::new(diff_file)?; - let opt = SqliteConnectOptions::new().filename(src_mbtiles.filepath()); - let mut conn = SqliteConnection::connect_with(&opt).await?; - let src_type = src_mbtiles.detect_type(&mut conn).await?; + let src_mbttype = open_and_detect_type(&src_mbtiles).await?; + let diff_mbttype = open_and_detect_type(&diff_mbtiles).await?; - if src_type != TileTables { - return Err(MbtError::IncorrectDataFormat( - src_mbtiles.filepath().to_string(), - TileTables, - src_type, - )); + let rusqlite_conn = RusqliteConnection::open(Path::new(&src_mbtiles.filepath()))?; + register_md5_function(&rusqlite_conn)?; + rusqlite_conn.execute("ATTACH DATABASE ? AS diffDb", [diff_mbtiles.filepath()])?; + + let select_from = if src_mbttype == Flat { + "SELECT * FROM diffDb.tiles " + } else { + match diff_mbttype { + Flat => "SELECT *, hex(md5(tile_data)) as hash FROM diffDb.tiles ", + FlatWithHash => "SELECT zoom_level, tile_column, tile_row, tile_data, tile_hash AS hash FROM diffDb.tiles_with_hash", + Normalized => "SELECT zoom_level, tile_column, tile_row, tile_data, map.tile_id AS hash FROM diffDb.map LEFT JOIN diffDb.images ON diffDb.map.tile_id=diffDb.images.tile_id" + } + }.to_string(); + + let (insert_sql, main_table) = match src_mbttype { + Flat => (vec![format!( + "INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) {select_from}" + )], "tiles".to_string()), + FlatWithHash => (vec![format!( + "INSERT OR REPLACE INTO tiles_with_hash {select_from}" + )], "tiles_with_hash".to_string()), + Normalized => (vec![format!("INSERT OR REPLACE INTO map (zoom_level, tile_column, tile_row, tile_id) SELECT zoom_level, tile_column, tile_row, hash as tile_id FROM ({select_from})"), format!( + "INSERT OR REPLACE INTO images SELECT tile_data, hash FROM ({select_from})" + )], "map".to_string()) + }; + + for statement in insert_sql { + rusqlite_conn.execute(&format!("{statement} WHERE tile_data NOTNULL"), ())?; } - open_and_detect_type(&diff_mbtiles).await?; + rusqlite_conn.execute( + &format!( + "DELETE FROM {main_table} WHERE (zoom_level, tile_column, tile_row) IN (SELECT zoom_level, tile_column, tile_row FROM ({select_from} WHERE tile_data ISNULL));" + ),() + )?; - let path = diff_mbtiles.filepath(); - query!("ATTACH DATABASE ? AS diffDb", path) - .execute(&mut conn) - .await?; + Ok(()) +} - query( - " - DELETE FROM tiles - WHERE (zoom_level, tile_column, tile_row) IN - (SELECT zoom_level, tile_column, tile_row FROM diffDb.tiles WHERE tile_data ISNULL);", - ) - .execute(&mut conn) - .await?; - - query( - "INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) - SELECT * FROM diffDb.tiles WHERE tile_data NOTNULL;", - ) - .execute(&mut conn) - .await?; - - Ok(conn) +pub async fn validate_mbtiles(file: PathBuf) -> MbtResult<()> { + let mbtiles = Mbtiles::new(file)?; + let mbttype = open_and_detect_type(&mbtiles).await?; + + let sql = match mbttype { + Flat => { + return Err(IncorrectDataFormat( + mbtiles.filepath().to_string(), + &[FlatWithHash, Normalized], + Flat, + )); + } + FlatWithHash => "SELECT * FROM tiles_with_hash WHERE tile_hash!=hex(md5(tile_data));", + Normalized => "SELECT * FROM images WHERE tile_id!=hex(md5(tile_data));", + } + .to_string(); + + let rusqlite_conn = RusqliteConnection::open(Path::new(&mbtiles.filepath()))?; + register_md5_function(&rusqlite_conn)?; + + if rusqlite_conn.prepare(&sql)?.exists(())? { + return Err(InvalidTileData(mbtiles.filepath().to_string())); + } + + Ok(()) } pub async fn copy_mbtiles_file(opts: TileCopierOptions) -> MbtResult { @@ -409,16 +500,10 @@ pub async fn copy_mbtiles_file(opts: TileCopierOptions) -> MbtResult SqliteConnection { - SqliteConnection::connect_with(&SqliteConnectOptions::new().filename(path)) - .await - .unwrap() - } - async fn get_one(conn: &mut SqliteConnection, sql: &str) -> T where for<'r> T: Decode<'r, Sqlite> + Type, @@ -426,22 +511,39 @@ mod tests { query(sql).fetch_one(conn).await.unwrap().get::(0) } - async fn verify_copy_all(src_filepath: PathBuf, dst_filepath: PathBuf) { - let mut dst_conn = copy_mbtiles_file(TileCopierOptions::new( - src_filepath.clone(), - dst_filepath.clone(), - )) + async fn verify_copy_all( + src_filepath: PathBuf, + dst_filepath: PathBuf, + dst_mbttype: Option, + expected_dst_mbttype: MbtType, + ) { + let mut dst_conn = copy_mbtiles_file( + TileCopierOptions::new(src_filepath.clone(), dst_filepath.clone()) + .dst_mbttype(dst_mbttype), + ) .await .unwrap(); + query("ATTACH DATABASE ? AS srcDb") + .bind(src_filepath.clone().to_str().unwrap()) + .execute(&mut dst_conn) + .await + .unwrap(); + assert_eq!( - get_one::( - &mut open_sql(&src_filepath).await, - "SELECT COUNT(*) FROM tiles;" - ) - .await, - get_one::(&mut dst_conn, "SELECT COUNT(*) FROM tiles;").await + open_and_detect_type(&Mbtiles::new(dst_filepath).unwrap()) + .await + .unwrap(), + expected_dst_mbttype ); + + assert!( + query("SELECT * FROM srcDb.tiles EXCEPT SELECT * FROM tiles") + .fetch_optional(&mut dst_conn) + .await + .unwrap() + .is_none() + ) } async fn verify_copy_with_zoom_filter(opts: TileCopierOptions, expected_zoom_levels: u8) { @@ -458,41 +560,82 @@ mod tests { } #[actix_rt::test] - async fn copy_tile_tables() { + async fn copy_flat_tables() { let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); - let dst = PathBuf::from(":memory:"); - verify_copy_all(src, dst).await; + let dst = PathBuf::from("file:copy_flat_tables_mem_db?mode=memory&cache=shared"); + verify_copy_all(src, dst, None, Flat).await; } #[actix_rt::test] - async fn copy_deduplicated() { + async fn copy_flat_from_flat_with_hash_tables() { + let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles"); + let dst = PathBuf::from( + "file:copy_flat_from_flat_with_hash_tables_mem_db?mode=memory&cache=shared", + ); + verify_copy_all(src, dst, Some(Flat), Flat).await; + } + + #[actix_rt::test] + async fn copy_flat_from_normalized_tables() { let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles"); - let dst = PathBuf::from(":memory:"); - verify_copy_all(src, dst).await; + let dst = + PathBuf::from("file:copy_flat_from_normalized_tables_mem_db?mode=memory&cache=shared"); + verify_copy_all(src, dst, Some(Flat), Flat).await; } #[actix_rt::test] - async fn copy_with_force_simple() { - let src = PathBuf::from("../tests/fixtures/files/geography-class-jpg.mbtiles"); - let dst = PathBuf::from(":memory:"); + async fn copy_flat_with_hash_tables() { + let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles"); + let dst = PathBuf::from("file:copy_flat_with_hash_tables_mem_db?mode=memory&cache=shared"); + verify_copy_all(src, dst, None, FlatWithHash).await; + } - let copy_opts = TileCopierOptions::new(src.clone(), dst.clone()).force_simple(true); + #[actix_rt::test] + async fn copy_flat_with_hash_from_flat_tables() { + let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let dst = PathBuf::from( + "file:copy_flat_with_hash_from_flat_tables_mem_db?mode=memory&cache=shared", + ); + verify_copy_all(src, dst, Some(FlatWithHash), FlatWithHash).await; + } - let mut dst_conn = copy_mbtiles_file(copy_opts).await.unwrap(); + #[actix_rt::test] + async fn copy_flat_with_hash_from_normalized_tables() { + let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles"); + let dst = PathBuf::from( + "file:copy_flat_with_hash_from_normalized_tables_mem_db?mode=memory&cache=shared", + ); + verify_copy_all(src, dst, Some(FlatWithHash), FlatWithHash).await; + } - assert!( - query("SELECT 1 FROM sqlite_schema WHERE type='table' AND tbl_name='tiles';") - .fetch_optional(&mut dst_conn) - .await - .unwrap() - .is_some() + #[actix_rt::test] + async fn copy_normalized_tables() { + let src = PathBuf::from("../tests/fixtures/files/geography-class-png.mbtiles"); + let dst = PathBuf::from("file:copy_normalized_tables_mem_db?mode=memory&cache=shared"); + verify_copy_all(src, dst, None, Normalized).await; + } + + #[actix_rt::test] + async fn copy_normalized_from_flat_tables() { + let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let dst = + PathBuf::from("file:copy_normalized_from_flat_tables_mem_db?mode=memory&cache=shared"); + verify_copy_all(src, dst, Some(Normalized), Normalized).await; + } + + #[actix_rt::test] + async fn copy_normalized_from_flat_with_hash_tables() { + let src = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles"); + let dst = PathBuf::from( + "file:copy_normalized_from_flat_with_hash_tables_mem_db?mode=memory&cache=shared", ); + verify_copy_all(src, dst, Some(Normalized), Normalized).await; } #[actix_rt::test] async fn copy_with_min_max_zoom() { let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); - let dst = PathBuf::from(":memory:"); + let dst = PathBuf::from("file:copy_with_min_max_zoom_mem_db?mode=memory&cache=shared"); let opt = TileCopierOptions::new(src, dst) .min_zoom(Some(2)) .max_zoom(Some(4)); @@ -502,7 +645,7 @@ mod tests { #[actix_rt::test] async fn copy_with_zoom_levels() { let src = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); - let dst = PathBuf::from(":memory:"); + let dst = PathBuf::from("file:copy_with_zoom_levels_mem_db?mode=memory&cache=shared"); let opt = TileCopierOptions::new(src, dst) .min_zoom(Some(2)) .max_zoom(Some(4)) @@ -513,67 +656,64 @@ mod tests { #[actix_rt::test] async fn copy_with_diff_with_file() { let src = PathBuf::from("../tests/fixtures/files/geography-class-jpg.mbtiles"); - let dst = PathBuf::from(":memory:"); + let dst = PathBuf::from("file:copy_with_diff_with_file_mem_db?mode=memory&cache=shared"); let diff_file = PathBuf::from("../tests/fixtures/files/geography-class-jpg-modified.mbtiles"); - let copy_opts = TileCopierOptions::new(src.clone(), dst.clone()) - .diff_with_file(diff_file.clone()) - .force_simple(true); + let copy_opts = + TileCopierOptions::new(src.clone(), dst.clone()).diff_with_file(diff_file.clone()); let mut dst_conn = copy_mbtiles_file(copy_opts).await.unwrap(); - assert!( - query("SELECT 1 FROM sqlite_schema WHERE type='table' AND tbl_name='tiles';") - .fetch_optional(&mut dst_conn) - .await - .unwrap() - .is_some() - ); + assert!(query("SELECT 1 FROM sqlite_schema WHERE name='tiles';") + .fetch_optional(&mut dst_conn) + .await + .unwrap() + .is_some()); assert_eq!( - get_one::(&mut dst_conn, "SELECT COUNT(*) FROM tiles;").await, + get_one::(&mut dst_conn, "SELECT COUNT(*) FROM map;").await, 3 ); - assert_eq!( - get_one::( - &mut dst_conn, - "SELECT tile_data FROM tiles WHERE zoom_level=2 AND tile_row=2 AND tile_column=2;" - ) - .await, - 2 - ); + assert!(get_one::>( + &mut dst_conn, + "SELECT * FROM tiles WHERE zoom_level=2 AND tile_row=2 AND tile_column=2;" + ) + .await + .is_some()); - assert_eq!( - get_one::( - &mut dst_conn, - "SELECT tile_data FROM tiles WHERE zoom_level=1 AND tile_row=1 AND tile_column=1;" - ) - .await, - "4" - ); + assert!(get_one::>( + &mut dst_conn, + "SELECT * FROM tiles WHERE zoom_level=1 AND tile_row=1 AND tile_column=1;" + ) + .await + .is_some()); assert!(get_one::>( &mut dst_conn, - "SELECT tile_data FROM tiles WHERE zoom_level=0 AND tile_row=0 AND tile_column=0;" + "SELECT tile_id FROM map WHERE zoom_level=0 AND tile_row=0 AND tile_column=0;" ) .await .is_none()); } #[actix_rt::test] - async fn copy_from_simple_to_existing_deduplicated() { - let src = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles"); - let dst = PathBuf::from("../tests/fixtures/files/geography-class-jpg.mbtiles"); + async fn ignore_dst_mbttype_when_copy_to_existing() { + let src_file = PathBuf::from("../tests/fixtures/files/world_cities_modified.mbtiles"); - let copy_opts = TileCopierOptions::new(src.clone(), dst.clone()); + // Copy the dst file to an in-memory DB + let dst_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); + let dst = PathBuf::from( + "file:ignore_dst_mbttype_when_copy_to_existing_mem_db?mode=memory&cache=shared", + ); - assert!(matches!( - copy_mbtiles_file(copy_opts).await.unwrap_err(), - MbtError::UnsupportedCopyOperation { .. } - )); + let _dst_conn = copy_mbtiles_file(TileCopierOptions::new(dst_file.clone(), dst.clone())) + .await + .unwrap(); + + verify_copy_all(src_file, dst, Some(Normalized), Flat).await; } #[actix_rt::test] @@ -586,7 +726,7 @@ mod tests { assert!(matches!( copy_mbtiles_file(copy_opts).await.unwrap_err(), - MbtError::DuplicateValues + MbtError::RusqliteError(..) )); } @@ -681,18 +821,18 @@ mod tests { } #[actix_rt::test] - async fn apply_diff_file() { + async fn apply_flat_diff_file() { // Copy the src file to an in-memory DB let src_file = PathBuf::from("../tests/fixtures/files/world_cities.mbtiles"); - let src = PathBuf::from("file::memory:?cache=shared"); + let src = PathBuf::from("file:apply_flat_diff_file_mem_db?mode=memory&cache=shared"); - let _src_conn = copy_mbtiles_file(TileCopierOptions::new(src_file.clone(), src.clone())) + let mut src_conn = copy_mbtiles_file(TileCopierOptions::new(src_file.clone(), src.clone())) .await .unwrap(); // Apply diff to the src data in in-memory DB let diff_file = PathBuf::from("../tests/fixtures/files/world_cities_diff.mbtiles"); - let mut src_conn = apply_mbtiles_diff(src, diff_file).await.unwrap(); + apply_mbtiles_diff(src, diff_file).await.unwrap(); // Verify the data is the same as the file the diff was generated from let path = "../tests/fixtures/files/world_cities_modified.mbtiles"; @@ -709,4 +849,51 @@ mod tests { .is_none() ); } + + #[actix_rt::test] + async fn apply_normalized_diff_file() { + // Copy the src file to an in-memory DB + let src_file = PathBuf::from("../tests/fixtures/files/geography-class-jpg.mbtiles"); + let src = PathBuf::from("file:apply_normalized_diff_file_mem_db?mode=memory&cache=shared"); + + let mut src_conn = copy_mbtiles_file(TileCopierOptions::new(src_file.clone(), src.clone())) + .await + .unwrap(); + + // Apply diff to the src data in in-memory DB + let diff_file = PathBuf::from("../tests/fixtures/files/geography-class-jpg-diff.mbtiles"); + apply_mbtiles_diff(src, diff_file).await.unwrap(); + + // Verify the data is the same as the file the diff was generated from + let path = "../tests/fixtures/files/geography-class-jpg-modified.mbtiles"; + query!("ATTACH DATABASE ? AS otherDb", path) + .execute(&mut src_conn) + .await + .unwrap(); + + assert!( + query("SELECT * FROM tiles EXCEPT SELECT * FROM otherDb.tiles;") + .fetch_optional(&mut src_conn) + .await + .unwrap() + .is_none() + ); + } + + #[actix_rt::test] + async fn validate_valid_file() { + let file = PathBuf::from("../tests/fixtures/files/zoomed_world_cities.mbtiles"); + + validate_mbtiles(file).await.unwrap(); + } + + #[actix_rt::test] + async fn validate_invalid_file() { + let file = PathBuf::from("../tests/fixtures/files/invalid_zoomed_world_cities.mbtiles"); + + assert!(matches!( + validate_mbtiles(file).await.unwrap_err(), + MbtError::InvalidTileData(..) + )); + } } diff --git a/tests/expected/auto/catalog_auto.json b/tests/expected/auto/catalog_auto.json index bbb9bd143..aac15712a 100644 --- a/tests/expected/auto/catalog_auto.json +++ b/tests/expected/auto/catalog_auto.json @@ -80,6 +80,12 @@ "name": "Geography Class", "description": "One of the example maps that comes with TileMill - a bright & colorful world map that blends retro and high-tech with its folded paper texture and interactive flag tooltips. " }, + { + "id": "geography-class-jpg-diff", + "content_type": "image/jpeg", + "name": "Geography Class", + "description": "One of the example maps that comes with TileMill - a bright & colorful world map that blends retro and high-tech with its folded paper texture and interactive flag tooltips. " + }, { "id": "geography-class-jpg-modified", "content_type": "image/jpeg", @@ -98,6 +104,13 @@ "name": "Geography Class", "description": "One of the example maps that comes with TileMill - a bright & colorful world map that blends retro and high-tech with its folded paper texture and interactive flag tooltips. " }, + { + "id": "invalid_zoomed_world_cities", + "content_type": "application/x-protobuf", + "content_encoding": "gzip", + "name": "Major cities from Natural Earth data", + "description": "Major cities from Natural Earth data" + }, { "id": "json", "content_type": "application/json", @@ -187,5 +200,12 @@ "content_encoding": "gzip", "name": "Major cities from Natural Earth data", "description": "A modified version of major cities from Natural Earth data" + }, + { + "id": "zoomed_world_cities", + "content_type": "application/x-protobuf", + "content_encoding": "gzip", + "name": "Major cities from Natural Earth data", + "description": "Major cities from Natural Earth data" } ] diff --git a/tests/expected/generated_config.yaml b/tests/expected/generated_config.yaml index a20c2cce9..47984d7c9 100644 --- a/tests/expected/generated_config.yaml +++ b/tests/expected/generated_config.yaml @@ -175,12 +175,15 @@ mbtiles: paths: tests/fixtures/files sources: geography-class-jpg: tests/fixtures/files/geography-class-jpg.mbtiles + geography-class-jpg-diff: tests/fixtures/files/geography-class-jpg-diff.mbtiles geography-class-jpg-modified: tests/fixtures/files/geography-class-jpg-modified.mbtiles geography-class-png: tests/fixtures/files/geography-class-png.mbtiles geography-class-png-no-bounds: tests/fixtures/files/geography-class-png-no-bounds.mbtiles + invalid_zoomed_world_cities: tests/fixtures/files/invalid_zoomed_world_cities.mbtiles json: tests/fixtures/files/json.mbtiles uncompressed_mvt: tests/fixtures/files/uncompressed_mvt.mbtiles webp: tests/fixtures/files/webp.mbtiles world_cities: tests/fixtures/files/world_cities.mbtiles world_cities_diff: tests/fixtures/files/world_cities_diff.mbtiles world_cities_modified: tests/fixtures/files/world_cities_modified.mbtiles + zoomed_world_cities: tests/fixtures/files/zoomed_world_cities.mbtiles diff --git a/tests/expected/mbtiles/help.txt b/tests/expected/mbtiles/help.txt index 832fc4492..6851b03ec 100644 --- a/tests/expected/mbtiles/help.txt +++ b/tests/expected/mbtiles/help.txt @@ -6,6 +6,7 @@ Commands: meta-get Gets a single value from the MBTiles metadata table copy Copy tiles from one mbtiles file to another apply-diff Apply diff file generated from 'copy' command + validate Validate tile data if hash of tile data exists in file help Print this message or the help of the given subcommand(s) Options: diff --git a/tests/fixtures/files/geography-class-jpg-diff.mbtiles b/tests/fixtures/files/geography-class-jpg-diff.mbtiles new file mode 100644 index 000000000..ab3e1b19b Binary files /dev/null and b/tests/fixtures/files/geography-class-jpg-diff.mbtiles differ diff --git a/tests/fixtures/files/geography-class-jpg-modified.mbtiles b/tests/fixtures/files/geography-class-jpg-modified.mbtiles index 819473235..5a4540f2e 100644 Binary files a/tests/fixtures/files/geography-class-jpg-modified.mbtiles and b/tests/fixtures/files/geography-class-jpg-modified.mbtiles differ diff --git a/tests/fixtures/files/invalid_zoomed_world_cities.mbtiles b/tests/fixtures/files/invalid_zoomed_world_cities.mbtiles new file mode 100644 index 000000000..9e0e31c0c Binary files /dev/null and b/tests/fixtures/files/invalid_zoomed_world_cities.mbtiles differ diff --git a/tests/fixtures/files/zoomed_world_cities.mbtiles b/tests/fixtures/files/zoomed_world_cities.mbtiles new file mode 100644 index 000000000..9edf9d4fd Binary files /dev/null and b/tests/fixtures/files/zoomed_world_cities.mbtiles differ diff --git a/tests/test.sh b/tests/test.sh index f613fee24..c2238e78f 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -296,7 +296,6 @@ if [[ "$MBTILES_BIN" != "-" ]]; then ./tests/fixtures/files/world_cities.mbtiles \ "$TEST_TEMP_DIR/world_cities_diff.mbtiles" \ --diff-with-file ./tests/fixtures/files/world_cities_modified.mbtiles \ - --force-simple \ 2>&1 | tee "$TEST_OUT_DIR/copy_diff.txt" if command -v sqlite3 > /dev/null; then @@ -313,7 +312,6 @@ if [[ "$MBTILES_BIN" != "-" ]]; then # Ensure that applying the diff resulted in the modified version of the file $MBTILES_BIN copy \ --diff-with-file "$TEST_TEMP_DIR/world_cities_copy.mbtiles" \ - --force-simple \ ./tests/fixtures/files/world_cities_modified.mbtiles \ "$TEST_TEMP_DIR/world_cities_diff_modified.mbtiles" \ 2>&1 | tee "$TEST_OUT_DIR/copy_diff2.txt"