Add flat hashing table format (maplibre#808)

* Add `MbtType::FlatWithHash`
* Support copying, diffing and applying diffs to and from any
`MbtTypes`s
* Support validating tile data if hash is contained in `*.mbtiles` file
(i.e it is of `MbtType::FlatWithHash` or `MbtType::Normalized`)

---------

Co-authored-by: rstanciu <[email protected]>
Co-authored-by: Yuri Astrakhan <[email protected]>

Cargo.toml

@@ -44,6 +44,7 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_yaml = "0.9"
 spreet = { version = "0.8", default-features = false }
+sqlite-hashes = "0.1"
 sqlx = { version = "0.7", features = ["sqlite"] }
 subst = { version = "0.2", features = ["yaml"] }
 thiserror = "1"

docs/src/tools.md

@@ -15,19 +15,24 @@ mbtiles meta-get my_file.mbtiles description
 ```
 
 ### copy
-Copy an mbtiles file, optionally filtering its content by zoom levels. Can also flatten mbtiles file from de-duplicated tiles to a simple table structure.
+Copy an mbtiles file, optionally filtering its content by zoom levels.
 
 ```shell
 mbtiles copy src_file.mbtiles dst_file.mbtiles \
-        --min-zoom 0 --max-zoom 10 --force-simple
+        --min-zoom 0 --max-zoom 10
 ```
 
 Copy command can also be used to compare two mbtiles files and generate a diff.
 ```shell
 mbtiles copy src_file.mbtiles diff_file.mbtiles \
-        --force-simple --diff-with-file modified_file.mbtiles
+         --diff-with-file modified_file.mbtiles
 ```
 
+This command can also be used to generate files of different [supported schema](##supported-schema).
+```shell
+mbtiles copy normalized.mbtiles dst.mbtiles \
+         --dst-mbttype flat-with-hash
+```
 ### apply-diff
 Apply the diff file generated from `copy` command above to an mbtiles file. The diff file can be applied to the `src_file.mbtiles` elsewhere, to avoid copying/transmitting the entire modified dataset.
 ```shell
@@ -44,4 +49,42 @@ sqlite3 src_file.mbtiles \
   "INSERT OR REPLACE INTO tiles (zoom_level, tile_column, tile_row, tile_data) SELECT * FROM diffDb.tiles WHERE tile_data NOTNULL;"
 ```
 
-**_NOTE:_** Both of these methods for applying a diff _only_ work for mbtiles files in the simple tables format; they do _not_ work for mbtiles files in deduplicated format.
+**_NOTE:_** Both of these methods for applying a diff _only_ work for mbtiles files in the simple tables format; they do _not_ work for mbtiles files in normalized_tables format.
+
+### validate
+If the `.mbtiles` file is of `flat_with_hash` or `normalized` type, then verify that the data stored in columns `tile_hash` and `tile_id` respectively are MD5 hashes of the `tile_data` column.
+```shell
+mbtiles validate src_file.mbtiles
+```
+
+## Supported Schema
+The `mbtiles` tool supports three different kinds of schema for `tiles` data in `.mbtiles` files:
+
+- `flat`: 
+    ```
+    CREATE TABLE tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob);
+    CREATE UNIQUE INDEX tile_index on tiles (zoom_level, tile_column, tile_row);
+    ```
+- `flat-with-hash`:
+    ```
+    CREATE TABLE tiles_with_hash (zoom_level integer NOT NULL, tile_column integer NOT NULL, tile_row integer NOT NULL, tile_data blob, tile_hash text);
+    CREATE UNIQUE INDEX tiles_with_hash_index on tiles_with_hash (zoom_level, tile_column, tile_row);
+    CREATE VIEW tiles AS SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles_with_hash;
+    ```
+- `normalized`:
+    ```
+    CREATE TABLE map (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, tile_id TEXT); 
+    CREATE UNIQUE INDEX map_index ON map (zoom_level, tile_column, tile_row);
+    CREATE TABLE images (tile_data blob, tile_id text);
+    CREATE UNIQUE INDEX images_id ON images (tile_id);
+    CREATE VIEW tiles AS
+      SELECT
+          map.zoom_level AS zoom_level,
+          map.tile_column AS tile_column,
+          map.tile_row AS tile_row,
+          images.tile_data AS tile_data
+      FROM map
+      JOIN images ON images.tile_id = map.tile_id;
+    ```
+
+For more general spec information, see [here](https://github.com/mapbox/mbtiles-spec#readme).

martin-mbtiles/Cargo.toml

@@ -29,6 +29,7 @@ tilejson.workspace = true
 # Bin dependencies
 anyhow = { workspace = true, optional = true }
 clap = { workspace = true, optional = true }
+sqlite-hashes.workspace = true
 tokio = { workspace = true, features = ["rt-multi-thread"], optional = true }
 
 [dev-dependencies]

martin-mbtiles/src/bin/main.rs

@@ -2,7 +2,9 @@ use std::path::{Path, PathBuf};
 
 use anyhow::Result;
 use clap::{Parser, Subcommand};
-use martin_mbtiles::{apply_mbtiles_diff, copy_mbtiles_file, Mbtiles, TileCopierOptions};
+use martin_mbtiles::{
+    apply_mbtiles_diff, copy_mbtiles_file, validate_mbtiles, Mbtiles, TileCopierOptions,
+};
 use sqlx::sqlite::SqliteConnectOptions;
 use sqlx::{Connection, SqliteConnection};
 
@@ -53,6 +55,12 @@ enum Commands {
         /// Diff file
         diff_file: PathBuf,
     },
+    /// Validate tile data if hash of tile data exists in file
+    #[command(name = "validate")]
+    Validate {
+        /// MBTiles file to validate
+        file: PathBuf,
+    },
 }
 
 #[tokio::main]
@@ -72,6 +80,9 @@ async fn main() -> Result<()> {
         } => {
             apply_mbtiles_diff(src_file, diff_file).await?;
         }
+        Commands::Validate { file } => {
+            validate_mbtiles(file).await?;
+        }
     }
 
     Ok(())
@@ -96,7 +107,7 @@ mod tests {
     use martin_mbtiles::{CopyDuplicateMode, TileCopierOptions};
 
     use crate::Args;
-    use crate::Commands::{ApplyDiff, Copy, MetaGetValue};
+    use crate::Commands::{ApplyDiff, Copy, MetaGetValue, Validate};
 
     #[test]
     fn test_copy_no_arguments() {
@@ -205,23 +216,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn test_copy_diff_with_file_no_force_simple_arguments() {
-        assert_eq!(
-            Args::try_parse_from([
-                "mbtiles",
-                "copy",
-                "src_file",
-                "dst_file",
-                "--diff-with-file",
-                "no_file",
-            ])
-            .unwrap_err()
-            .kind(),
-            ErrorKind::MissingRequiredArgument
-        );
-    }
-
     #[test]
     fn test_copy_diff_with_file_arguments() {
         assert_eq!(
@@ -232,14 +226,12 @@ mod tests {
                 "dst_file",
                 "--diff-with-file",
                 "no_file",
-                "--force-simple"
             ]),
             Args {
                 verbose: false,
                 command: Copy(
                     TileCopierOptions::new(PathBuf::from("src_file"), PathBuf::from("dst_file"))
                         .diff_with_file(PathBuf::from("no_file"))
-                        .force_simple(true)
                 )
             }
         );
@@ -345,4 +337,17 @@ mod tests {
             }
         );
     }
+
+    #[test]
+    fn test_validate() {
+        assert_eq!(
+            Args::parse_from(["mbtiles", "validate", "src_file"]),
+            Args {
+                verbose: false,
+                command: Validate {
+                    file: PathBuf::from("src_file"),
+                }
+            }
+        );
+    }
 }