From c3af2a0f42851cb6bc2e39c01e9378d2060edd16 Mon Sep 17 00:00:00 2001 From: upsicleclown Date: Fri, 30 Jun 2023 13:48:22 -0700 Subject: [PATCH] WIP - add filtering by BBox --- Cargo.lock | 247 +++++++++++++++++++++++++++--- Cargo.toml | 2 + martin-mbtiles/Cargo.toml | 5 +- martin-mbtiles/src/bin/main.rs | 4 +- martin-mbtiles/src/tile_copier.rs | 69 ++++++++- 5 files changed, 302 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6defa4e6d..d62c0a086 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,7 +88,7 @@ dependencies = [ "mime", "percent-encoding", "pin-project-lite", - "rand", + "rand 0.8.5", "sha1", "smallvec", "tokio", @@ -156,7 +156,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b894941f818cfdc7ccc4b9e60fa7e53b5042a2e8567270f9147d5591893373a" dependencies = [ "futures-core", - "paste", + "paste 1.0.12", "pin-project-lite", ] @@ -255,7 +255,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom", + "getrandom 0.2.10", "once_cell", "version_check", ] @@ -267,7 +267,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" dependencies = [ "cfg-if", - "getrandom", + "getrandom 0.2.10", "once_cell", "version_check", ] @@ -290,6 +290,17 @@ dependencies = [ "memchr", ] +[[package]] +name = "alga" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f823d037a7ec6ea2197046bafd4ae150e6bc36f9ca347404f46a46823fa84f2" +dependencies = [ + "approx", + "num-complex", + "num-traits", +] + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -372,6 +383,15 @@ version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +[[package]] +name = "approx" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3" +dependencies = [ + "num-traits", +] + [[package]] name = "arrayref" version = "0.3.7" @@ -479,6 +499,18 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bbox" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9712ee319cd1ea1ec89883e9e6bf9439ccf89ab761ea1aacc6aad59724c5216c" +dependencies = [ + "alga", + "approx", + "nalgebra", + "num-traits", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -503,7 +535,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -853,7 +885,7 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array", + "generic-array 0.14.7", "typenum", ] @@ -1317,6 +1349,15 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f797e67af32588215eaaab8327027ee8e71b9dd0b2b26996aedf20c030fce309" +dependencies = [ + "typenum", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1327,6 +1368,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.10" @@ -1335,7 +1387,7 @@ checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", ] [[package]] @@ -1556,7 +1608,7 @@ dependencies = [ "bytemuck", "byteorder", "color_quant", - "num-rational", + "num-rational 0.4.1", "num-traits", "png", ] @@ -1840,10 +1892,13 @@ version = "0.2.2" dependencies = [ "actix-rt", "anyhow", + "bbox", "clap", "futures", "log", "martin-tile-utils", + "nalgebra", + "serde", "serde_json", "sqlx", "thiserror", @@ -1855,6 +1910,15 @@ dependencies = [ name = "martin-tile-utils" version = "0.1.2" +[[package]] +name = "matrixmultiply" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "916806ba0031cd542105d916a97c8572e1fa6dd79c9c51e7eb43a09ec2dd84c1" +dependencies = [ + "rawpointer", +] + [[package]] name = "md-5" version = "0.10.5" @@ -1936,7 +2000,7 @@ checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.48.0", ] @@ -1949,6 +2013,26 @@ dependencies = [ "serde", ] +[[package]] +name = "nalgebra" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500430fe18b836c0099c22e716139eae37210ecf8e1ae3931b73078e9a2de7a9" +dependencies = [ + "alga", + "approx", + "generic-array 0.13.3", + "matrixmultiply", + "num-complex", + "num-rational 0.2.4", + "num-traits", + "rand 0.7.3", + "rand_distr", + "serde", + "simba", + "typenum", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -1989,11 +2073,22 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "smallvec", "zeroize", ] +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg", + "num-traits", + "serde", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -2015,6 +2110,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.4.1" @@ -2191,12 +2297,31 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "paste" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + [[package]] name = "paste" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +[[package]] +name = "paste-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" +dependencies = [ + "proc-macro-hack", +] + [[package]] name = "pem-rfc7468" version = "0.6.0" @@ -2407,7 +2532,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand", + "rand 0.8.5", "sha2", "stringprep", ] @@ -2461,6 +2586,12 @@ dependencies = [ "termtree", ] +[[package]] +name = "proc-macro-hack" +version = "0.5.20+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" + [[package]] name = "proc-macro2" version = "1.0.60" @@ -2485,6 +2616,19 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -2492,8 +2636,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", ] [[package]] @@ -2503,7 +2657,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", ] [[package]] @@ -2512,9 +2675,33 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.10", +] + +[[package]] +name = "rand_distr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96977acbdd3a6576fb1d27391900035bf3863d4a16422973a409b488cf29ffb2" +dependencies = [ + "rand 0.7.3", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.7.0" @@ -2639,7 +2826,7 @@ dependencies = [ "num-traits", "pkcs1", "pkcs8", - "rand_core", + "rand_core 0.6.4", "signature", "subtle", "zeroize", @@ -2885,7 +3072,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" dependencies = [ "digest", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "simba" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051b345c9ec09734622cbb49e20dfcf72c9b55e473f0ca6a8025409e37c25fb8" +dependencies = [ + "approx", + "num-complex", + "num-traits", + "paste 0.1.18", ] [[package]] @@ -3049,7 +3248,7 @@ dependencies = [ "memchr", "native-tls", "once_cell", - "paste", + "paste 1.0.12", "percent-encoding", "serde", "serde_json", @@ -3120,7 +3319,7 @@ dependencies = [ "futures-core", "futures-io", "futures-util", - "generic-array", + "generic-array 0.14.7", "hex", "hkdf", "hmac", @@ -3130,7 +3329,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rand", + "rand 0.8.5", "rsa", "serde", "sha1", @@ -3169,7 +3368,7 @@ dependencies = [ "md-5", "memchr", "once_cell", - "rand", + "rand 0.8.5", "serde", "serde_json", "sha1", @@ -3792,6 +3991,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 90217312c..c0d636574 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ actix-rt = "2" actix-web = "4" anyhow = "1.0" async-trait = "0.1" +bbox = "0.11.2" brotli = "3" cargo-husky = { version = "1", features = ["user-hooks"], default-features = false } clap = { version = "4", features = ["derive"] } @@ -29,6 +30,7 @@ itertools = "0.11" log = "0.4" martin-mbtiles = { path = "./martin-mbtiles", version = "0.2.0", default-features = false } # disable CLI tools martin-tile-utils = { path = "./martin-tile-utils", version = "0.1.0" } +nalgebra = { version = "0.22.1", features = ["serde-serialize"] } num_cpus = "1" openssl = "0.10" pmtiles = { version = "0.2.2", features = ["mmap-async-tokio", "tilejson"] } diff --git a/martin-mbtiles/Cargo.toml b/martin-mbtiles/Cargo.toml index 5788cd83d..5adbbcc72 100644 --- a/martin-mbtiles/Cargo.toml +++ b/martin-mbtiles/Cargo.toml @@ -12,12 +12,13 @@ license.workspace = true [features] # TODO: Disable "cli" feature in default builds default = ["cli"] -cli = ["dep:anyhow", "dep:clap", "dep:tokio"] +cli = ["dep:anyhow", "dep:bbox", "dep:clap", "dep:nalgebra","dep:tokio"] [dependencies] futures.workspace = true log.workspace = true martin-tile-utils.workspace = true +serde.workspace = true serde_json.workspace = true sqlx.workspace = true thiserror.workspace = true @@ -25,7 +26,9 @@ tilejson.workspace = true # Bin dependencies anyhow = { workspace = true, optional = true } +bbox = { workspace = true, optional = true } clap = { workspace = true, optional = true } +nalgebra = { workspace = true, optional = true } tokio = { workspace = true, optional = true } [dev-dependencies] diff --git a/martin-mbtiles/src/bin/main.rs b/martin-mbtiles/src/bin/main.rs index 25d6b1a3d..4b5924f7c 100644 --- a/martin-mbtiles/src/bin/main.rs +++ b/martin-mbtiles/src/bin/main.rs @@ -6,7 +6,7 @@ use martin_mbtiles::{apply_mbtiles_diff, copy_mbtiles_file, Mbtiles, TileCopierO use sqlx::sqlite::SqliteConnectOptions; use sqlx::{Connection, SqliteConnection}; -#[derive(Parser, PartialEq, Eq, Debug)] +#[derive(Parser, PartialEq, Debug)] #[command( version, name = "mbtiles", @@ -20,7 +20,7 @@ pub struct Args { command: Commands, } -#[derive(Subcommand, PartialEq, Eq, Debug)] +#[derive(Subcommand, PartialEq, Debug)] enum Commands { // /// Prints all values in the metadata table. // #[command(name = "meta-all")] diff --git a/martin-mbtiles/src/tile_copier.rs b/martin-mbtiles/src/tile_copier.rs index b77b308c0..982a88d96 100644 --- a/martin-mbtiles/src/tile_copier.rs +++ b/martin-mbtiles/src/tile_copier.rs @@ -1,6 +1,8 @@ extern crate core; +use serde::{Deserialize, Serialize}; use std::collections::HashSet; +use std::fmt::Debug; use std::path::PathBuf; #[cfg(feature = "cli")] @@ -11,8 +13,10 @@ use sqlx::{query, query_with, Arguments, Connection, Row, SqliteConnection}; use crate::errors::MbtResult; use crate::mbtiles::MbtType; use crate::{MbtError, Mbtiles}; +use bbox::BoundingBox; +use nalgebra::Point3; -#[derive(Clone, Default, PartialEq, Eq, Debug)] +#[derive(Clone, PartialEq, Debug)] #[cfg_attr(feature = "cli", derive(Args))] pub struct TileCopierOptions { /// MBTiles file to read from @@ -31,15 +35,72 @@ pub struct TileCopierOptions { /// List of zoom levels to copy #[cfg_attr(feature = "cli", arg(long, value_parser(ValueParser::new(HashSetValueParser{})), default_value=""))] zoom_levels: HashSet, + /// Bbox to be used to filter tiles + #[cfg_attr(feature = "cli", arg(long, value_parser(ValueParser::new(BoundingBoxValueParser{}))))] + bbox: BoundingBox, /// Compare source file with this file, and only copy non-identical tiles to destination #[cfg_attr(feature = "cli", arg(long, requires("force_simple")))] diff_with_file: Option, } +#[cfg(feature = "cli")] +#[derive(Clone)] +struct BoundingBoxValueParser; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(remote = "BoundingBox")] +pub struct BoundingBoxDef { + pub min: Point3, + pub max: Point3, +} + +#[derive(serde::Serialize)] +struct BoundingBoxWrapper<'a>(#[serde(with = "BoundingBoxDef")] &'a BoundingBox); + #[cfg(feature = "cli")] #[derive(Clone)] struct HashSetValueParser; +#[cfg(feature = "cli")] +impl clap::builder::TypedValueParser for BoundingBoxValueParser { + type Value = BoundingBox; + + fn parse_ref( + &self, + _cmd: &clap::Command, + _arg: Option<&clap::Arg>, + value: &std::ffi::OsStr, + ) -> Result { + /*let mut result = HashSet::::new(); + let values = value + .to_str() + .ok_or(clap::Error::new(ErrorKind::ValueValidation))? + .trim(); + if !values.is_empty() { + for val in values.split(',') { + result.insert( + val.trim() + .parse::() + .map_err(|_| clap::Error::new(ErrorKind::ValueValidation))?, + ); + } + }*/ + /* let bbox = BoundingBox::new(&Point3::new(1.0, 2.0, 3.0), &Point3::new(1.1, 2.2, 3.3)); + println!( + "{}", + serde_json::to_string(&BoundingBoxWrapper(&bbox)).unwrap() + + {"min":[1.0,2.0,3.0],"max":[1.1,2.2,3.3]} + );*/ + let value_string = value + .to_str() + .ok_or(clap::Error::new(ErrorKind::ValueValidation))? + .trim(); + let mut de = serde_json::Deserializer::from_str(value_string); + Ok(BoundingBoxDef::::deserialize(&mut de).unwrap()) + } +} + #[cfg(feature = "cli")] impl clap::builder::TypedValueParser for HashSetValueParser { type Value = HashSet; @@ -83,6 +144,7 @@ impl TileCopierOptions { force_simple: false, min_zoom: None, max_zoom: None, + bbox: BoundingBox::new(&Point3::new(1.0, 2.0, 3.0), &Point3::new(1.1, 2.2, 3.3)), diff_with_file: None, } } @@ -111,6 +173,11 @@ impl TileCopierOptions { self.diff_with_file = Some(diff_with_file); self } + + pub fn bbox(mut self, bbox: BoundingBox) -> Self { + self.bbox = bbox; + self + } } impl TileCopier {