Change origin from French IGN datasource #1834

osm-fr · Dec 26, 2023 · 76b5f58 · 76b5f58
1 parent c708953
commit 76b5f58
Show file tree

Hide file tree

Showing 9 changed files with 40 additions and 40 deletions.
diff --git a/analysers/Analyser_Merge.py b/analysers/Analyser_Merge.py
@@ -37,6 +37,7 @@
 import fnmatch
 import shutil
 import subprocess
+import pathlib
 from typing import Optional, Dict, Union, Callable
 from collections import defaultdict
 from .Analyser_Osmosis import Analyser_Osmosis
@@ -403,7 +404,7 @@ def open(self, binary = False):
             with libarchive.public.memory_reader(f.read()) as archive:
                 f = io.BytesIO()
                 for entry in archive:
-                    if entry.pathname == self.extract:
+                    if pathlib.Path(entry.pathname).match(self.extract):
                         for block in entry.get_blocks():
                             f.write(block)
                         break
@@ -525,11 +526,23 @@ def get_millesime(self):
         )
 
 class SourceIGN(Source):
-    """Get millesime from IGN BDTOPO MetaData"""
-    def get_millesime(self) -> datetime.datetime:
-        response = downloader.request_get("https://files.opendatarchives.fr/professionnels.ign.fr/bdtopo/latest/geopackage/meta.json")
+    def __init__(self, dep_code, **kwargs):
+        response = downloader.request_get("https://geoservices.ign.fr/bdtopo")
         response.raise_for_status()
-        return datetime.datetime.fromisoformat(response.json()["millesime"])
+        if len(str(dep_code)) == 2:
+            dep_code = f"0{dep_code}"
+        match = re.search(
+            f"https://wxs.ign.fr/859x8t863h6a09o9o6fy4v60/telechargement/prepackage/BDTOPOV3-TOUSTHEMES-DEPARTEMENT_GPKG_PACK_233\$BDTOPO_3-[0-9]+_TOUSTHEMES_GPKG_[A-Z0-9]+_D{dep_code}_[-0-9]+/file/BDTOPO_3-[0-9]+_TOUSTHEMES_GPKG_[A-Z0-9]+_D{dep_code}_([-0-9]+)\.7z",
+            response.text
+        )
+        url, date = match[0], match[1]
+        kwargs.update({
+            "fileUrl": url,
+            "millesime": date,
+            "extract": "**/*.gpkg",
+            "attribution": "IGN",
+        })
+        super().__init__(**kwargs)
 
 class Parser:
     def __init__(self, srid: Optional[int] = None):

diff --git a/analysers/analyser_merge_cemetery_FR.py b/analysers/analyser_merge_cemetery_FR.py
@@ -33,8 +33,8 @@ def __init__(self, config, logger = None):
         self.init(
             "https://ign.fr",
             "IGN-Cimetière",
-            GPKG(SourceIGN(attribution = "IGN", gzip = True,
-                    fileUrl = "http://files.opendatarchives.fr/professionnels.ign.fr/bdtopo/latest/geopackage/cimetiere.gpkg.gz")),
+            GPKG(SourceIGN(dep_code = config.options.get('dep_code') or config.options.get('country').split('-')[1]),
+                layer = "cimetiere"),
             LoadGeomCentroid(
                 select = {"etat_de_l_objet": "En service"}),
             Conflate(

diff --git a/analysers/analyser_merge_man_made_FR.py b/analysers/analyser_merge_man_made_FR.py
@@ -132,8 +132,8 @@ def __init__(self, config, error_file, logger, item, id, level, title, select, t
         self.init(
             "https://ign.fr",
             "IGN-Construction ponctuelle",
-            GPKG(SourceIGN(attribution = "IGN", gzip = True,
-                    fileUrl = "http://files.opendatarchives.fr/professionnels.ign.fr/bdtopo/latest/geopackage/construction_ponctuelle.gpkg.gz")),
+            GPKG(SourceIGN(dep_code = config.options.get('dep_code') or config.options.get('country').split('-')[1]),
+                layer = "construction_ponctuelle"),
             LoadGeomCentroid(
                 select = select),
             Conflate(

diff --git a/analysers/analyser_merge_natural_FR.py b/analysers/analyser_merge_natural_FR.py
@@ -275,8 +275,8 @@ def __init__(self, config, error_file, logger, item, id, level, title, select, t
         self.init(
             "https://ign.fr",
             "IGN-Détail orographique",
-            GPKG(SourceIGN(attribution = "IGN", gzip = True,
-                    fileUrl = "http://files.opendatarchives.fr/professionnels.ign.fr/bdtopo/latest/geopackage/detail_orographique.gpkg.gz")),
+            GPKG(SourceIGN(dep_code = config.options.get('dep_code') or config.options.get('country').split('-')[1]),
+                layer = "detail_orographique"),
             LoadGeomCentroid(
                 select = select),
             Conflate(

diff --git a/analysers/analyser_merge_poi_FR.py b/analysers/analyser_merge_poi_FR.py
@@ -304,8 +304,8 @@ def __init__(self, config, error_file, logger, item, id, level, title, select, t
         self.init(
             "https://ign.fr",
             "IGN-Zone d'activité ou d'intérêt",
-            GPKG(SourceIGN(attribution = "IGN", gzip = True,
-                    fileUrl = "http://files.opendatarchives.fr/professionnels.ign.fr/bdtopo/latest/geopackage/zone_d_activite_ou_d_interet.gpkg.gz")),
+            GPKG(SourceIGN(dep_code = config.options.get('dep_code') or config.options.get('country').split('-')[1]),
+                layer = "zone_d_activite_ou_d_interet"),
             LoadGeomCentroid(
                 select = select),
             Conflate(

diff --git a/analysers/analyser_merge_reservoir_FR.py b/analysers/analyser_merge_reservoir_FR.py
@@ -71,8 +71,8 @@ def __init__(self, config, error_file, logger, item, id, level, title, select, t
         self.init(
             "https://ign.fr",
             "IGN-Construction ponctuelle",
-            GPKG(SourceIGN(attribution = "IGN", gzip = True,
-                    fileUrl = "http://files.opendatarchives.fr/professionnels.ign.fr/bdtopo/latest/geopackage/reservoir.gpkg.gz")),
+            GPKG(SourceIGN(dep_code = config.options.get('dep_code') or config.options.get('country').split('-')[1]),
+                layer = "reservoir"),
             LoadGeomCentroid(
                 select = select),
             Conflate(

diff --git a/analysers/analyser_merge_road_FR.py b/analysers/analyser_merge_road_FR.py
@@ -26,16 +26,6 @@
 
 
 class Analyser_Merge_Road_FR(Analyser_Merge_Network):
-    dep_proj = {
-        "971": "RGAF09UTM20",
-        "972": "RGAF09UTM20",
-        "973": "UTM22RGFG95",
-        "974": "RGR92UTM40S",
-        "976": "RGM04UTM38S",
-        "977": "RGAF09UTM20",
-        "978": "RGAF09UTM20",
-    }
-
     def __init__(self, config, logger = None):
         Analyser_Merge_Network.__init__(self, config, logger)
         self.def_class_missing_official(item = 7170, id = 13, level = 2, tags = ['merge', 'highway', 'fix:survey', 'fix:imagery'],
@@ -44,15 +34,12 @@ def __init__(self, config, logger = None):
         dep_code = config.options.get('dep_code') or config.options.get('country').split('-')[1]
         if len(str(dep_code)) == 2:
             dep_code = f"0{dep_code}"
-        proj = self.dep_proj.get(str(dep_code), "LAMB93")
         self.init(
             "https://ign.fr",
             "IGN-troncon_de_route",
-            GPKG(SourceIGN(attribution = "IGN",
-                    fileUrl = f"http://files.opendatarchives.fr/professionnels.ign.fr/bdtopo/latest/BDTOPO_3-0_TOUSTHEMES_GPKG_{proj}_D{dep_code}_2022-06-15.7z",
-                    extract = f"BDTOPO_3-0_TOUSTHEMES_GPKG_{proj}_D{dep_code}_2022-06-15/BDTOPO/1_DONNEES_LIVRAISON_2022-06-00173/BDT_3-0_GPKG_{proj}_D{dep_code}-ED2022-06-15/BDT_3-0_GPKG_{proj}_D{dep_code}-ED2022-06-15.gpkg"),
+            GPKG(SourceIGN(dep_code = config.options.get('dep_code') or config.options.get('country').split('-')[1]),
                 layer = "troncon_de_route",
-                fields = ['importance', 'fictif', 'etat_de_l_objet', 'nature', 'nom_1_gauche', 'nom_1_droite', 'nom_2_gauche', 'nom_2_droite']),
+                fields = ['importance', 'fictif', 'etat_de_l_objet', 'nature', 'nom_collaboratif_gauche', 'nom_collaboratif_droite', 'alias_gauche', 'alias_droit']),
             Load('geom',
                 table_name = 'road_fr_' + config.options['country'].replace("-", "_"),
                 # Ignore : 'Chemin', 'Bac ou liaison maritime', 'Sentier'
@@ -68,7 +55,7 @@ def __init__(self, config, logger = None):
                     'importance': ['1', '2', '3', '4', '5'],
                     'fictif': 'false',
                     'etat_de_l_objet': 'En service',
-                    'nom_1_gauche': {'like': '_%'},
+                    'nom_collaboratif_gauche': {'like': '_%'},
                     # 'Bretelle', 'Rond-point', 'Route à 1 chaussée', 'Route à 2 chaussées', 'Type autoroutier', 'Piste cyclable'
                     'nature': ['Escalier', 'Route empierrée'] }] ),
             ConflateNetwork(
@@ -80,4 +67,4 @@ def __init__(self, config, logger = None):
                 mapping = Mapping(
                     static1 = {'highway': 'road'},
                     static2 = {'source': self.source},
-                    text = lambda tags, fields: {'en': ', '.join(filter(lambda x: x and x != 'None', set([fields['nature'], fields['nom_1_gauche'], fields['nom_1_droite'], fields['nom_2_gauche'], fields['nom_2_droite']]) ))} )))
+                    text = lambda tags, fields: {'en': ', '.join(filter(lambda x: x and x != 'None', set([fields['nature'], fields['nom_collaboratif_gauche'], fields['nom_collaboratif_droite'], fields['alias_gauche'], fields['alias_droit']]) ))} )))
diff --git a/analysers/analyser_merge_water_FR.py b/analysers/analyser_merge_water_FR.py
@@ -125,8 +125,8 @@ def __init__(self, config, error_file, logger, item, id, level, title, select, t
         self.init(
             "https://ign.fr",
             "IGN-Détail Hydrographique",
-            GPKG(SourceIGN(attribution = "IGN", gzip = True,
-                    fileUrl = "http://files.opendatarchives.fr/professionnels.ign.fr/bdtopo/latest/geopackage/detail_hydrographique.gpkg.gz")),
+            GPKG(SourceIGN(dep_code = config.options.get('dep_code') or config.options.get('country').split('-')[1]),
+                layer = "detail_hydrographique"),
             LoadGeomCentroid(
                 select = select),
             Conflate(

diff --git a/osmose_config.py b/osmose_config.py
@@ -282,6 +282,12 @@ class gen(default_country):
     'osmosis_highway_zone',
     'merge_milestone_FR_metropole',
     'merge_shop_FR',
+    'merge_cemetery_FR',
+    'merge_man_made_FR',
+    'merge_poi_FR',
+    'merge_natural_FR',
+    'merge_reservoir_FR',
+    'merge_water_FR',
     'merge_road_FR',
 ], **{'addr:city-admin_level': '8,9'})
 
@@ -611,12 +617,6 @@ class gen(default_country):
 france_local_db.analyser["merge_charging_station_FR"] = "xxx"
 france_local_db.analyser["merge_parking_FR_BNLS"] = "xxx"
 france_local_db.analyser["merge_tourism_FR"] = "xxx"
-france_local_db.analyser["merge_cemetery_FR"] = "xxx"
-france_local_db.analyser["merge_man_made_FR"] = "xxx"
-france_local_db.analyser["merge_poi_FR"] = "xxx"
-france_local_db.analyser["merge_natural_FR"] = "xxx"
-france_local_db.analyser["merge_reservoir_FR"] = "xxx"
-france_local_db.analyser["merge_water_FR"] = "xxx"
 france_local_db.analyser["merge_defibrillators_FR"] = "xxx"
 france_local_db.analyser["merge_defibrillators_FR_aedmap"] = "xxx"