Merge branch 'develop' into feature/faster-docker

.dockerignore

@@ -1,2 +1,10 @@
 .venv
-skaffold.env
+skaffold.env
+.git
+.github
+./polytope_server.egg-info
+**/.git
+./polytope-mars/tests
+./polytope/tests
+./polytope/examples
+./polytope/performance

Dockerfile

@@ -191,7 +191,7 @@ RUN set -eux \
 
 RUN set -eux \
     && apt-get update \
-    && apt install -y gribjump-client=0.5.3-gribjump
+    && apt install -y gribjump-client=0.5.4-gribjump
 
 RUN set -eux \
     ls -R /opt
@@ -289,12 +289,11 @@ RUN apt update
 RUN apt-get install -y --no-install-recommends gcc libc6-dev make gnupg2
 
 COPY ./requirements.txt /requirements.txt
-
 RUN pip install uv --user
 ENV PATH="/root/.venv/bin:/root/.local/bin:${PATH}"
 RUN uv venv /root/.venv
-
 RUN uv pip install -r requirements.txt
+RUN uv pip install geopandas==1.0.1
 
 COPY . ./polytope
 RUN set -eux \

polytope_server/common/datasource/polytope.py

@@ -21,6 +21,7 @@
 import json
 import logging
 import os
+import copy
 
 import yaml
 from polytope.utility.exceptions import PolytopeError
@@ -35,19 +36,17 @@ def __init__(self, config):
         self.type = config["type"]
         assert self.type == "polytope"
         self.match_rules = config.get("match", {})
+        self.req_single_keys = config.get("options", {}).pop("req_single_keys", [])
         self.patch_rules = config.get("patch", {})
         self.output = None
 
         # Create a temp file to store gribjump config
         self.config_file = "/tmp/gribjump.yaml"
         with open(self.config_file, "w") as f:
-            f.write(yaml.dump(self.config["gribjump_config"]))
+            f.write(yaml.dump(self.config.pop("gribjump_config")))
         self.config["datacube"]["config"] = self.config_file
         os.environ["GRIBJUMP_CONFIG_FILE"] = self.config_file
 
-        self.polytope_options = self.config.get("polytope-options", {})
-        self.polytope_mars = PolytopeMars(self.config)
-
         logging.info("Set up gribjump")
 
     def get_type(self):
@@ -60,13 +59,32 @@ def retrieve(self, request):
         r = yaml.safe_load(request.user_request)
         logging.info(r)
 
+        # Set the "pre-path" for this request
+        pre_path = {}
+        for k,v in r.items():
+            if k in self.req_single_keys:
+                if isinstance(v, list):
+                    v = v[0]
+                pre_path[k] = v
+
+        polytope_mars_config = copy.deepcopy(self.config)
+        polytope_mars_config["options"]["pre_path"] = pre_path
+
+
+        polytope_mars = PolytopeMars(
+            polytope_mars_config,
+            log_context= {
+                "user": request.user.realm + ':' + request.user.username,
+                "id": request.id,
+            })
+
+
         try:
-            self.output = self.polytope_mars.extract(r)
+            self.output = polytope_mars.extract(r)
             self.output = json.dumps(self.output).encode("utf-8")
         except PolytopeError as e:
-            self.output = json.dumps({"error": str(e.message)}).encode("utf-8")
-            raise e
-        # logging.info(self.output)
+            raise Exception("Polytope Feature Extraction Error: {}".format(e.message))
+
         return True
 
     def result(self, request):
@@ -77,6 +95,10 @@ def match(self, request):
 
         r = yaml.safe_load(request.user_request) or {}
 
+        # Check that there is a feature specified in the request
+        if "feature" not in r:
+            raise Exception("Request does not contain expected key 'feature'")
+
         for k, v in self.match_rules.items():
             # Check that all required keys exist
             if k not in r:
@@ -88,9 +110,14 @@ def match(self, request):
             if r[k] not in v:
                 raise Exception("got {} : {}, but expected one of {}".format(k, r[k], v))
 
-            # Finally check that there is a feature specified in the request
-            if "feature" not in r:
-                raise Exception("Request does not contain expected key 'feature'")
+        # Check that there is only one value if required
+        for k, v in r.items():    
+            if k in self.req_single_keys:
+                v = [v] if isinstance(v, str) else v
+                if len(v) > 1:
+                    raise Exception("Expected only one value for key {}".format(k))
+                elif len(v) == 0:
+                    raise Exception("Expected a value for key {}".format(k))
 
     def destroy(self, request) -> None:
         pass

polytope_server/worker/worker.py

@@ -245,7 +245,7 @@ def process_request(self, request):
                 request.url = self.staging.create(id, datasource.result(request), datasource.mime_type())
 
         except Exception as e:
-            request.user_message += f"Failed to finalize request: {str(e)}"
+            request.user_message += f"Failed to finalize request: [{str(type(e))}] {str(e)}"
             logging.info(request.user_message, extra={"request_id": id})
             logging.exception("Failed to finalize request", extra={"request_id": id, "exception": str(e)})
             raise

requirements.txt

@@ -1,6 +1,6 @@
 attrdict==2.0.1
 boto3==1.35.19
-covjsonkit==0.0.21
+covjsonkit==0.0.22
 deepmerge==2.0
 docker==7.1.0
 ecmwf-api-client==1.6.3
@@ -19,7 +19,7 @@ ldap3==2.9.1
 Markdown==3.7
 minio==7.2.8
 pika==1.3.2
-polytope-mars==0.0.9
+polytope-mars==0.0.10
 polytope-python==1.0.7
 py-bcrypt==0.4
 pykwalify==1.8.0

skaffold.yaml

@@ -18,7 +18,8 @@ build:
     concurrency: 1
 
   tagPolicy:
-    gitCommit: {}
+    gitCommit:
+      ignoreChanges: true
 
   artifacts:
 
@@ -27,7 +28,7 @@ build:
     docker:
       target: polytope-common
       buildArgs:
-        developer_mode: False
+        developer_mode: "{{ .developer_mode }}"
 
   # Worker with all clients (FDB, GribJump, MARS C, MARS CPP)
   - image: "worker"
@@ -39,7 +40,7 @@ build:
         mars_config_branch: "{{ .mars_config_branch }}"
         ssh_pub_key: "{{ .ssh_pub_key }}"
         ssh_prv_key: "{{ .ssh_prv_key }}"
-        developer_mode: True
+        developer_mode: "{{ .developer_mode }}"
         mars_client_cpp_version: 6.99.3.0
         mars_base_c: mars-base-c
         mars_base_cpp: mars-base-cpp