Merge pull request #82 from inab/lazy_loading_magic

Added Python lazy loading capabilities to ease ReadTheDocs documentation generation.

WfExS-config-replicator.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # SPDX-License-Identifier: Apache-2.0
-# Copyright 2020-2023 Barcelona Supercomputing Center (BSC), Spain
+# Copyright 2020-2024 Barcelona Supercomputing Center (BSC), Spain
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -25,53 +25,61 @@
 import sys
 
 from typing import (
-    Any,
     cast,
-    Mapping,
-    MutableMapping,
-    MutableSequence,
-    Optional,
-    Sequence,
-    Tuple,
-    Type,
-    Union,
+    TYPE_CHECKING,
 )
 
+
 import openpyxl
 import xlrd2  # type:ignore
 import yaml
 
+if TYPE_CHECKING:
+    from typing import (
+        Any,
+        Mapping,
+        MutableMapping,
+        MutableSequence,
+        Optional,
+        Sequence,
+        Tuple,
+        Type,
+        Union,
+    )
+
+    from openpyxl.worksheet.worksheet import Worksheet
+
 # We have preference for the C based loader and dumper, but the code
 # should fallback to default implementations when C ones are not present
-YAMLDumper: Type[Union[yaml.Dumper, yaml.CDumper]]
+YAMLDumper: "Type[Union[yaml.Dumper, yaml.CDumper]]"
 try:
     from yaml import CDumper as YAMLDumper
 except ImportError:
     from yaml import Dumper as YAMLDumper
 
 
-def loadWorkflowConfig(workflowConfigFilename: str) -> Mapping[str, Any]:
+def loadWorkflowConfig(workflowConfigFilename: "str") -> "Mapping[str, Any]":
     with open(workflowConfigFilename, mode="r", encoding="utf-8") as wcf:
         workflow_config = yaml.safe_load(wcf)
 
-        return cast(Mapping[str, Any], workflow_config)
+        return cast("Mapping[str, Any]", workflow_config)
 
 
-def loadXLSParams(paramsFilename: str) -> Sequence[Mapping[str, Any]]:
+def loadXLSParams(paramsFilename: "str") -> "Sequence[Mapping[str, Any]]":
     paramsArray = []
 
     wb = xlrd2.open_workbook(filename=paramsFilename)
 
     for sheet in wb.sheets():
         gotHeader = False
-        header: MutableSequence[Tuple[str, int]] = []
+        header: "MutableSequence[Tuple[str, int]]" = []
         for row in sheet.get_rows():
             if row is None:
                 continue
 
             # Either get the header or the data
             if gotHeader:
-                params: MutableMapping[str, MutableSequence[Any]] = dict()
+                params: "MutableMapping[str, MutableSequence[Any]]" = dict()
                 for headerName, iCell in header:
                     theVal = row[iCell].value
                     params.setdefault(headerName, []).append(theVal)
@@ -91,13 +99,16 @@ def loadXLSParams(paramsFilename: str) -> Sequence[Mapping[str, Any]]:
     return paramsArray
 
 
-def loadXLSXParams(paramsFilename: str) -> Sequence[Mapping[str, Any]]:
+def loadXLSXParams(paramsFilename: "str") -> "Sequence[Mapping[str, Any]]":
     paramsArray = []
 
     wb = openpyxl.load_workbook(filename=paramsFilename, data_only=True, read_only=True)
     sheets = wb.worksheets
 
-    for sheet in sheets:
+    for sheet_raw in sheets:
+        # This is needed to avoid a mypy error fired
+        # by latest openpyxl annotations
+        sheet = cast("Worksheet", sheet_raw)  # type: ignore[redundant-cast]
         gotHeader = False
         headerMap: "MutableMapping[int,str]" = {}
         for cells_in_row in sheet.iter_rows():
@@ -123,7 +134,7 @@ def loadXLSXParams(paramsFilename: str) -> Sequence[Mapping[str, Any]]:
     return paramsArray
 
 
-def loadCSVParams(paramsFilename: str) -> Sequence[Mapping[str, Any]]:
+def loadCSVParams(paramsFilename: "str") -> "Sequence[Mapping[str, Any]]":
     paramsArray = []
 
     with open(paramsFilename, mode="rb") as cR:
@@ -182,13 +193,13 @@ def loadCSVParams(paramsFilename: str) -> Sequence[Mapping[str, Any]]:
 }
 
 
-def loadParamsFiles(paramsFilenames: Sequence[str]) -> Sequence[Mapping[str, Any]]:
+def loadParamsFiles(paramsFilenames: "Sequence[str]") -> "Sequence[Mapping[str, Any]]":
     """
     This method returns a list of dictionaries
     being each dictionary a set of values to substitute
     into the workflow configuration template
     """
-    paramsArray: MutableSequence[Mapping[str, Any]] = list()
+    paramsArray: "MutableSequence[Mapping[str, Any]]" = list()
 
     if not mimetypes.inited:
         mimetypes.init()
@@ -227,14 +238,14 @@ def loadParamsFiles(paramsFilenames: Sequence[str]) -> Sequence[Mapping[str, Any
 
 
 def applyValuesToTemplate(
-    workflow_config_template: Mapping[str, Any], params: Mapping[str, Any]
-) -> Mapping[str, Any]:
+    workflow_config_template: "Mapping[str, Any]", params: "Mapping[str, Any]"
+) -> "Mapping[str, Any]":
     """
     The parameters are set using as template the input
     """
 
     workflow_config = cast(
-        MutableMapping[str, Any], copy.deepcopy(workflow_config_template)
+        "MutableMapping[str, Any]", copy.deepcopy(workflow_config_template)
     )
 
     workflow_config.setdefault("params", {})
@@ -277,12 +288,12 @@ def applyValuesToTemplate(
 
 
 def writeWorkflowConfigVariations(
-    workflow_config_template: Mapping[str, Any],
-    paramsArray: Sequence[Mapping[str, Any]],
-    fnameTemplate: str,
-    destdir: str = ".",
-    paramSymbolTemplate: Optional[str] = None,
-) -> Sequence[str]:
+    workflow_config_template: "Mapping[str, Any]",
+    paramsArray: "Sequence[Mapping[str, Any]]",
+    fnameTemplate: "str",
+    destdir: "str" = ".",
+    paramSymbolTemplate: "Optional[str]" = None,
+) -> "Sequence[str]":
     # Creating the directory, in case it does not exist
     destdir = os.path.abspath(destdir)
     os.makedirs(destdir, exist_ok=True)
@@ -371,7 +382,7 @@ def writeWorkflowConfigVariations(
     if args.params_files:
         paramsArray = loadParamsFiles(args.params_files)
     else:
-        params: MutableMapping[str, Any] = {}
+        params: "MutableMapping[str, Any]" = {}
         paramsArray = [params]
         for param, value in args.inline_params:
             params.setdefault(param, []).append(value)

wfexs_backend/abstract_docker_container.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # SPDX-License-Identifier: Apache-2.0
-# Copyright 2020-2023 Barcelona Supercomputing Center (BSC), Spain
+# Copyright 2020-2024 Barcelona Supercomputing Center (BSC), Spain
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -34,7 +34,11 @@
     TYPE_CHECKING,
 )
 
-import magic
+from .utils.misc import lazy_import
+
+magic = lazy_import("magic")
+# import magic
+
 import pgzip
 
 from .common import (

wfexs_backend/ro_crate.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # SPDX-License-Identifier: Apache-2.0
-# Copyright 2020-2023 Barcelona Supercomputing Center (BSC), Spain
+# Copyright 2020-2024 Barcelona Supercomputing Center (BSC), Spain
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -94,7 +94,11 @@
 import urllib.parse
 import uuid
 
-import magic
+from .utils.misc import lazy_import
+
+magic = lazy_import("magic")
+# import magic
+
 from rfc6920.methods import extract_digest
 import rocrate.model.entity
 import rocrate.model.dataset

wfexs_backend/utils/contents.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # SPDX-License-Identifier: Apache-2.0
-# Copyright 2020-2023 Barcelona Supercomputing Center (BSC), Spain
+# Copyright 2020-2024 Barcelona Supercomputing Center (BSC), Spain
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,7 +27,11 @@
 )
 
 import data_url
-import magic
+
+from .misc import lazy_import
+
+magic = lazy_import("magic")
+# import magic
 
 from ..common import (
     ContentKind,

wfexs_backend/utils/misc.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 
 # SPDX-License-Identifier: Apache-2.0
-# Copyright 2020-2023 Barcelona Supercomputing Center (BSC), Spain
+# Copyright 2020-2024 Barcelona Supercomputing Center (BSC), Spain
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -29,6 +29,10 @@
 )
 
 if TYPE_CHECKING:
+    from types import (
+        ModuleType,
+    )
+
     from typing import (
         Any,
         Iterator,
@@ -268,3 +272,26 @@ def config_validate(
         raise ConfigValidationException(
             f"FATAL ERROR: corrupted schema {relSchemaFile}. Reason: {e}"
         )
+
+
+import importlib.util
+import sys
+
+
+def lazy_import(name: "str") -> "ModuleType":
+    module = sys.modules.get(name)
+    if module is None:
+        spec = importlib.util.find_spec(name)
+        if spec is not None and spec.loader is not None:
+            loader = importlib.util.LazyLoader(spec.loader)
+            spec.loader = loader
+
+            module = importlib.util.module_from_spec(spec)
+            sys.modules[name] = module
+
+            loader.exec_module(module)
+
+    if module is None:
+        raise ModuleNotFoundError(f"No module named '{name}'")
+
+    return module

wfexs_backend/wfexs_backend.py

@@ -53,7 +53,10 @@
 import crypt4gh.keys.kdf
 import crypt4gh.keys.c4gh
 
-import magic
+from .utils.misc import lazy_import
+
+magic = lazy_import("magic")
+# import magic
 
 from .common import (
     AbstractWfExSException,