Merge pull request #108 from inab/full_circle

Full circle 1.0.0b0

CITATION.cff

@@ -27,5 +27,5 @@ message: "If you use this software, please cite it using these metadata."
 repository-code: "https://github.com/inab/WfExS-backend"
 type: software
 title: "WfExS-backend"
-version: 1.0.0a2
-date-released: "2024-07-28"
+version: 1.0.0b0
+date-released: "2024-08-07"

mypy-requirements.txt

@@ -12,6 +12,7 @@ types-mypy-extensions
 types-openpyxl
 types-paramiko
 types-pluggy
+types-psutil
 types-pyxdg
 types-PyYAML
 types-requests

requirements.txt

@@ -36,3 +36,5 @@ aiohttp
 pyld
 rdflib >= 7.0.0 ; python_version >= '3.8'
 rdflib < 7.0.0 ; python_version < '3.8'
+psutil
+RWFileLock >= 0.2.2

tests/core/data/test-hello-cwl.wfex.stage → tests/core/data/test-hello-cwl-docker.wfex.stage

@@ -1,6 +1,7 @@
 workflow_id: github:inab/hello-workflows/b0afc5871c6fdbd66576fcc5a3813ea49aca5104/cwl/hello-workflow.cwl
 workflow_config:
   secure: false
+  containerType: docker
 # All the inputs must be URLs or CURIEs from identifiers.org
 params:
   an_input:

tests/core/data/test-hello-cwl-singularity.wfex.stage

@@ -0,0 +1,14 @@
+workflow_id: github:inab/hello-workflows/b0afc5871c6fdbd66576fcc5a3813ea49aca5104/cwl/hello-workflow.cwl
+workflow_config:
+  secure: false
+  containerType: singularity
+# All the inputs must be URLs or CURIEs from identifiers.org
+params:
+  an_input:
+    c-l-a-s-s: File
+    url: github:inab/hello-workflows/b0afc5871c6fdbd66576fcc5a3813ea49aca5104/cwl/hello.yml
+environment:
+  SECRET_VARIABLE: "The secret content"
+outputs:
+  hello_output:
+    c-l-a-s-s: File

tests/core/data/test-hello-nxf-docker.wfex.stage

@@ -0,0 +1,20 @@
+workflow_id: github:inab/hello-workflows/26b45a412bb91e05c230a39fd268e65386c59d48/nextflow
+workflow_config:
+  secure: false
+  containerType: docker
+# All the inputs must be URLs or CURIEs from identifiers.org
+params:
+  an_input:
+    c-l-a-s-s: File
+    url: github:inab/hello-workflows/840236984ef4ea967b62b8a2dac1da22f7766b29/nextflow/singularity-params.config
+  output_dir:
+    c-l-a-s-s: Directory
+    # This hint tells that this parameter is the output directory
+    autoFill: true
+    autoPrefix: true
+environment:
+  SECRET_VARIABLE: "The secret content"
+outputs:
+  hello_output:
+    c-l-a-s-s: File
+    glob: "output_dir/*"

tests/core/data/test-hello-nxf-singularity.wfex.stage

@@ -0,0 +1,20 @@
+workflow_id: github:inab/hello-workflows/26b45a412bb91e05c230a39fd268e65386c59d48/nextflow
+workflow_config:
+  secure: false
+  containerType: singularity
+# All the inputs must be URLs or CURIEs from identifiers.org
+params:
+  an_input:
+    c-l-a-s-s: File
+    url: github:inab/hello-workflows/840236984ef4ea967b62b8a2dac1da22f7766b29/nextflow/singularity-params.config
+  output_dir:
+    c-l-a-s-s: Directory
+    # This hint tells that this parameter is the output directory
+    autoFill: true
+    autoPrefix: true
+environment:
+  SECRET_VARIABLE: "The secret content"
+outputs:
+  hello_output:
+    c-l-a-s-s: File
+    glob: "output_dir/*"

tests/core/data/test-hello-nxf-too-long-docker-fails.wfex.stage

@@ -0,0 +1,20 @@
+workflow_id: github:inab/hello-workflows/26b45a412bb91e05c230a39fd268e65386c59d48/nextflow-too-long
+workflow_config:
+  secure: false
+  containerType: docker
+# All the inputs must be URLs or CURIEs from identifiers.org
+params:
+  an_input:
+    c-l-a-s-s: File
+    url: github:inab/hello-workflows/840236984ef4ea967b62b8a2dac1da22f7766b29/nextflow/singularity-params.config
+  output_dir:
+    c-l-a-s-s: Directory
+    # This hint tells that this parameter is the output directory
+    autoFill: true
+    autoPrefix: true
+environment:
+  SECRET_VARIABLE: "The secret content"
+outputs:
+  hello_output:
+    c-l-a-s-s: File
+    glob: "output_dir/*"

tests/core/data/test-hello-nxf-too-long-docker-fixed.wfex.stage

@@ -0,0 +1,22 @@
+workflow_id: github:inab/hello-workflows/26b45a412bb91e05c230a39fd268e65386c59d48/nextflow-too-long
+workflow_config:
+  secure: false
+  containerType: docker
+  nextflow:
+    maxTaskDuration: "1m"
+# All the inputs must be URLs or CURIEs from identifiers.org
+params:
+  an_input:
+    c-l-a-s-s: File
+    url: github:inab/hello-workflows/840236984ef4ea967b62b8a2dac1da22f7766b29/nextflow/singularity-params.config
+  output_dir:
+    c-l-a-s-s: Directory
+    # This hint tells that this parameter is the output directory
+    autoFill: true
+    autoPrefix: true
+environment:
+  SECRET_VARIABLE: "The secret content"
+outputs:
+  hello_output:
+    c-l-a-s-s: File
+    glob: "output_dir/*"

tests/core/data/test-sleephello-cwl-singularity.wfex.stage

@@ -0,0 +1,15 @@
+workflow_id: github:inab/hello-workflows/af6f6c3946072994761bd435e9f36b46b9a9f2e9/cwl/sleephello-workflow.cwl
+workflow_config:
+  secure: false
+  containerType: singularity
+# All the inputs must be URLs or CURIEs from identifiers.org
+params:
+  an_input:
+    c-l-a-s-s: File
+    url: github:inab/hello-workflows/af6f6c3946072994761bd435e9f36b46b9a9f2e9/cwl/sleephello.yml
+  sleep_time: 10
+environment:
+  SECRET_VARIABLE: "The secret content"
+outputs:
+  sleephello_output:
+    c-l-a-s-s: File

tests/core/test_wfexsbackend.py

@@ -30,12 +30,27 @@
 if TYPE_CHECKING:
     from typing import (
         Optional,
+        Sequence,
         Tuple,
+        Type,
+        Union,
     )
     from wfexs_backend.wfexs_backend import (
         WfExSConfigBlock,
     )
 
+import yaml
+
+# We have preference for the C based loader and dumper, but the code
+# should fallback to default implementations when C ones are not present
+YAMLLoader: "Type[Union[yaml.Loader, yaml.CLoader]]"
+YAMLDumper: "Type[Union[yaml.Dumper, yaml.CDumper]]"
+try:
+    from yaml import CLoader as YAMLLoader, CDumper as YAMLDumper
+except ImportError:
+    from yaml import Loader as YAMLLoader, Dumper as YAMLDumper
+
+
 from wfexs_backend.wfexs_backend import WfExSBackend
 from wfexs_backend.workflow import WF
 
@@ -57,6 +72,11 @@ def test_wfexsbackend_bootstrap(
         config_directory=tmppath,
         key_prefix="crypt4gh",
     )
+
+    local_path = tmppath / "test-local_config.yaml"
+    with local_path.open(mode="w", encoding="utf-8") as cf:
+        yaml.dump(test_local_config, cf, Dumper=YAMLDumper)
+
     assert bootstrap_ok
     assert config_directory.is_dir()
     assert (tmppath / "WORKDIRS").is_dir()
@@ -105,18 +125,30 @@ def test_wfexsbackend_list_workflow_engines(tmppath: "pathlib.Path") -> "None":
 
 
 WORKFLOW_TESTBED = pytest.mark.parametrize(
-    ["stage_file", "context_file"],
+    ["stage_file", "context_file", "should_fail"],
     [
-        ("test-hello-cwl.wfex.stage", None),
-        ("test-hello-cwl-secure.wfex.stage", None),
+        ("test-hello-cwl-singularity.wfex.stage", None, None),
+        ("test-hello-cwl-docker.wfex.stage", None, None),
+        ("test-hello-cwl-secure.wfex.stage", None, None),
+        ("test-hello-nxf-docker.wfex.stage", None, None),
+        ("test-hello-nxf-singularity.wfex.stage", None, None),
+        (
+            "test-hello-nxf-too-long-docker-fails.wfex.stage",
+            None,
+            ["test_workflow_offline_exec"],
+        ),
+        ("test-hello-nxf-too-long-docker-fixed.wfex.stage", None, None),
     ],
 )
 
 
 @pytest.mark.filterwarnings("ignore:.*:pytest.PytestReturnNotNoneWarning")
 @WORKFLOW_TESTBED
 def test_wfexsbackend_stage(
-    tmppath: "pathlib.Path", stage_file: "str", context_file: "Optional[str]"
+    tmppath: "pathlib.Path",
+    stage_file: "str",
+    context_file: "Optional[str]",
+    should_fail: "Optional[Sequence[str]]",
 ) -> "WF":
     wfBackend = test_wfexsbackend_init(tmppath)