feat(cli): add test command (#724)

Closes #719

AUTHORS.md

@@ -3,6 +3,7 @@
 The list of contributors in alphabetical order:
 
 - [Adelina Lintuluoto](https://orcid.org/0000-0002-0726-1452)
+- [Alastair Lyall](https://orcid.org/0009-0000-4955-8935)
 - [Anton Khodak](https://orcid.org/0000-0003-3263-4553)
 - [Audrius Mecionis](https://orcid.org/0000-0002-3759-1663)
 - [Camila Diaz](https://orcid.org/0000-0001-5543-797X)

docs/cmd_list.txt

@@ -55,3 +55,6 @@ Secret management commands:
   secrets-add     Add secrets from literal string or from file.
   secrets-delete  Delete user secrets by name.
   secrets-list    List user secrets.
+
+Workflow run test commands:
+  test  Test workflow execution, based on a given Gherkin file.

reana_client/cli/__init__.py

@@ -11,7 +11,15 @@
 import sys
 
 import click
-from reana_client.cli import files, ping, quotas, retention_rules, secrets, workflow
+from reana_client.cli import (
+    files,
+    ping,
+    quotas,
+    retention_rules,
+    secrets,
+    test,
+    workflow,
+)
 from reana_client.utils import get_api_url
 from urllib3 import disable_warnings
 
@@ -45,6 +53,7 @@ class ReanaCLI(click.Group):
         files.files_group,
         retention_rules.retention_rules_group,
         secrets.secrets_group,
+        test.test_group,
     ]
 
     def __init__(self, name=None, commands=None, **attrs):

reana_client/cli/files.py

@@ -326,10 +326,11 @@ def upload_files(  # noqa: C901
                 msg_type="error",
             )
             sys.exit(1)
-
+        filenames = []
+        if reana_spec.get("tests"):
+            for f in reana_spec["tests"].get("files") or []:
+                filenames.append(os.path.join(os.getcwd(), f))
         if reana_spec.get("inputs"):
-            filenames = []
-
             # collect all files in input.files
             for f in reana_spec["inputs"].get("files") or []:
                 # check for directories in files
@@ -340,7 +341,6 @@ def upload_files(  # noqa: C901
                     )
                     sys.exit(1)
                 filenames.append(os.path.join(os.getcwd(), f))
-
             # collect all files in input.directories
             files_from_directories = []
             directories = reana_spec["inputs"].get("directories") or []

reana_client/cli/test.py

@@ -0,0 +1,169 @@
+# -*- coding: utf-8 -*-
+#
+# This file is part of REANA.
+# Copyright (C) 2024 CERN.
+#
+# REANA is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+"""REANA client test commands."""
+
+import sys
+import click
+import logging
+import traceback
+import time
+from reana_client.cli.utils import add_access_token_options, check_connection
+from reana_commons.gherkin_parser.parser import (
+    parse_and_run_tests,
+    AnalysisTestStatus,
+)
+from reana_commons.gherkin_parser.data_fetcher import DataFetcherBase
+from reana_commons.gherkin_parser.errors import FeatureFileError
+from reana_client.printer import display_message
+from reana_client.api.client import (
+    list_files,
+    get_workflow_disk_usage,
+    get_workflow_logs,
+    get_workflow_status,
+    get_workflow_specification,
+    download_file,
+)
+from reana_client.cli.utils import add_workflow_option
+
+
+class DataFetcherClient(DataFetcherBase):
+    """Implementation of the DataFetcherBase using reana_client.api.client methods."""
+
+    def __init__(self, access_token):
+        """Initialize DataFetcherClient with access token."""
+        self.access_token = access_token
+
+    def list_files(self, workflow, file_name=None, page=None, size=None, search=None):
+        """Return the list of files for a given workflow workspace."""
+        return list_files(workflow, self.access_token, file_name, page, size, search)
+
+    def get_workflow_disk_usage(self, workflow, parameters):
+        """Display disk usage workflow."""
+        return get_workflow_disk_usage(workflow, parameters, self.access_token)
+
+    def get_workflow_logs(self, workflow, steps=None, page=None, size=None):
+        """Get logs from a workflow engine, use existing API function."""
+        return get_workflow_logs(workflow, self.access_token, steps, page, size)
+
+    def get_workflow_status(self, workflow):
+        """Get of a previously created workflow."""
+        return get_workflow_status(workflow, self.access_token)
+
+    def get_workflow_specification(self, workflow):
+        """Get specification of previously created workflow."""
+        return get_workflow_specification(workflow, self.access_token)
+
+    def download_file(self, workflow, file_path):
+        """Download the requested file if it exists."""
+        return download_file(workflow, file_path, self.access_token)
+
+
+@click.group(help="Workflow run test commands")
+def test_group():
+    """Workflow run test commands."""
+
+
+@test_group.command("test")
+@click.option(
+    "-n",
+    "--test-files",
+    multiple=True,
+    default=None,
+    help="Gherkin file for testing properties of a workflow execution. Overrides files in reana.yaml if provided.",
+)
+@click.pass_context
+@add_access_token_options
+@check_connection
+@add_workflow_option
+def test(ctx, workflow, test_files, access_token):
+    r"""
+    Test workflow execution, based on a given Gherkin file.
+
+    Gherkin files can be specified in the reana specification file (reana.yaml),
+    or by using the ``-n`` option.
+
+    The ``test`` command allows for testing of a workflow execution,
+    by assessing whether it meets certain properties specified in a
+    chosen gherkin file.
+
+    Example:
+        $ reana-client test -w myanalysis -n test_analysis.feature
+        $ reana-client test -w myanalysis
+        $ reana-client test -w myanalysis -n test1.feature -n test2.feature
+    """
+    start_time = time.time()
+    try:
+        workflow_status = get_workflow_status(
+            workflow=workflow, access_token=access_token
+        )
+        status = workflow_status["status"]
+        workflow_name = workflow_status["name"]
+    except Exception as e:
+        logging.debug(traceback.format_exc())
+        logging.debug(str(e))
+        display_message(f"Could not find workflow ``{workflow}``.", msg_type="error")
+        sys.exit(1)
+
+    if status != "finished":
+        display_message(
+            f"``{workflow}`` is {status}. It must be finished to run tests.",
+            msg_type="error",
+        )
+        sys.exit(1)
+
+    if not test_files:
+        reana_specification = get_workflow_specification(workflow, access_token)
+        try:
+            test_files = reana_specification["specification"]["tests"]["files"]
+        except KeyError as e:
+            logging.debug(traceback.format_exc())
+            logging.debug(str(e))
+            display_message(
+                "No test files specified in reana.yaml and no -n option provided.",
+                msg_type="error",
+            )
+            sys.exit(1)
+
+    passed = 0
+    failed = 0
+    data_fetcher = DataFetcherClient(access_token)
+    for test_file in test_files:
+        click.echo("\n", nl=False)
+        display_message(f'Testing file "{test_file}"...', msg_type="info")
+        try:
+            results = parse_and_run_tests(test_file, workflow_name, data_fetcher)
+        except FileNotFoundError as e:
+            logging.debug(traceback.format_exc())
+            logging.debug(str(e))
+            display_message(f"Test file {test_file} not found.", msg_type="error")
+            sys.exit(1)
+        except FeatureFileError as e:
+            logging.debug(traceback.format_exc())
+            logging.debug(str(e))
+            display_message(
+                f"Error parsing feature file {test_file}: {e}", msg_type="error"
+            )
+            sys.exit(1)
+
+        for scenario in results[1]:
+            if scenario.result == AnalysisTestStatus.failed:
+                display_message(
+                    f'Scenario "{scenario.scenario}"', msg_type="error", indented=True
+                )
+                failed += 1
+            else:
+                display_message(
+                    f'Scenario "{scenario.scenario}"', msg_type="success", indented=True
+                )
+                passed += 1
+
+    end_time = time.time()
+    duration = round(end_time - start_time)
+    click.echo(f"\n{passed} passed, {failed} failed in {duration}s")
+    if failed > 0:
+        sys.exit(1)

tests/test_cli_files.py

@@ -240,6 +240,35 @@ def test_upload_file(create_yaml_workflow_schema):
                 assert message in result.output
 
 
+def test_upload_file_with_test_files_from_spec(
+    get_workflow_specification_with_directory,
+):
+    """Test upload file with test files from the specification, not from the command line."""
+    reana_token = "000000"
+    file = "upload-this-test.feature"
+    env = {"REANA_SERVER_URL": "http://localhost"}
+    runner = CliRunner(env=env)
+
+    with patch(
+        "reana_client.api.client.get_workflow_specification"
+    ) as mock_specification, patch("reana_client.api.client.requests.post"):
+        with runner.isolated_filesystem():
+            with open(file, "w") as f:
+                f.write("Scenario: Test scenario")
+
+            get_workflow_specification_with_directory["specification"]["tests"] = {
+                "files": [file]
+            }
+            mock_specification.return_value = get_workflow_specification_with_directory
+            result = runner.invoke(
+                cli, ["upload", "-t", reana_token, "--workflow", "test-workflow.1"]
+            )
+            assert result.exit_code == 0
+            assert (
+                "upload-this-test.feature was successfully uploaded." in result.output
+            )
+
+
 def test_upload_file_respect_gitignore(
     get_workflow_specification_with_directory,
 ):