From 2dba1570a1c55615f02475369f48ffb7f73d12ee Mon Sep 17 00:00:00 2001 From: Marco Donadoni Date: Tue, 12 Sep 2023 10:29:48 +0200 Subject: [PATCH] cli: add git-upgrade-requirements Closes #741 --- reana/config.py | 16 +++++++++++ reana/reana_dev/git.py | 60 ++++++++++++++++++++++++++++++++++++++++ reana/reana_dev/utils.py | 38 +++++++++++++++++++++++++ 3 files changed, 114 insertions(+) diff --git a/reana/config.py b/reana/config.py index a0538b64..e4f02e11 100644 --- a/reana/config.py +++ b/reana/config.py @@ -114,6 +114,16 @@ ) """List of git repositories related to cluster components.""" +REPO_LIST_PYTHON_REQUIREMENTS = [ + "reana-server", + "reana-workflow-controller", + "reana-job-controller", + "reana-workflow-engine-cwl", + "reana-workflow-engine-serial", + "reana-workflow-engine-yadage", + "reana-workflow-engine-snakemake", +] +"""List of cluster components that have a Python requirements file.""" WORKFLOW_ENGINE_LIST_ALL = ["cwl", "serial", "yadage", "snakemake"] """List of supported workflow engines.""" @@ -268,6 +278,9 @@ PYTHON_VERSION_FILE = "version.py" """Python package version file.""" +PYTHON_REQUIREMENTS_FILE = "requirements.txt" +"""Python requirements file.""" + GIT_DEFAULT_BASE_BRANCH = "master" """Default git base branch we shall be working against.""" @@ -279,3 +292,6 @@ CODECOV_REANAHUB_URL = "https://codecov.io/gh/reanahub" """REANA Hub organisation Codecov URL.""" + +PYTHON_DOCKER_IMAGE = "docker.io/library/python:3.8" +"""Python docker image with same version as cluster components.""" diff --git a/reana/reana_dev/git.py b/reana/reana_dev/git.py index 75ef6242..70438ba8 100644 --- a/reana/reana_dev/git.py +++ b/reana/reana_dev/git.py @@ -19,7 +19,9 @@ COMPONENTS_USING_SHARED_MODULE_COMMONS, COMPONENTS_USING_SHARED_MODULE_DB, GIT_DEFAULT_BASE_BRANCH, + PYTHON_REQUIREMENTS_FILE, REPO_LIST_ALL, + REPO_LIST_PYTHON_REQUIREMENTS, REPO_LIST_SHARED, ) from reana.reana_dev.utils import ( @@ -33,6 +35,7 @@ run_command, select_components, update_module_in_cluster_components, + upgrade_requirements, validate_directory, ) @@ -1186,6 +1189,63 @@ def _create_commit_or_amend(components): git_push_to_origin(components) +@git_commands.command(name="git-upgrade-requirements") +@click.option( + "--component", + "-c", + multiple=True, + default=["CLUSTER"], + help="Which components? [shortname|name|.|CLUSTER|ALL]", +) +@click.option( + "--exclude-components", + default="", + help="Which components to exclude? [c1,c2,c3]", +) +@click.pass_context +def git_upgrade_requirements(ctx, component, exclude_components): # noqa: D301 + """Upgrade Python dependencies for selected components. + + \b + :param components: The option ``component`` can be repeated. The value may + consist of: + * (1) standard component name such as + 'reana-workflow-controller'; + * (2) short component name such as 'r-w-controller'; + * (3) special value '.' indicating component of the + current working directory; + * (4) special value 'CLUSTER' that will expand to + cover all REANA cluster components [default]; + * (5) special value 'CLIENT' that will expand to + cover all REANA client components; + * (6) special value 'DEMO' that will expand + to include several runable REANA demo examples; + * (7) special value 'ALL' that will expand to include + all REANA repositories. + :param exclude_components: List of components to exclude. + :type component: str + :type exclude_components: str + """ + if exclude_components: + exclude_components = exclude_components.split(",") + components = select_components(component, exclude_components) + components = sorted(set(components).intersection(REPO_LIST_PYTHON_REQUIREMENTS)) + + for component in components: + if not is_feature_branch(component): + display_message( + f"Current branch is {GIT_DEFAULT_BASE_BRANCH}. " + "Please switch to a feature branch.", + component, + ) + sys.exit(1) + if upgrade_requirements(component): + run_command(f"git add {PYTHON_REQUIREMENTS_FILE}", component) + run_command( + 'git commit -m "installation: bump all dependencies"', component + ) + + @click.option( "--component", "-c", diff --git a/reana/reana_dev/utils.py b/reana/reana_dev/utils.py index 6991ec72..108c00a5 100644 --- a/reana/reana_dev/utils.py +++ b/reana/reana_dev/utils.py @@ -32,6 +32,8 @@ HELM_VERSION_FILE, JAVASCRIPT_VERSION_FILE, OPENAPI_VERSION_FILE, + PYTHON_DOCKER_IMAGE, + PYTHON_REQUIREMENTS_FILE, PYTHON_VERSION_FILE, REPO_LIST_ALL, REPO_LIST_CLIENT, @@ -578,6 +580,42 @@ def update_module_in_cluster_components( ) +def upgrade_requirements(component: str) -> bool: + """Update the Python requirements file using pip-compile.""" + requirements_path = os.path.join(get_srcdir(component), PYTHON_REQUIREMENTS_FILE) + if not os.path.exists(requirements_path): + display_message(f"File {PYTHON_REQUIREMENTS_FILE} not found.", component) + return False + + pip_compile_cmd = None + with open(requirements_path) as requirements_file: + # find pip-compile command contained in requirements.txt + for line in requirements_file: + stripped_line = line.lstrip("#").strip() + if stripped_line.startswith("pip-compile"): + pip_compile_cmd = stripped_line + break + + if not pip_compile_cmd: + display_message( + f"File {PYTHON_REQUIREMENTS_FILE} does not contain a valid pip-compile command.", + component, + ) + return False + + if "annotation-style" not in pip_compile_cmd: + executable, options = pip_compile_cmd.split(maxsplit=1) + pip_compile_cmd = " ".join([executable, "--annotation-style=line", options]) + pip_compile_cmd += " -U" + + docker_cmd = ( + f"docker run --rm -it -v {get_srcdir(component)}:/code {PYTHON_DOCKER_IMAGE} " + f"bash -c 'cd /code && pip install --upgrade pip-tools setuptools pip && {pip_compile_cmd}'" + ) + run_command(docker_cmd, component) + return True + + def get_component_version_files(component, abs_path=False) -> Dict[str, str]: """Get a dictionary with all component's version files.""" version_files = {}