From a8e87316bc361228723d23f78fbebcf7f362590f Mon Sep 17 00:00:00 2001 From: Lucas PASCAL Date: Wed, 8 Nov 2023 09:42:32 +0100 Subject: [PATCH 01/10] [add] Python script checkers --- .github/workflows/fast-checks.yml | 60 +++++++++++++++++++++++++++++++ pyproject.toml | 2 ++ 2 files changed, 62 insertions(+) create mode 100644 .github/workflows/fast-checks.yml create mode 100644 pyproject.toml diff --git a/.github/workflows/fast-checks.yml b/.github/workflows/fast-checks.yml new file mode 100644 index 0000000..00c24a7 --- /dev/null +++ b/.github/workflows/fast-checks.yml @@ -0,0 +1,60 @@ +name: Fast Python checks + +on: + workflow_dispatch: + push: + branches: + - develop + - master + pull_request: + +jobs: + lint: + strategy: + fail-fast: false + matrix: + directory: + - build_and_test + - create_app_list + - output_scripts + - split_input + name: Linting + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v3 + - run: pip install flake8 + - name: Flake8 lint Python code + run: find scripts/${{ matrix.directory }} -type f -name '*.py' -exec flake8 --max-line-length=120 '{}' '+' + + mypy: + strategy: + fail-fast: false + matrix: + directory: + - build_and_test + - create_app_list + - output_scripts + - split_input + name: Type checking + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v3 + - run: pip install mypy types-requests + - name: Mypy type checking + run: mypy scripts/${{ matrix.directory }} + + misspell: + strategy: + fail-fast: false + name: Check misspellings + runs-on: ubuntu-latest + steps: + - name: Clone + uses: actions/checkout@v3 + - name: Check misspellings + uses: codespell-project/actions-codespell@v1 + with: + builtin: clear,rare + check_filenames: true diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..479d38e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.mypy] +ignore_missing_imports = true From f9b9ffacb826908d476287fa04fbeec6635ff7e6 Mon Sep 17 00:00:00 2001 From: Lucas PASCAL Date: Mon, 13 Nov 2023 11:20:38 +0100 Subject: [PATCH 02/10] [ci][fix] Comply with CI checks --- README.md | 4 +-- scripts/build_and_test/__init__.py | 0 scripts/build_and_test/build_app.py | 21 ++++++++++++---- scripts/build_and_test/device.py | 3 ++- scripts/build_and_test/main.py | 8 +++--- scripts/build_and_test/scan_app.py | 22 ++++++++++++---- scripts/build_and_test/test_app.py | 25 +++++++++++-------- scripts/build_and_test/utils.py | 9 ++++--- .../create_app_list/app_load_params_check.py | 8 +++--- .../create_app_list/app_load_params_utils.py | 6 ++--- scripts/create_app_list/gen_variant.py | 21 ++++++++++------ scripts/create_app_list/main.py | 4 +-- scripts/create_app_list/makefile_dump.py | 5 +++- scripts/create_app_list/parse_github.py | 7 +++--- scripts/create_app_list/utils.py | 3 ++- scripts/output_scripts/__init__.py | 0 scripts/output_scripts/convert.py | 13 +++++++--- scripts/output_scripts/merge.py | 12 +++++---- scripts/output_scripts/slack.py | 10 +++----- scripts/split_input/__init__.py | 0 scripts/split_input/split_input.py | 6 ++--- 21 files changed, 116 insertions(+), 71 deletions(-) create mode 100644 scripts/build_and_test/__init__.py create mode 100644 scripts/output_scripts/__init__.py create mode 100644 scripts/split_input/__init__.py diff --git a/README.md b/README.md index 5b8d2c1..9185ebb 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ file generated previously. - `scan_all.yml`: Scans for selected devices - `refresh_inputs.yml`: Check whether the input list needs updating. -To reduce CI run times, the input file is automatically splitted into 10 sub-inputs, and then all the inputs are run through a matrix strategy. +To reduce CI run times, the input file is automatically split into 10 sub-inputs, and then all the inputs are run through a matrix strategy. ### 3. Planned Improvements - **Support for ZEMU Tests** @@ -58,5 +58,5 @@ Alternatively you can run the script from the actions tab of the repo. You can view the result in the summary of the GH action: :red_circle: means a fail. :heavy_check_mark: means success, -:fast_forward: means the action was explicitely blacklisted in the input file. +:fast_forward: means the action was explicitly blacklisted in the input file. nothing: means there was no variant associated to the device when running make listvariants. diff --git a/scripts/build_and_test/__init__.py b/scripts/build_and_test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/build_and_test/build_app.py b/scripts/build_and_test/build_app.py index 3a9f86a..ec62293 100644 --- a/scripts/build_and_test/build_app.py +++ b/scripts/build_and_test/build_app.py @@ -3,8 +3,13 @@ from utils import run_cmd import os -def build_variant(target: str, sdk_path: str, variant_param: str, variant_value: str, app_build_path: - Path, extra_flags: str=""): + +def build_variant(target: str, + sdk_path: str, + variant_param: str, + variant_value: str, + app_build_path: Path, + extra_flags: str = ""): if not os.path.exists(app_build_path): print("\t=> KO") @@ -12,9 +17,11 @@ def build_variant(target: str, sdk_path: str, variant_param: str, variant_value: error = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True) if variant_param: - error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} {extra_flags}", cwd=app_build_path, no_throw=True) + error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} {extra_flags}", + cwd=app_build_path, no_throw=True) else: - error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make -j {extra_flags}", cwd=app_build_path, no_throw=True) + error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make -j {extra_flags}", + cwd=app_build_path, no_throw=True) if error: print("\t=> KO") @@ -50,7 +57,11 @@ def build_device(device: Device, variant_param: str, app_build_path: Path, sdk_p variants = app_json.get(f"variants_{device.model_name}", []) variant_output = {} if len(variants) > 0: - variant_output, error_log = build_all_variants(device.target_name, sdk_path, variant_param, variants, app_build_path) + variant_output, error_log = build_all_variants(device.target_name, + sdk_path, + variant_param, + variants, + app_build_path) return variant_output, error_log diff --git a/scripts/build_and_test/device.py b/scripts/build_and_test/device.py index 0b51ca7..e5ef5ed 100644 --- a/scripts/build_and_test/device.py +++ b/scripts/build_and_test/device.py @@ -8,8 +8,9 @@ NANOX_API_LEVEL = 5 STAX_API_LEVEL = 12 + class Device: - def __init__(self, target_name: str, model_name:str, sdk_name: str, api_level: int, enabled: bool): + def __init__(self, target_name: str, model_name: str, sdk_name: str, api_level: int, enabled: bool): self.selected = enabled self.target_name = target_name self.model_name = model_name diff --git a/scripts/build_and_test/main.py b/scripts/build_and_test/main.py index 3a1ca77..f1818ae 100644 --- a/scripts/build_and_test/main.py +++ b/scripts/build_and_test/main.py @@ -1,12 +1,13 @@ import json from pathlib import Path from argparse import ArgumentParser +from sha1 import override_sha1 + from build_app import build_all_devices from test_app import test_all_devices from scan_app import scan_all_devices from device import Devices from utils import git_setup, merge_json -from sha1 import override_sha1 SDK_NAME = "sdk" SDK_URL = "https://github.com/LedgerHQ/ledger-secure-sdk.git" @@ -35,7 +36,7 @@ parser.add_argument("--input_file", required=False, type=Path, default=Path("input_files/test_input.json")) parser.add_argument("--output_file", required=False, type=Path, default=Path("output_files/output.json")) parser.add_argument("--logs_file", required=False, type=Path, - default=Path("output_files/error_logs.txt")) + default=Path("output_files/error_logs.txt")) parser.add_argument("--workdir", required=False, type=str, default="workdir") parser.add_argument("--use_sha1_from_live", required=False, action='store_true') @@ -90,11 +91,10 @@ input_json = override_sha1(input_json, args.provider, args.device, args.version) - git_setup(SDK_NAME, args.sdk_ref, SDK_URL, abs_workdir) output = {} - test_output = [] + test_output = {} build_output = [] logs = "" diff --git a/scripts/build_and_test/scan_app.py b/scripts/build_and_test/scan_app.py index 3b3e5fd..07df451 100644 --- a/scripts/build_and_test/scan_app.py +++ b/scripts/build_and_test/scan_app.py @@ -2,13 +2,21 @@ from device import Devices, Device from utils import run_cmd -def scan_variant(target: str, sdk_path: str, variant_param: str, variant_value: str, app_build_path: - Path, extra_flags: str=""): + +def scan_variant(target: str, + sdk_path: str, + variant_param: str, + variant_value: str, + app_build_path: Path, + extra_flags: str = ""): error = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True) if variant_param: - error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} -j ENABLE_SDK_WERROR=1 scan-build", cwd=app_build_path, no_throw=True) + error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} " + "-j ENABLE_SDK_WERROR=1 scan-build", + cwd=app_build_path, no_throw=True) else: - error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make -j ENABLE_SDK_WERROR=1 scan-build", cwd=app_build_path, no_throw=True) + error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make -j ENABLE_SDK_WERROR=1 scan-build", + cwd=app_build_path, no_throw=True) if error: print("\t=> KO") @@ -44,7 +52,11 @@ def scan_device(device: Device, variant_param: str, app_build_path: Path, sdk_pa variants = app_json.get(f"variants_{device.model_name}", []) variant_output = {} if len(variants) > 0: - variant_output, error_log = scan_all_variants(device.target_name, sdk_path, variant_param, variants, app_build_path) + variant_output, error_log = scan_all_variants(device.target_name, + sdk_path, + variant_param, + variants, + app_build_path) return variant_output, error_log diff --git a/scripts/build_and_test/test_app.py b/scripts/build_and_test/test_app.py index 56d8632..c149a8d 100644 --- a/scripts/build_and_test/test_app.py +++ b/scripts/build_and_test/test_app.py @@ -2,10 +2,13 @@ from device import Devices, Device from build_app import build_variant from utils import run_cmd +from typing import Tuple -def test(model: str, app_test_path: Path, app_build_path: Path, test_params: str): - output = {} - error, log = run_cmd(f"pytest {app_test_path}/ --tb=short -v --device {model} {test_params}", cwd=app_build_path, no_throw=True) + +def test(model: str, app_test_path: Path, app_build_path: Path, test_params: str) -> Tuple[str, str]: + output: str + error, log = run_cmd(f"pytest {app_test_path}/ --tb=short -v --device {model} {test_params}", + cwd=app_build_path, no_throw=True) if (error): output = "Fail" @@ -19,9 +22,10 @@ def install_dependencies(app_test_path: Path): error, log = run_cmd("pip install -r requirements.txt", cwd=app_test_path, no_throw=True) return error, log + def test_device(device: Device, variant_param: str, app_build_path: Path, app_test_path: Path, - sdk_path: Path, extra_flags: str, blacklist: str, test_params: str): - test_output = {} + sdk_path: Path, extra_flags: str, blacklist: str, test_params: str): + test_output: str log = "" if not device.selected: @@ -58,24 +62,23 @@ def test_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: } output["test"] = {} - blacklist = app_json.get(f"test_blacklist", []) + blacklist = app_json.get("test_blacklist", []) test_params = app_json.get("test_param_nanos", "") nanos_output, nanos_log = test_device(devices.nanos, variant_param, app_build_path, app_test_path, - sdk_path, extra_flags, blacklist, test_params) + sdk_path, extra_flags, blacklist, test_params) test_params = app_json.get("test_param_nanosp", "") nanosp_output, nanosp_log = test_device(devices.nanosp, variant_param, app_build_path, app_test_path, - sdk_path, extra_flags, blacklist, test_params) - + sdk_path, extra_flags, blacklist, test_params) test_params = app_json.get("test_param_nanox", "") nanox_output, nanox_log = test_device(devices.nanox, variant_param, app_build_path, app_test_path, - sdk_path, extra_flags, blacklist, test_params) + sdk_path, extra_flags, blacklist, test_params) test_params = app_json.get("test_param_stax", "") stax_output, stax_log = test_device(devices.stax, variant_param, app_build_path, app_test_path, - sdk_path, extra_flags, blacklist, test_params) + sdk_path, extra_flags, blacklist, test_params) if nanos_output: output["test"]["nanos"] = nanos_output diff --git a/scripts/build_and_test/utils.py b/scripts/build_and_test/utils.py index 9d16ef9..705a906 100644 --- a/scripts/build_and_test/utils.py +++ b/scripts/build_and_test/utils.py @@ -1,12 +1,13 @@ import subprocess import shutil from pathlib import Path +from typing import Tuple def run_cmd(cmd: str, cwd: Path, print_output: bool = True, - no_throw: bool = False) -> str: + no_throw: bool = False) -> Tuple[int, str]: error_log = "" print(f"[run_cmd] Running: {cmd} from {cwd}") @@ -40,7 +41,7 @@ def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path): if not Path.exists(workdir/Path(repo_name)): run_cmd(f"git {GIT_CONFIG} clone {repo_url} {repo_name}", cwd=workdir) else: - run_cmd(f"git fetch", cwd=workdir/Path(repo_name)) + run_cmd("git fetch", cwd=workdir/Path(repo_name)) error, _ = run_cmd(f"git checkout {repo_ref}", cwd=workdir/Path(repo_name), no_throw=True) if error: @@ -48,7 +49,9 @@ def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path): shutil.rmtree(workdir/Path(repo_name)) return - error, _ = run_cmd(f"git {GIT_CONFIG} submodule update --init --recursive", cwd=workdir/Path(repo_name), no_throw=True) + error, _ = run_cmd(f"git {GIT_CONFIG} submodule update --init --recursive", + cwd=workdir/Path(repo_name), + no_throw=True) if error: print("Error: removing folder") shutil.rmtree(workdir/Path(repo_name)) diff --git a/scripts/create_app_list/app_load_params_check.py b/scripts/create_app_list/app_load_params_check.py index f96f801..59c26b4 100755 --- a/scripts/create_app_list/app_load_params_check.py +++ b/scripts/create_app_list/app_load_params_check.py @@ -72,7 +72,7 @@ def parse_listapploadparams(app_load_params_str: str) -> Dict: return dict(app_load_params) -def check_manifest(manifest: dict, database: dict) -> None: +def check_manifest(manifest: dict, database: dict) -> int: ret = 0 for variant, data in manifest["VARIANTS"].items(): @@ -103,7 +103,8 @@ def check_manifest(manifest: dict, database: dict) -> None: app_load_params_value = app_load_params.get(key) if key == "appName": if len(app_load_params_value) != 1: - print(f"[ERROR] Expected a single value for 'appName' ({app_load_params_value} vs {app_params_ref_value})") + print("[ERROR] Expected a single value for 'appName' " + f"({app_load_params_value} vs {app_params_ref_value})") ret = -1 continue app_load_params_value = app_load_params_value[0] @@ -112,7 +113,8 @@ def check_manifest(manifest: dict, database: dict) -> None: app_load_params_value = ["0x000"] if len(app_load_params_value) != 1: - print(f"[ERROR] Expected a single value for 'appFlags' ({app_load_params_value} vs {app_params_ref_value})") + print("[ERROR] Expected a single value for 'appFlags' " + f"({app_load_params_value} vs {app_params_ref_value})") ret = -1 continue diff --git a/scripts/create_app_list/app_load_params_utils.py b/scripts/create_app_list/app_load_params_utils.py index 0dfb391..a8d603f 100755 --- a/scripts/create_app_list/app_load_params_utils.py +++ b/scripts/create_app_list/app_load_params_utils.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 -from pathlib import Path import json +from pathlib import Path +from typing import Dict def format_database(database: dict) -> str: @@ -20,7 +21,7 @@ def format_database(database: dict) -> str: return database_str -def load_database(database_path: Path): +def load_database(database_path: Path) -> Dict: database = {} if database_path.exists(): with open(database_path, 'r') as f: @@ -28,7 +29,6 @@ def load_database(database_path: Path): else: with open(database_path, 'w') as f: print("File created:", database_path) - database = [] return database diff --git a/scripts/create_app_list/gen_variant.py b/scripts/create_app_list/gen_variant.py index 8b0775e..9262e62 100644 --- a/scripts/create_app_list/gen_variant.py +++ b/scripts/create_app_list/gen_variant.py @@ -1,21 +1,26 @@ #!/usr/bin/env python3 from pathlib import Path -from makefile_dump import get_app_listvariants from collections import namedtuple +from typing import Dict, List, Union + +from makefile_dump import get_app_listvariants Models = namedtuple('Models', ['sdk_value', 'device_name']) MODELS = [Models("$NANOS_SDK", "nanos"), - Models("$NANOX_SDK", "nanox"), - Models("$NANOSP_SDK", "nanosp"), - Models("$STAX_SDK", "stax")] + Models("$NANOX_SDK", "nanox"), + Models("$NANOSP_SDK", "nanosp"), + Models("$STAX_SDK", "stax")] -def gen_variant(app_name: str, build_path: str, output_file: Path = "", workdir: Path = "") -> dict: +def gen_variant(app_name: str, + build_path: Path, + output_file: Path = Path(), + workdir: Path = Path()) -> dict: print(f"Generating for {app_name}") - database_params = { + database_params: Dict[str, Union[str, List]] = { "name": app_name, } @@ -23,8 +28,8 @@ def gen_variant(app_name: str, build_path: str, output_file: Path = "", workdir: for model in MODELS: try: variant_param_name, variants = get_app_listvariants(build_path, model.sdk_value, allow_failure=True) - except: - print("Skipping generation due to error") + except Exception as e: + print(f"Skipping generation due to error ({e})") continue database_params["variant_param"] = variant_param_name diff --git a/scripts/create_app_list/main.py b/scripts/create_app_list/main.py index 5d27a0f..8e7bcd9 100755 --- a/scripts/create_app_list/main.py +++ b/scripts/create_app_list/main.py @@ -3,8 +3,7 @@ import json from parse_github import parse_github -import sys -sys.path.append('..') + from gen_variant import gen_variant from utils import git_setup, merge_json @@ -69,4 +68,3 @@ with open(args.full_output_file, 'w') as json_file: json.dump(full_output, json_file, indent=1) - diff --git a/scripts/create_app_list/makefile_dump.py b/scripts/create_app_list/makefile_dump.py index 41b5e0d..cf0bdf9 100755 --- a/scripts/create_app_list/makefile_dump.py +++ b/scripts/create_app_list/makefile_dump.py @@ -10,7 +10,10 @@ from pathlib import Path from typing import Tuple, List -def get_app_listvariants(app_build_path: Path, sdk: str = "$NANOS_SDK", allow_failure: bool = False) -> Tuple[str, List[str]]: + +def get_app_listvariants(app_build_path: Path, + sdk: str = "$NANOS_SDK", + allow_failure: bool = False) -> Tuple[str, List[str]]: # Using listvariants Makefile target listvariants = run_cmd(f"make BOLOS_SDK={sdk} listvariants", cwd=app_build_path, no_throw=allow_failure) if "VARIANTS" not in listvariants: diff --git a/scripts/create_app_list/parse_github.py b/scripts/create_app_list/parse_github.py index d0035ca..063c6a1 100755 --- a/scripts/create_app_list/parse_github.py +++ b/scripts/create_app_list/parse_github.py @@ -1,13 +1,12 @@ import requests -import json -from pathlib import Path +from typing import Dict, List, Union base_url = "https://api.github.com" org_name = "LedgerHQ" repos_endpoint = f"{base_url}/orgs/{org_name}/repos" -params = { +params: Dict[str, Union[str, int]] = { "type": "public", "archived": "false", "sort": "full_name", @@ -16,7 +15,7 @@ } -def parse_github(access_token: str = "") -> str: +def parse_github(access_token: str = "") -> List[Dict]: repos = [] headers = { "Authorization": f"Bearer {access_token}", diff --git a/scripts/create_app_list/utils.py b/scripts/create_app_list/utils.py index a6c7a2f..548cc81 100644 --- a/scripts/create_app_list/utils.py +++ b/scripts/create_app_list/utils.py @@ -1,6 +1,7 @@ import subprocess from pathlib import Path + def run_cmd(cmd: str, cwd: Path, print_output: bool = True, @@ -32,7 +33,7 @@ def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path): if not Path.exists(workdir/Path(repo_name)): run_cmd(f"git {GIT_CONFIG} clone {repo_url} --recurse-submodules {repo_name}", cwd=workdir) else: - run_cmd(f"git fetch", cwd=workdir/Path(repo_name)) + run_cmd("git fetch", cwd=workdir/Path(repo_name)) run_cmd(f"git checkout {repo_ref}", cwd=workdir/Path(repo_name)) run_cmd("git submodule update --recursive", cwd=workdir/Path(repo_name)) diff --git a/scripts/output_scripts/__init__.py b/scripts/output_scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/output_scripts/convert.py b/scripts/output_scripts/convert.py index 7ebcce8..55766f8 100755 --- a/scripts/output_scripts/convert.py +++ b/scripts/output_scripts/convert.py @@ -2,6 +2,7 @@ from argparse import ArgumentParser from pathlib import Path + def json_to_markdown(json_list, key): # Set up the markdown table headers markdown_table = "| App Name | nanos | nanosp | nanox | stax |\n" @@ -20,9 +21,15 @@ def json_to_markdown(json_list, key): for d in ["nanos", "nanosp", "nanox", "stax"]: build_data = app_data.get(d, {}) if isinstance(build_data, dict): # nested structure - status_icon = ":red_circle:" if "Fail" in build_data.values() else ":heavy_check_mark:" if "Success" in build_data.values() else ":fast_forward:" if "Skipped" in build_data.values() else "" - else: - status_icon = ":heavy_check_mark:" if build_data == "Success" else ":red_circle:" if build_data == "Fail" else ":fast_forward:" if build_data == "Skipped" else "" + status_icon = (":red_circle:" if "Fail" in build_data.values() + else ":heavy_check_mark:" if "Success" in build_data.values() + else ":fast_forward:" if "Skipped" in build_data.values() + else "") + else: + status_icon = (":heavy_check_mark:" if build_data == "Success" + else ":red_circle:" if build_data == "Fail" + else ":fast_forward:" if build_data == "Skipped" + else "") row += " {} |".format(status_icon) markdown_table += row + "\n" diff --git a/scripts/output_scripts/merge.py b/scripts/output_scripts/merge.py index b1688ef..346a62b 100644 --- a/scripts/output_scripts/merge.py +++ b/scripts/output_scripts/merge.py @@ -2,10 +2,11 @@ import glob import json + def merge_jsons(json1, json2, key): # Create a dictionary to store the merged data by name merged_data = {} - + # Merge data from the first JSON array for item in json1: name = item["name"] @@ -13,7 +14,7 @@ def merge_jsons(json1, json2, key): merged_data[name] = item[key] else: merged_data[name].update(item[key]) - + # Merge data from the second JSON array for item in json2: name = item["name"] @@ -21,12 +22,13 @@ def merge_jsons(json1, json2, key): merged_data[name] = item[key] else: merged_data[name].update(item[key]) - + # Convert the merged dictionary back to a list of JSON objects merged_json = [{"name": name, key: merged_data[name]} for name in merged_data] - + return merged_json + def merge_multiple_jsons(input_files, key): result = {} for j in input_files: @@ -43,7 +45,7 @@ def main(): parser.add_argument("--key", help="Key to use for merging") args = parser.parse_args() - + input_files = glob.glob(args.input_pattern) if not input_files: print("No input files found.") diff --git a/scripts/output_scripts/slack.py b/scripts/output_scripts/slack.py index cea4bd1..91142bc 100644 --- a/scripts/output_scripts/slack.py +++ b/scripts/output_scripts/slack.py @@ -2,6 +2,7 @@ from argparse import ArgumentParser from pathlib import Path + def count_test_status(json_list): success_count = 0 fail_count = 0 @@ -32,6 +33,7 @@ def count_test_status(json_list): return success_count, fail_count, total_count, fail_list + def count_status(json_list, key): success_count = 0 fail_count = 0 @@ -75,7 +77,6 @@ def count_status(json_list, key): args = parser.parse_args() - with open(args.input_file) as json_file: json_list = json.load(json_file) @@ -84,13 +85,11 @@ def count_status(json_list, key): else: success_count, fail_count, total_count, fail_list = count_status(json_list, args.key) - title = f"{args.key}" if args.devices: title += f" on {args.devices}" - status_detail = "" if fail_count == 0: status = f":large_green_circle: Success for {total_count} apps" @@ -102,9 +101,9 @@ def count_status(json_list, key): fail_status += f"\t• {app_name}\n" if isinstance(details, dict): for device, variant in details.items(): - fail_status += f"\t\t - {device}: \n"; + fail_status += f"\t\t - {device}: \n" for v in variant: - fail_status += f"\t\t\t{v}\n"; + fail_status += f"\t\t\t{v}\n" else: for device in details: fail_status += f"\t\t- {device}\n" @@ -121,4 +120,3 @@ def count_status(json_list, key): if args.output_file: with open(args.output_file, 'w') as f: json.dump(slack_json, f, indent=1) - diff --git a/scripts/split_input/__init__.py b/scripts/split_input/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/split_input/split_input.py b/scripts/split_input/split_input.py index 70a6dff..035d69a 100755 --- a/scripts/split_input/split_input.py +++ b/scripts/split_input/split_input.py @@ -2,12 +2,13 @@ from pathlib import Path from argparse import ArgumentParser + if __name__ == "__main__": parser = ArgumentParser() input_json = {} parser.add_argument("--input_file", required=False, type=Path, - default=Path("input_files/input.json")) + default=Path("input_files/input.json")) parser.add_argument("--split_count", required=False, type=Path, default=10) args = parser.parse_args() @@ -22,7 +23,6 @@ num_files = args.split_count items_per_file = len(input_json) // num_files - for file_num in range(num_files): start_idx = file_num * items_per_file @@ -32,5 +32,5 @@ with open(file_name, 'w') as file: json.dump(input_json[start_idx:end_idx], file, indent=1) # Split data into ten JSON files - + print(f"Data split into {args.split_count} JSON files.") From 29a0e568076f2066b2b0bead308ff82e22199af6 Mon Sep 17 00:00:00 2001 From: Lucas PASCAL Date: Mon, 20 Nov 2023 16:48:52 +0100 Subject: [PATCH 03/10] [clean] Single script entrypoint --- .github/workflows/build_all.yml | 30 ++-- .github/workflows/fast-checks.yml | 16 +-- .github/workflows/refresh_inputs.yml | 8 +- .github/workflows/scan_all.yml | 26 ++-- .github/workflows/test_all.yml | 34 ++--- scripts/build_and_test/__init__.py | 104 ++++++++++++++ scripts/build_and_test/build_app.py | 7 +- scripts/build_and_test/main.py | 133 ------------------ scripts/build_and_test/scan_app.py | 5 +- scripts/build_and_test/test_app.py | 7 +- .../create_app_list/{main.py => __init__.py} | 24 +--- .../create_app_list/app_load_params_utils.py | 2 - scripts/create_app_list/gen_variant.py | 4 +- scripts/create_app_list/makefile_dump.py | 5 +- scripts/entrypoint.py | 106 ++++++++++++++ scripts/output_scripts/convert.py | 13 +- scripts/output_scripts/merge.py | 15 +- scripts/output_scripts/slack.py | 15 +- scripts/output_scripts/status.py | 13 +- scripts/{split_input => }/split_input.py | 11 +- scripts/split_input/__init__.py | 0 21 files changed, 290 insertions(+), 288 deletions(-) delete mode 100644 scripts/build_and_test/main.py rename scripts/create_app_list/{main.py => __init__.py} (61%) mode change 100755 => 100644 create mode 100644 scripts/entrypoint.py mode change 100755 => 100644 scripts/output_scripts/convert.py mode change 100755 => 100644 scripts/output_scripts/status.py rename scripts/{split_input => }/split_input.py (70%) mode change 100755 => 100644 delete mode 100644 scripts/split_input/__init__.py diff --git a/.github/workflows/build_all.yml b/.github/workflows/build_all.yml index 8260499..3025817 100644 --- a/.github/workflows/build_all.yml +++ b/.github/workflows/build_all.yml @@ -100,8 +100,8 @@ jobs: uses: actions/checkout@v3 - name: Split input into 10 files - run: | - python3 scripts/split_input/split_input.py --input_file input_files/input.json + run: | + python3 scripts/entrypoint.py split_input --input_file input_files/input.json - name: Archive output file uses: actions/upload-artifact@v3 @@ -186,17 +186,17 @@ jobs: - name: Setup repos if: ${{ inputs.use_sha1_from_live }} - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json --use_sha1_from_live --provider ${{ inputs.provider }} --device ${{ inputs.device }} --version ${{ inputs.version }} + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json --use_sha1_from_live --provider ${{ inputs.provider }} --device ${{ inputs.device }} --version ${{ inputs.version }} - name: Setup repos if: ${{ !inputs.use_sha1_from_live }} - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json - name: Launch build - run: | - python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --build --${{ matrix.device }} --skip_setup --output_file build_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt + run: | + python3 scripts/entrypoint.py build_and_test --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --build --${{ matrix.device }} --skip_setup --output_file build_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt #- name: Push info to DB # run: | @@ -534,8 +534,8 @@ jobs: name: build_${{ matrix.device }}_10.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "build_"${{ matrix.device }}"_*.json" --output_file merged_build_${{ matrix.device }}.json --key "build" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "build_"${{ matrix.device }}"_*.json" --output_file merged_build_${{ matrix.device }}.json --key "build" - name: Archive output file uses: actions/upload-artifact@v3 @@ -577,12 +577,12 @@ jobs: name: merged_build_stax.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "merged_build_*.json" --output_file full_build_output.json --key "build" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "merged_build_*.json" --output_file full_build_output.json --key "build" - name: Convert to markdown run: | - python3 scripts/output_scripts/convert.py --input_file full_build_output.json --output_file out.md --key build + python3 scripts/entrypoint.py convert_output --input_file full_build_output.json --output_file out.md --key build cat out.md >> $GITHUB_STEP_SUMMARY - name: Echo GHA url @@ -590,7 +590,7 @@ jobs: - name: Convert to slack json run: | - python3 scripts/output_scripts/slack.py --input_file full_build_output.json --output_file slack.json --key build --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} + python3 scripts/entrypoint.py slack_output --input_file full_build_output.json --output_file slack.json --key build --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} - name: Send custom JSON data to Slack workflow if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }} @@ -603,7 +603,7 @@ jobs: - name: Set job status run: | - python3 scripts/output_scripts/status.py --input_file full_build_output.json --key build + python3 scripts/entrypoint.py status_output --input_file full_build_output.json --key build - name: Archive output file if: always() diff --git a/.github/workflows/fast-checks.yml b/.github/workflows/fast-checks.yml index 00c24a7..99183cd 100644 --- a/.github/workflows/fast-checks.yml +++ b/.github/workflows/fast-checks.yml @@ -12,12 +12,6 @@ jobs: lint: strategy: fail-fast: false - matrix: - directory: - - build_and_test - - create_app_list - - output_scripts - - split_input name: Linting runs-on: ubuntu-latest steps: @@ -25,17 +19,11 @@ jobs: uses: actions/checkout@v3 - run: pip install flake8 - name: Flake8 lint Python code - run: find scripts/${{ matrix.directory }} -type f -name '*.py' -exec flake8 --max-line-length=120 '{}' '+' + run: find scripts/ -type f -name '*.py' -exec flake8 --max-line-length=120 '{}' '+' mypy: strategy: fail-fast: false - matrix: - directory: - - build_and_test - - create_app_list - - output_scripts - - split_input name: Type checking runs-on: ubuntu-latest steps: @@ -43,7 +31,7 @@ jobs: uses: actions/checkout@v3 - run: pip install mypy types-requests - name: Mypy type checking - run: mypy scripts/${{ matrix.directory }} + run: mypy scripts/ misspell: strategy: diff --git a/.github/workflows/refresh_inputs.yml b/.github/workflows/refresh_inputs.yml index bc18a5d..703a580 100644 --- a/.github/workflows/refresh_inputs.yml +++ b/.github/workflows/refresh_inputs.yml @@ -20,14 +20,14 @@ jobs: runs-on: ubuntu-latest container: image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder:latest - + steps: - name: Checkout repository uses: actions/checkout@v3 - + - name: Build db - run: | - python3 scripts/create_app_list/main.py --full_output_file out.json --access_token ${{ secrets.GH_ACCESS_TOKEN }} + run: | + python3 scripts/entrypoint.py create_app_list --full_output_file out.json --access_token ${{ secrets.GH_ACCESS_TOKEN }} - name: Upload file uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/scan_all.yml b/.github/workflows/scan_all.yml index 5802697..a5ca7f2 100644 --- a/.github/workflows/scan_all.yml +++ b/.github/workflows/scan_all.yml @@ -83,8 +83,8 @@ jobs: uses: actions/checkout@v3 - name: Split input into 10 files - run: | - python3 scripts/split_input/split_input.py --input_file input_files/input.json + run: | + python3 scripts/entrypoint.py split_input --input_file input_files/input.json - name: Archive output file uses: actions/upload-artifact@v3 @@ -168,12 +168,12 @@ jobs: name: input_${{ matrix.index }}.json - name: Setup repos - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json - name: Launch scan - run: | - python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --scan --${{ matrix.device }} --skip_setup --output_file scan_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt + run: | + python3 scripts/entrypoint.py build_and_test --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --scan --${{ matrix.device }} --skip_setup --output_file scan_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt #- name: Push info to DB # run: | @@ -254,8 +254,8 @@ jobs: name: scan_${{ matrix.device }}_10.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "scan_"${{ matrix.device }}"_*.json" --output_file merged_scan_${{ matrix.device }}.json --key "scan" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "scan_"${{ matrix.device }}"_*.json" --output_file merged_scan_${{ matrix.device }}.json --key "scan" - name: Archive output file uses: actions/upload-artifact@v3 @@ -554,12 +554,12 @@ jobs: name: merged_scan_stax.json - name: Merge output files - run: | - python3 scripts/output_scripts/merge.py --input_pattern "merged_scan_*.json" --output_file full_scan_output.json --key "scan" + run: | + python3 scripts/entrypoint.py merge_output --input_pattern "merged_scan_*.json" --output_file full_scan_output.json --key "scan" - name: Convert to markdown run: | - python3 scripts/output_scripts/convert.py --input_file full_scan_output.json --output_file out.md --key scan + python3 scripts/entrypoint.py convert_output --input_file full_scan_output.json --output_file out.md --key scan cat out.md >> $GITHUB_STEP_SUMMARY - name: Echo GHA url @@ -567,7 +567,7 @@ jobs: - name: Convert to slack json run: | - python3 scripts/output_scripts/slack.py --input_file full_scan_output.json --output_file slack.json --key scan --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} + python3 scripts/entrypoint.py slack_output --input_file full_scan_output.json --output_file slack.json --key scan --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }} - name: Send custom JSON data to Slack workflow if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }} @@ -580,7 +580,7 @@ jobs: - name: Set job status run: | - python3 scripts/output_scripts/status.py --input_file full_scan_output.json --key scan + python3 scripts/entrypoint.py status_output --input_file full_scan_output.json --key scan - name: Archive output file uses: actions/upload-artifact@v3 diff --git a/.github/workflows/test_all.yml b/.github/workflows/test_all.yml index 538333b..7e3b642 100644 --- a/.github/workflows/test_all.yml +++ b/.github/workflows/test_all.yml @@ -47,8 +47,8 @@ jobs: uses: actions/checkout@v3 - name: Split input into 10 files - run: | - python3 scripts/split_input/split_input.py --input_file input_files/input.json + run: | + python3 scripts/entrypoint.py split_input --input_file input_files/input.json - name: Archive output file uses: actions/upload-artifact@v3 @@ -136,8 +136,8 @@ jobs: name: input_${{ matrix.index }}.json - name: Setup repos - run: | - python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json + run: | + python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json - name: Install tests dependencies run: | @@ -167,22 +167,22 @@ jobs: - name: Launch test nanosp if: ${{ env.test_nanosp }} - run: | - python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_nanosp }} --input_file input_${{ matrix.index }}.json --test --nanosp --skip_setup --output_file test_nanosp_${{ matrix.index }}.json --logs_file log_nanosp.txt + run: | + python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_nanosp }} --input_file input_${{ matrix.index }}.json --test --nanosp --skip_setup --output_file test_nanosp_${{ matrix.index }}.json --logs_file log_nanosp.txt - name: Launch test nanox if: ${{ env.test_nanox }} - run: | - python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_nanox }} --input_file input_${{ matrix.index }}.json --test --nanox --skip_setup --output_file test_nanox_${{ matrix.index }}.json --logs_file log_nanox.txt + run: | + python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_nanox }} --input_file input_${{ matrix.index }}.json --test --nanox --skip_setup --output_file test_nanox_${{ matrix.index }}.json --logs_file log_nanox.txt - name: Launch test stax if: ${{ env.test_stax }} - run: | - python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_stax }} --input_file input_${{ matrix.index }}.json --test --stax --skip_setup --output_file test_stax_${{ matrix.index }}.json --logs_file log_stax.txt + run: | + python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_stax }} --input_file input_${{ matrix.index }}.json --test --stax --skip_setup --output_file test_stax_${{ matrix.index }}.json --logs_file log_stax.txt - name: Merge output files - run: | - python scripts/output_scripts/merge.py --input_pattern "test_*.json" --output_file test_output_${{ matrix.index }}.json --key "test" + run: | + python scripts/entrypoint.py merge_output --input_pattern "test_*.json" --output_file test_output_${{ matrix.index }}.json --key "test" - name: Archive output file uses: actions/upload-artifact@v3 with: @@ -339,8 +339,8 @@ jobs: name: test_output_10.json - name: Merge output files - run: | - python scripts/output_scripts/merge.py --input_pattern "test_output_*.json" --output_file test_output.json --key "test" + run: | + python scripts/entrypoint.py merge_output --input_pattern "test_output_*.json" --output_file test_output.json --key "test" - name: Archive output file uses: actions/upload-artifact@v3 @@ -351,7 +351,7 @@ jobs: - name: Convert to markdown run: | - python scripts/output_scripts/convert.py --input_file test_output.json --output_file out.md --key test + python scripts/entrypoint.py convert_output --input_file test_output.json --output_file out.md --key test cat out.md >> $GITHUB_STEP_SUMMARY - name: Echo GHA url @@ -359,7 +359,7 @@ jobs: - name: Convert to slack json run: | - python scripts/output_scripts/slack.py --input_file test_output.json --output_file slack.json --key test --url ${{ env.url }} + python scripts/entrypoint.py slack_output --input_file test_output.json --output_file slack.json --key test --url ${{ env.url }} - name: Send custom JSON data to Slack workflow if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }} @@ -372,4 +372,4 @@ jobs: - name: Set job status run: | - python scripts/output_scripts/status.py --input_file test_output.json --key test + python scripts/entrypoint.py status_output --input_file test_output.json --key test diff --git a/scripts/build_and_test/__init__.py b/scripts/build_and_test/__init__.py index e69de29..eb36c52 100644 --- a/scripts/build_and_test/__init__.py +++ b/scripts/build_and_test/__init__.py @@ -0,0 +1,104 @@ +import json +from pathlib import Path +from argparse import Namespace + +from build_and_test.sha1 import override_sha1 +from build_and_test.build_app import build_all_devices +from build_and_test.test_app import test_all_devices +from build_and_test.scan_app import scan_all_devices +from build_and_test.device import Devices +from build_and_test.utils import git_setup, merge_json + +SDK_NAME = "sdk" +SDK_URL = "https://github.com/LedgerHQ/ledger-secure-sdk.git" +SDK_BRANCH = "origin/master" + + +def main(args: Namespace) -> None: + input_json = {} + + abs_workdir = Path.cwd() / args.workdir + + if not abs_workdir.exists(): + abs_workdir.mkdir() + + nanos_enable = False + nanosp_enable = False + nanox_enable = False + stax_enable = False + + if args.all or args.nanos: + print("Nanos enabled") + nanos_enable = True + if args.all or args.nanosp: + print("Nanosp enabled") + nanosp_enable = True + if args.all or args.nanox: + print("Nanox enabled") + nanox_enable = True + if args.all or args.stax: + print("Stax enabled") + stax_enable = True + + devices = Devices(nanos_enable, nanosp_enable, nanox_enable, stax_enable) + + if Path(args.input_file).exists(): + with open(args.input_file) as json_file: + input_json = json.load(json_file) + else: + print("Error: input file does not exist") + exit() + + if args.use_sha1_from_live: + if not args.provider: + print("Error: you must specify provider") + exit() + if not args.device: + print("Error: you must specify device") + exit() + if not args.version: + print("Error: you must specify version") + exit() + + input_json = override_sha1(input_json, args.provider, args.device, args.version) + + git_setup(SDK_NAME, args.sdk_ref, SDK_URL, abs_workdir) + + output = {} + test_output = {} + build_output = [] + logs = "" + + for app_json in input_json: + repo_name = app_json.get("name") + if not args.skip_setup: + repo_ref = app_json.get("ref") + repo_url = app_json.get("url") + print(f"Setup {repo_name}") + git_setup(repo_name, repo_ref, repo_url, abs_workdir) + + if args.build: + print(f"Build {repo_name}") + build_app, log = build_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) + build_output.append(build_app) + logs += log + + if args.test: + print(f"Test {repo_name}") + test_app, log = test_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) + build_output.append(test_app) + logs += log + + if args.scan_build: + print(f"Scan build {repo_name}") + scan_app, log = scan_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) + build_output.append(scan_app) + logs += log + + output = merge_json(build_output, test_output, "name") + + with open(args.output_file, 'w') as json_file: + json.dump(output, json_file, indent=1) + + with open(args.logs_file, 'w') as file: + file.write(logs) diff --git a/scripts/build_and_test/build_app.py b/scripts/build_and_test/build_app.py index ec62293..6dc9f6e 100644 --- a/scripts/build_and_test/build_app.py +++ b/scripts/build_and_test/build_app.py @@ -1,7 +1,8 @@ -from pathlib import Path -from device import Devices, Device -from utils import run_cmd import os +from pathlib import Path + +from build_and_test.device import Devices, Device +from build_and_test.utils import run_cmd def build_variant(target: str, diff --git a/scripts/build_and_test/main.py b/scripts/build_and_test/main.py deleted file mode 100644 index f1818ae..0000000 --- a/scripts/build_and_test/main.py +++ /dev/null @@ -1,133 +0,0 @@ -import json -from pathlib import Path -from argparse import ArgumentParser -from sha1 import override_sha1 - -from build_app import build_all_devices -from test_app import test_all_devices -from scan_app import scan_all_devices -from device import Devices -from utils import git_setup, merge_json - -SDK_NAME = "sdk" -SDK_URL = "https://github.com/LedgerHQ/ledger-secure-sdk.git" -SDK_BRANCH = "origin/master" - - -if __name__ == "__main__": - parser = ArgumentParser() - input_json = {} - - parser.add_argument("--skip_setup", action='store_true') - - # Devices - parser.add_argument("--all", action='store_true') - parser.add_argument("--nanos", action='store_true') - parser.add_argument("--nanosp", action='store_true') - parser.add_argument("--nanox", action='store_true') - parser.add_argument("--stax", action='store_true') - - parser.add_argument("--test", action='store_true') - parser.add_argument("--build", action='store_true') - parser.add_argument("--scan_build", action='store_true') - - parser.add_argument("--sdk_ref", required=False, type=Path, default="origin/master") - - parser.add_argument("--input_file", required=False, type=Path, default=Path("input_files/test_input.json")) - parser.add_argument("--output_file", required=False, type=Path, default=Path("output_files/output.json")) - parser.add_argument("--logs_file", required=False, type=Path, - default=Path("output_files/error_logs.txt")) - parser.add_argument("--workdir", required=False, type=str, default="workdir") - - parser.add_argument("--use_sha1_from_live", required=False, action='store_true') - parser.add_argument("--provider", required=False, type=str) - parser.add_argument("--device", required=False, type=str) - parser.add_argument("--version", required=False, type=str) - - args = parser.parse_args() - - abs_workdir = Path.cwd()/args.workdir - - if not abs_workdir.exists(): - abs_workdir.mkdir() - - nanos_enable = False - nanosp_enable = False - nanox_enable = False - stax_enable = False - - if args.all or args.nanos: - print("Nanos enabled") - nanos_enable = True - if args.all or args.nanosp: - print("Nanosp enabled") - nanosp_enable = True - if args.all or args.nanox: - print("Nanox enabled") - nanox_enable = True - if args.all or args.stax: - print("Stax enabled") - stax_enable = True - - devices = Devices(nanos_enable, nanosp_enable, nanox_enable, stax_enable) - - if Path(args.input_file).exists(): - with open(args.input_file) as json_file: - input_json = json.load(json_file) - else: - print("Error: input file does not exist") - exit() - - if args.use_sha1_from_live: - if not args.provider: - print("Error: you must specify provider") - exit() - if not args.device: - print("Error: you must specify device") - exit() - if not args.version: - print("Error: you must specify version") - exit() - - input_json = override_sha1(input_json, args.provider, args.device, args.version) - - git_setup(SDK_NAME, args.sdk_ref, SDK_URL, abs_workdir) - - output = {} - test_output = {} - build_output = [] - logs = "" - - for app_json in input_json: - repo_name = app_json.get("name") - if not args.skip_setup: - repo_ref = app_json.get("ref") - repo_url = app_json.get("url") - print(f"Setup {repo_name}") - git_setup(repo_name, repo_ref, repo_url, abs_workdir) - - if args.build: - print(f"Build {repo_name}") - build_app, log = build_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) - build_output.append(build_app) - logs += log - - if args.test: - print(f"Test {repo_name}") - test_app, log = test_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) - build_output.append(test_app) - logs += log - - if args.scan_build: - print(f"Scan build {repo_name}") - scan_app, log = scan_all_devices(devices, abs_workdir/Path(SDK_NAME), app_json, abs_workdir) - build_output.append(scan_app) - logs += log - - output = merge_json(build_output, test_output, "name") - - with open(args.output_file, 'w') as json_file: - json.dump(output, json_file, indent=1) - - with open(args.logs_file, 'w') as file: - file.write(logs) diff --git a/scripts/build_and_test/scan_app.py b/scripts/build_and_test/scan_app.py index 07df451..23615a9 100644 --- a/scripts/build_and_test/scan_app.py +++ b/scripts/build_and_test/scan_app.py @@ -1,6 +1,7 @@ from pathlib import Path -from device import Devices, Device -from utils import run_cmd + +from build_and_test.device import Devices, Device +from build_and_test.utils import run_cmd def scan_variant(target: str, diff --git a/scripts/build_and_test/test_app.py b/scripts/build_and_test/test_app.py index c149a8d..34a9d78 100644 --- a/scripts/build_and_test/test_app.py +++ b/scripts/build_and_test/test_app.py @@ -1,9 +1,10 @@ from pathlib import Path -from device import Devices, Device -from build_app import build_variant -from utils import run_cmd from typing import Tuple +from build_and_test.device import Devices, Device +from build_and_test.build_app import build_variant +from build_and_test.utils import run_cmd + def test(model: str, app_test_path: Path, app_build_path: Path, test_params: str) -> Tuple[str, str]: output: str diff --git a/scripts/create_app_list/main.py b/scripts/create_app_list/__init__.py old mode 100755 new mode 100644 similarity index 61% rename from scripts/create_app_list/main.py rename to scripts/create_app_list/__init__.py index 8e7bcd9..51e610f --- a/scripts/create_app_list/main.py +++ b/scripts/create_app_list/__init__.py @@ -1,27 +1,13 @@ -from argparse import ArgumentParser +from argparse import Namespace from pathlib import Path import json -from parse_github import parse_github +from create_app_list.parse_github import parse_github +from create_app_list.gen_variant import gen_variant +from create_app_list.utils import git_setup, merge_json -from gen_variant import gen_variant -from utils import git_setup, merge_json - -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--access_token", required=True, type=str) - parser.add_argument("--workdir", required=False, type=str, default="workdir") - parser.add_argument("--extra_info_file", required=False, type=Path, default=Path("input_files/extra_info.json")) - parser.add_argument("--repo_file", required=False, type=Path) - parser.add_argument("--skip_setup", required=False, action='store_true') - - parser.add_argument("--full_output_file", required=False, type=Path, default=Path("output_files/full_out.json")) - parser.add_argument("--repo_output_file", required=False, type=Path, default=Path("output_files/repo.json")) - parser.add_argument("--variant_output_file", required=False, type=Path, default=Path("output_files/variant.json")) - - args = parser.parse_args() +def main(args: Namespace) -> None: abs_workdir = Path.cwd()/args.workdir if not abs_workdir.exists(): diff --git a/scripts/create_app_list/app_load_params_utils.py b/scripts/create_app_list/app_load_params_utils.py index a8d603f..5853479 100755 --- a/scripts/create_app_list/app_load_params_utils.py +++ b/scripts/create_app_list/app_load_params_utils.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - import json from pathlib import Path from typing import Dict diff --git a/scripts/create_app_list/gen_variant.py b/scripts/create_app_list/gen_variant.py index 9262e62..1c5f6a9 100644 --- a/scripts/create_app_list/gen_variant.py +++ b/scripts/create_app_list/gen_variant.py @@ -1,10 +1,8 @@ -#!/usr/bin/env python3 - from pathlib import Path from collections import namedtuple from typing import Dict, List, Union -from makefile_dump import get_app_listvariants +from create_app_list.makefile_dump import get_app_listvariants Models = namedtuple('Models', ['sdk_value', 'device_name']) diff --git a/scripts/create_app_list/makefile_dump.py b/scripts/create_app_list/makefile_dump.py index cf0bdf9..b6fa6e2 100755 --- a/scripts/create_app_list/makefile_dump.py +++ b/scripts/create_app_list/makefile_dump.py @@ -1,15 +1,14 @@ -#!/usr/bin/env python3 - """ This is a pure duplicate of https://github.com/LedgerHQ/ledger-app-workflows/blob/master/scripts/makefile_dump.py This is to allow easily generating the db from the apps code. """ -from utils import run_cmd from pathlib import Path from typing import Tuple, List +from create_app_list.utils import run_cmd + def get_app_listvariants(app_build_path: Path, sdk: str = "$NANOS_SDK", diff --git a/scripts/entrypoint.py b/scripts/entrypoint.py new file mode 100644 index 0000000..dc78881 --- /dev/null +++ b/scripts/entrypoint.py @@ -0,0 +1,106 @@ +import sys +from argparse import ArgumentParser, Namespace +from pathlib import Path + +sys.path.insert(1, Path(__file__).resolve().parent) + + +def parse_args() -> Namespace: + parser = ArgumentParser() + subparsers = parser.add_subparsers(help="Specific operation", dest="operation") + + # split_input + subparser = subparsers.add_parser('split_input') + subparser.add_argument("--split_count", required=False, type=int, default=10) + subparser.add_argument("--input_file", required=False, type=Path, default=Path("input_files/input.json")) + + # build_and_test + subparser = subparsers.add_parser('build_and_test') + subparser.add_argument("--skip_setup", action='store_true') + + subparser.add_argument("--all", action='store_true') + subparser.add_argument("--nanos", action='store_true') + subparser.add_argument("--nanosp", action='store_true') + subparser.add_argument("--nanox", action='store_true') + subparser.add_argument("--stax", action='store_true') + + subparser.add_argument("--test", action='store_true') + subparser.add_argument("--build", action='store_true') + subparser.add_argument("--scan_build", action='store_true') + + subparser.add_argument("--sdk_ref", required=False, type=Path, default="origin/master") + + subparser.add_argument("--input_file", required=False, type=Path, default=Path("input_files/test_input.json")) + subparser.add_argument("--output_file", required=False, type=Path, default=Path("output_files/output.json")) + subparser.add_argument("--logs_file", required=False, type=Path, + default=Path("output_files/error_logs.txt")) + subparser.add_argument("--workdir", required=False, type=str, default="workdir") + + subparser.add_argument("--use_sha1_from_live", required=False, action='store_true') + subparser.add_argument("--provider", required=False, type=str) + subparser.add_argument("--device", required=False, type=str) + subparser.add_argument("--version", required=False, type=str) + + # output_scripts + # # convert + subparser = subparsers.add_parser('convert_output') + subparser.add_argument("--input_file", required=True, type=Path) + subparser.add_argument("--output_file", required=False, type=Path) + subparser.add_argument("--key", required=False, type=str, default="build") + # # merge + subparser = subparsers.add_parser('merge_output', description="Merge JSON files based on a specified key") + subparser.add_argument("--input_pattern", help="Pattern for input JSON files (e.g., input*.json)") + subparser.add_argument("--output_file", help="Output merged JSON file") + subparser.add_argument("--key", help="Key to use for merging") + # # status + subparser = subparsers.add_parser('status_output') + subparser.add_argument("--input_file", required=True, type=Path) + subparser.add_argument("--key", required=True) + # # slack + subparser = subparsers.add_parser('slack_output') + subparser.add_argument("--input_file", required=True, type=Path) + subparser.add_argument("--output_file", required=False, type=Path) + subparser.add_argument("--key", required=False, type=str, default="build") + subparser.add_argument("--devices", required=False, type=str) + subparser.add_argument("--url", required=False, type=str) + + # create_app_list + subparser = subparsers.add_parser('create_app_list') + subparser.add_argument("--access_token", required=True, type=str) + subparser.add_argument("--workdir", required=False, type=str, default="workdir") + subparser.add_argument("--extra_info_file", required=False, type=Path, default=Path("input_files/extra_info.json")) + subparser.add_argument("--repo_file", required=False, type=Path) + subparser.add_argument("--skip_setup", required=False, action='store_true') + + subparser.add_argument("--full_output_file", required=False, type=Path, default=Path("output_files/full_out.json")) + subparser.add_argument("--repo_output_file", required=False, type=Path, default=Path("output_files/repo.json")) + subparser.add_argument("--variant_output_file", required=False, type=Path, + default=Path("output_files/variant.json")) + + return parser.parse_args() + + +if __name__ == '__main__': + args = parse_args() + + if args.operation == 'split_input': + import split_input + split_input.main(args) + elif args.operation == 'build_and_test': + import build_and_test + build_and_test.main(args) + elif args.operation == 'convert_output': + import output_scripts.convert + output_scripts.convert.main(args) + elif args.operation == 'merge_output': + import output_scripts.merge + output_scripts.merge.main(args) + elif args.operation == 'status_output': + import output_scripts.status + output_scripts.status.main(args) + elif args.operation == 'slack_output': + import output_scripts.slack + output_scripts.slack.main(args) + elif args.operation == 'create_app_list': + import create_app_list + create_app_list.main(args) diff --git a/scripts/output_scripts/convert.py b/scripts/output_scripts/convert.py old mode 100755 new mode 100644 index 55766f8..118358a --- a/scripts/output_scripts/convert.py +++ b/scripts/output_scripts/convert.py @@ -1,6 +1,5 @@ import json -from argparse import ArgumentParser -from pathlib import Path +from argparse import Namespace def json_to_markdown(json_list, key): @@ -69,15 +68,7 @@ def count_status(json_list, key): """ -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--input_file", required=True, type=Path) - parser.add_argument("--output_file", required=False, type=Path) - parser.add_argument("--key", required=False, type=str, default="build") - - args = parser.parse_args() - +def main(args: Namespace) -> None: with open(args.input_file) as json_file: json_list = json.load(json_file) diff --git a/scripts/output_scripts/merge.py b/scripts/output_scripts/merge.py index 346a62b..083721e 100644 --- a/scripts/output_scripts/merge.py +++ b/scripts/output_scripts/merge.py @@ -1,6 +1,6 @@ -import argparse import glob import json +from argparse import Namespace def merge_jsons(json1, json2, key): @@ -38,14 +38,7 @@ def merge_multiple_jsons(input_files, key): return result -def main(): - parser = argparse.ArgumentParser(description="Merge JSON files based on a specified key") - parser.add_argument("--input_pattern", help="Pattern for input JSON files (e.g., input*.json)") - parser.add_argument("--output_file", help="Output merged JSON file") - parser.add_argument("--key", help="Key to use for merging") - - args = parser.parse_args() - +def main(args: Namespace) -> None: input_files = glob.glob(args.input_pattern) if not input_files: print("No input files found.") @@ -54,7 +47,3 @@ def main(): merged_json = merge_multiple_jsons(input_files, args.key) with open(args.output_file, 'w') as f: json.dump(merged_json, f, indent=1) - - -if __name__ == "__main__": - main() diff --git a/scripts/output_scripts/slack.py b/scripts/output_scripts/slack.py index 91142bc..7e309a1 100644 --- a/scripts/output_scripts/slack.py +++ b/scripts/output_scripts/slack.py @@ -1,6 +1,5 @@ import json -from argparse import ArgumentParser -from pathlib import Path +from argparse import Namespace def count_test_status(json_list): @@ -66,17 +65,7 @@ def count_status(json_list, key): return success_count, fail_count, total_count, fail_list -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--input_file", required=True, type=Path) - parser.add_argument("--output_file", required=False, type=Path) - parser.add_argument("--key", required=False, type=str, default="build") - parser.add_argument("--devices", required=False, type=str) - parser.add_argument("--url", required=False, type=str) - - args = parser.parse_args() - +def main(args: Namespace) -> None: with open(args.input_file) as json_file: json_list = json.load(json_file) diff --git a/scripts/output_scripts/status.py b/scripts/output_scripts/status.py old mode 100755 new mode 100644 index bc3defd..3c15f20 --- a/scripts/output_scripts/status.py +++ b/scripts/output_scripts/status.py @@ -1,6 +1,5 @@ import json -from argparse import ArgumentParser -from pathlib import Path +from argparse import Namespace def check_status(json_data, key): @@ -19,15 +18,7 @@ def check_status(json_data, key): raise ValueError(f"Failed for {app_name}") -if __name__ == "__main__": - parser = ArgumentParser() - - parser.add_argument("--input_file", required=True, type=Path) - parser.add_argument("--key", required=True) - - args = parser.parse_args() - +def main(args: Namespace) -> None: with open(args.input_file) as json_file: json_data = json.load(json_file) - check_status(json_data, args.key) diff --git a/scripts/split_input/split_input.py b/scripts/split_input.py old mode 100755 new mode 100644 similarity index 70% rename from scripts/split_input/split_input.py rename to scripts/split_input.py index 035d69a..4559cc3 --- a/scripts/split_input/split_input.py +++ b/scripts/split_input.py @@ -1,18 +1,11 @@ import json from pathlib import Path -from argparse import ArgumentParser +from argparse import Namespace -if __name__ == "__main__": - parser = ArgumentParser() +def main(args: Namespace) -> None: input_json = {} - parser.add_argument("--input_file", required=False, type=Path, - default=Path("input_files/input.json")) - parser.add_argument("--split_count", required=False, type=Path, default=10) - - args = parser.parse_args() - if Path(args.input_file).exists(): with open(args.input_file) as json_file: input_json = json.load(json_file) diff --git a/scripts/split_input/__init__.py b/scripts/split_input/__init__.py deleted file mode 100644 index e69de29..0000000 From 6886c653885814f68e1b3595b407d7b6de9ac672 Mon Sep 17 00:00:00 2001 From: Lucas PASCAL Date: Mon, 20 Nov 2023 17:00:23 +0100 Subject: [PATCH 04/10] [clean] Shared 'utils.py' --- scripts/build_and_test/__init__.py | 2 +- scripts/build_and_test/build_app.py | 2 +- scripts/build_and_test/scan_app.py | 2 +- scripts/build_and_test/test_app.py | 2 +- scripts/create_app_list/__init__.py | 2 +- scripts/create_app_list/makefile_dump.py | 4 +- scripts/create_app_list/utils.py | 53 ------------------------ scripts/entrypoint.py | 4 +- scripts/{build_and_test => }/utils.py | 20 ++++----- 9 files changed, 18 insertions(+), 73 deletions(-) delete mode 100644 scripts/create_app_list/utils.py rename scripts/{build_and_test => }/utils.py (85%) diff --git a/scripts/build_and_test/__init__.py b/scripts/build_and_test/__init__.py index eb36c52..dfc19c9 100644 --- a/scripts/build_and_test/__init__.py +++ b/scripts/build_and_test/__init__.py @@ -7,7 +7,7 @@ from build_and_test.test_app import test_all_devices from build_and_test.scan_app import scan_all_devices from build_and_test.device import Devices -from build_and_test.utils import git_setup, merge_json +from utils import git_setup, merge_json SDK_NAME = "sdk" SDK_URL = "https://github.com/LedgerHQ/ledger-secure-sdk.git" diff --git a/scripts/build_and_test/build_app.py b/scripts/build_and_test/build_app.py index 6dc9f6e..1f4b9ac 100644 --- a/scripts/build_and_test/build_app.py +++ b/scripts/build_and_test/build_app.py @@ -2,7 +2,7 @@ from pathlib import Path from build_and_test.device import Devices, Device -from build_and_test.utils import run_cmd +from utils import run_cmd def build_variant(target: str, diff --git a/scripts/build_and_test/scan_app.py b/scripts/build_and_test/scan_app.py index 23615a9..f129284 100644 --- a/scripts/build_and_test/scan_app.py +++ b/scripts/build_and_test/scan_app.py @@ -1,7 +1,7 @@ from pathlib import Path from build_and_test.device import Devices, Device -from build_and_test.utils import run_cmd +from utils import run_cmd def scan_variant(target: str, diff --git a/scripts/build_and_test/test_app.py b/scripts/build_and_test/test_app.py index 34a9d78..001914d 100644 --- a/scripts/build_and_test/test_app.py +++ b/scripts/build_and_test/test_app.py @@ -3,7 +3,7 @@ from build_and_test.device import Devices, Device from build_and_test.build_app import build_variant -from build_and_test.utils import run_cmd +from utils import run_cmd def test(model: str, app_test_path: Path, app_build_path: Path, test_params: str) -> Tuple[str, str]: diff --git a/scripts/create_app_list/__init__.py b/scripts/create_app_list/__init__.py index 51e610f..484c312 100644 --- a/scripts/create_app_list/__init__.py +++ b/scripts/create_app_list/__init__.py @@ -4,7 +4,7 @@ from create_app_list.parse_github import parse_github from create_app_list.gen_variant import gen_variant -from create_app_list.utils import git_setup, merge_json +from utils import git_setup, merge_json def main(args: Namespace) -> None: diff --git a/scripts/create_app_list/makefile_dump.py b/scripts/create_app_list/makefile_dump.py index b6fa6e2..a1e0dbd 100755 --- a/scripts/create_app_list/makefile_dump.py +++ b/scripts/create_app_list/makefile_dump.py @@ -7,14 +7,14 @@ from pathlib import Path from typing import Tuple, List -from create_app_list.utils import run_cmd +from utils import run_cmd def get_app_listvariants(app_build_path: Path, sdk: str = "$NANOS_SDK", allow_failure: bool = False) -> Tuple[str, List[str]]: # Using listvariants Makefile target - listvariants = run_cmd(f"make BOLOS_SDK={sdk} listvariants", cwd=app_build_path, no_throw=allow_failure) + _, listvariants = run_cmd(f"make BOLOS_SDK={sdk} listvariants", cwd=app_build_path, no_throw=allow_failure) if "VARIANTS" not in listvariants: raise ValueError(f"Invalid variants retrieved: {listvariants}") diff --git a/scripts/create_app_list/utils.py b/scripts/create_app_list/utils.py deleted file mode 100644 index 548cc81..0000000 --- a/scripts/create_app_list/utils.py +++ /dev/null @@ -1,53 +0,0 @@ -import subprocess -from pathlib import Path - - -def run_cmd(cmd: str, - cwd: Path, - print_output: bool = True, - no_throw: bool = False) -> str: - print(f"[run_cmd] Running: {cmd} from {cwd}") - - ret = subprocess.run(cmd, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True, - cwd=cwd) - if no_throw is False and ret.returncode: - print(f"[run_cmd] Error {ret.returncode} raised while running cmd: {cmd}") - print("[run_cmd] Output was:") - print(ret.stdout) - raise ValueError() - - if ret.returncode: - print(f"[run_cmd] Output:\n{ret.stdout}") - - return ret.stdout.strip() - - -def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path): - # Force clone in https over SSH - GIT_CONFIG = ' -c url."https://github.com/".insteadOf="git@github.com:" -c url."https://".insteadOf="git://"' - - if not Path.exists(workdir/Path(repo_name)): - run_cmd(f"git {GIT_CONFIG} clone {repo_url} --recurse-submodules {repo_name}", cwd=workdir) - else: - run_cmd("git fetch", cwd=workdir/Path(repo_name)) - - run_cmd(f"git checkout {repo_ref}", cwd=workdir/Path(repo_name)) - run_cmd("git submodule update --recursive", cwd=workdir/Path(repo_name)) - - -def merge_json(json1: dict, json2: dict, key: str): - merged_data = [] - - for obj1 in json1: - merged_obj = obj1.copy() - for obj2 in json2: - if obj1[key] == obj2[key]: - merged_obj.update(obj2) - break - merged_data.append(merged_obj) - - return merged_data diff --git a/scripts/entrypoint.py b/scripts/entrypoint.py index dc78881..10039fc 100644 --- a/scripts/entrypoint.py +++ b/scripts/entrypoint.py @@ -2,7 +2,7 @@ from argparse import ArgumentParser, Namespace from pathlib import Path -sys.path.insert(1, Path(__file__).resolve().parent) +sys.path.insert(1, str(Path(__file__).resolve().parent)) def parse_args() -> Namespace: @@ -33,7 +33,7 @@ def parse_args() -> Namespace: subparser.add_argument("--input_file", required=False, type=Path, default=Path("input_files/test_input.json")) subparser.add_argument("--output_file", required=False, type=Path, default=Path("output_files/output.json")) subparser.add_argument("--logs_file", required=False, type=Path, - default=Path("output_files/error_logs.txt")) + default=Path("output_files/error_logs.txt")) subparser.add_argument("--workdir", required=False, type=str, default="workdir") subparser.add_argument("--use_sha1_from_live", required=False, action='store_true') diff --git a/scripts/build_and_test/utils.py b/scripts/utils.py similarity index 85% rename from scripts/build_and_test/utils.py rename to scripts/utils.py index 705a906..6bb9ac8 100644 --- a/scripts/build_and_test/utils.py +++ b/scripts/utils.py @@ -8,7 +8,7 @@ def run_cmd(cmd: str, cwd: Path, print_output: bool = True, no_throw: bool = False) -> Tuple[int, str]: - error_log = "" + stdout = "" print(f"[run_cmd] Running: {cmd} from {cwd}") ret = subprocess.run(cmd, @@ -16,25 +16,23 @@ def run_cmd(cmd: str, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True, - cwd=cwd) - if no_throw is False and ret.returncode: - print(f"[run_cmd] Error {ret.returncode} raised while running cmd: {cmd}") - print("[run_cmd] Output was:") - print(ret.stdout) - raise ValueError() + cwd=cwd, + check=not no_throw) if ret.returncode: print(f"[run_cmd] Output:\n{ret.stdout}") - - error_log = f''' + stdout = f''' ############################################################################### [run_cmd] Running: {cmd} from {cwd}" ############################################################################### ''' + ret.stdout - return ret.returncode, error_log + else: + stdout = ret.stdout.strip() + + return ret.returncode, stdout -def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path): +def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path) -> None: # Force clone in https over SSH GIT_CONFIG = ' -c url."https://github.com/".insteadOf="git@github.com:" -c url."https://".insteadOf="git://"' From 704b11beff147ea1465263f80387611e84e84f1b Mon Sep 17 00:00:00 2001 From: Lucas PASCAL Date: Mon, 20 Nov 2023 18:22:55 +0100 Subject: [PATCH 05/10] [fix] mypy review --- scripts/build_and_test/__init__.py | 10 ++--- scripts/build_and_test/build_app.py | 35 +++++++++------ scripts/build_and_test/scan_app.py | 45 +++++++++++-------- scripts/build_and_test/test_app.py | 24 ++++++---- scripts/create_app_list/__init__.py | 2 +- .../create_app_list/app_load_params_check.py | 4 +- scripts/create_app_list/parse_github.py | 27 ++++++----- scripts/utils.py | 4 +- 8 files changed, 89 insertions(+), 62 deletions(-) diff --git a/scripts/build_and_test/__init__.py b/scripts/build_and_test/__init__.py index dfc19c9..ad5ae69 100644 --- a/scripts/build_and_test/__init__.py +++ b/scripts/build_and_test/__init__.py @@ -64,16 +64,14 @@ def main(args: Namespace) -> None: git_setup(SDK_NAME, args.sdk_ref, SDK_URL, abs_workdir) - output = {} - test_output = {} build_output = [] logs = "" for app_json in input_json: - repo_name = app_json.get("name") + repo_name = app_json["name"] if not args.skip_setup: - repo_ref = app_json.get("ref") - repo_url = app_json.get("url") + repo_ref = app_json["ref"] + repo_url = app_json["url"] print(f"Setup {repo_name}") git_setup(repo_name, repo_ref, repo_url, abs_workdir) @@ -95,7 +93,7 @@ def main(args: Namespace) -> None: build_output.append(scan_app) logs += log - output = merge_json(build_output, test_output, "name") + output = merge_json(build_output, {}, "name") with open(args.output_file, 'w') as json_file: json.dump(output, json_file, indent=1) diff --git a/scripts/build_and_test/build_app.py b/scripts/build_and_test/build_app.py index 1f4b9ac..93b2b28 100644 --- a/scripts/build_and_test/build_app.py +++ b/scripts/build_and_test/build_app.py @@ -1,23 +1,24 @@ import os from pathlib import Path +from typing import Dict, Optional, Tuple, Union from build_and_test.device import Devices, Device from utils import run_cmd def build_variant(target: str, - sdk_path: str, - variant_param: str, + sdk_path: Path, + variant_param: Optional[str], variant_value: str, app_build_path: Path, - extra_flags: str = ""): + extra_flags: str = "") -> Tuple[int, str]: if not os.path.exists(app_build_path): print("\t=> KO") return True, f"Error: {app_build_path} does not exists\n" - error = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True) - if variant_param: + run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True) + if variant_param is not None: error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} {extra_flags}", cwd=app_build_path, no_throw=True) else: @@ -30,7 +31,11 @@ def build_variant(target: str, return error, log -def build_all_variants(target: str, sdk_path: str, variant_param: str, variant_list: list, app_build_path: Path): +def build_all_variants(target: str, + sdk_path: Path, + variant_param: Optional[str], + variant_list: list, + app_build_path: Path) -> Tuple[Dict[str, str], str]: output = {} error_log = "" for variant in variant_list: @@ -45,18 +50,22 @@ def build_all_variants(target: str, sdk_path: str, variant_param: str, variant_l return output, error_log -def build_device(device: Device, variant_param: str, app_build_path: Path, sdk_path: Path, app_json: dict): - blacklist = app_json.get("build_blacklist", "[]") +def build_device(device: Device, + variant_param: Optional[str], + app_build_path: Path, + sdk_path: Path, + app_json: dict) -> Tuple[Union[str, Dict[str, str]], str]: + blacklist = app_json.get("build_blacklist", []) error_log = "" if not device.selected: - return None, error_log + return "Skipped - not selected", error_log if device.model_name in blacklist: - return "Skipped", error_log + return "Skipped - blacklisted", error_log variants = app_json.get(f"variants_{device.model_name}", []) - variant_output = {} + variant_output: Dict[str, str] = {} if len(variants) > 0: variant_output, error_log = build_all_variants(device.target_name, sdk_path, @@ -68,9 +77,9 @@ def build_device(device: Device, variant_param: str, app_build_path: Path, sdk_p def build_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: Path): - repo_name = app_json.get("name") + repo_name = app_json["name"] variant_param = app_json.get("variant_param") - app_build_path = workdir / Path(app_json.get("name") + "/" + app_json.get("build_path", ".")) + app_build_path = workdir / Path(repo_name + "/" + app_json.get("build_path", ".")) output = { "name": repo_name, diff --git a/scripts/build_and_test/scan_app.py b/scripts/build_and_test/scan_app.py index f129284..6d93a77 100644 --- a/scripts/build_and_test/scan_app.py +++ b/scripts/build_and_test/scan_app.py @@ -1,31 +1,36 @@ from pathlib import Path +from typing import Dict, Optional, Tuple, Union from build_and_test.device import Devices, Device from utils import run_cmd def scan_variant(target: str, - sdk_path: str, - variant_param: str, + sdk_path: Path, + variant_param: Optional[str], variant_value: str, app_build_path: Path, - extra_flags: str = ""): - error = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True) + extra_flags: str = "") -> Tuple[int, str]: + run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True) if variant_param: - error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} " - "-j ENABLE_SDK_WERROR=1 scan-build", - cwd=app_build_path, no_throw=True) + status, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} " + "-j ENABLE_SDK_WERROR=1 scan-build", + cwd=app_build_path, no_throw=True) else: - error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make -j ENABLE_SDK_WERROR=1 scan-build", - cwd=app_build_path, no_throw=True) + status, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make -j ENABLE_SDK_WERROR=1 scan-build", + cwd=app_build_path, no_throw=True) - if error: + if status: print("\t=> KO") - return error, log + return status, log -def scan_all_variants(target: str, sdk_path: str, variant_param: str, variant_list: list, app_build_path: Path): +def scan_all_variants(target: str, + sdk_path: Path, + variant_param: Optional[str], + variant_list: list, + app_build_path: Path) -> Tuple[Dict, str]: output = {} error_log = "" for variant in variant_list: @@ -40,18 +45,22 @@ def scan_all_variants(target: str, sdk_path: str, variant_param: str, variant_li return output, error_log -def scan_device(device: Device, variant_param: str, app_build_path: Path, sdk_path: Path, app_json: dict): +def scan_device(device: Device, + variant_param: Optional[str], + app_build_path: Path, + sdk_path: Path, + app_json: dict) -> Tuple[Union[str, Dict[str, str]], str]: whitelist = app_json.get("scan_whitelist", "[]") error_log = "" if not device.selected: - return None, error_log + return "Skipped - not selected", error_log if device.model_name not in whitelist: - return "Skipped", error_log + return "Skipped - blacklisted", error_log variants = app_json.get(f"variants_{device.model_name}", []) - variant_output = {} + variant_output: Dict[str, str] = {} if len(variants) > 0: variant_output, error_log = scan_all_variants(device.target_name, sdk_path, @@ -63,9 +72,9 @@ def scan_device(device: Device, variant_param: str, app_build_path: Path, sdk_pa def scan_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: Path): - repo_name = app_json.get("name") + repo_name = app_json["name"] variant_param = app_json.get("variant_param") - app_build_path = workdir / Path(app_json.get("name") + "/" + app_json.get("build_path", ".")) + app_build_path = workdir / Path(repo_name + "/" + app_json.get("build_path", ".")) output = { "name": repo_name, diff --git a/scripts/build_and_test/test_app.py b/scripts/build_and_test/test_app.py index 001914d..7d609a0 100644 --- a/scripts/build_and_test/test_app.py +++ b/scripts/build_and_test/test_app.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Tuple +from typing import Optional, Tuple from build_and_test.device import Devices, Device from build_and_test.build_app import build_variant @@ -19,21 +19,27 @@ def test(model: str, app_test_path: Path, app_build_path: Path, test_params: str return output, log -def install_dependencies(app_test_path: Path): +def install_dependencies(app_test_path: Path) -> Tuple[int, str]: error, log = run_cmd("pip install -r requirements.txt", cwd=app_test_path, no_throw=True) return error, log -def test_device(device: Device, variant_param: str, app_build_path: Path, app_test_path: Path, - sdk_path: Path, extra_flags: str, blacklist: str, test_params: str): +def test_device(device: Device, + variant_param: Optional[str], + app_build_path: Path, + app_test_path: Path, + sdk_path: Path, + extra_flags: str, + blacklist: str, + test_params: str) -> Tuple[str, str]: test_output: str log = "" if not device.selected: - return None, log + return "Skipped - not selected", log if device.model_name in blacklist: - return "Skipped", log + return "Skipped - blacklisted", log error, log = install_dependencies(app_test_path) if error: @@ -51,10 +57,10 @@ def test_device(device: Device, variant_param: str, app_build_path: Path, app_te def test_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: Path): - repo_name = app_json.get("name") + repo_name = app_json["name"] variant_param = app_json.get("variant_param") - app_build_path = workdir / Path(app_json.get("name") + "/" + app_json.get("build_path", ".")) - app_test_path = workdir / Path(app_json.get("name") + "/" + app_json.get("test_dir", ".")) + app_build_path = workdir / Path(repo_name + "/" + app_json.get("build_path", ".")) + app_test_path = workdir / Path(repo_name + "/" + app_json.get("test_dir", ".")) extra_flags = app_json.get("extra_flags", "") blacklist = app_json.get("build_blacklist", "[]") diff --git a/scripts/create_app_list/__init__.py b/scripts/create_app_list/__init__.py index 484c312..15755cf 100644 --- a/scripts/create_app_list/__init__.py +++ b/scripts/create_app_list/__init__.py @@ -34,7 +34,7 @@ def main(args: Namespace) -> None: repo_name = repo.get("name") repo_ref = repo.get("ref") repo_url = repo.get("url") - repo_build_path = abs_workdir/Path(repo_name)/Path(repo.get("build_path", ".")) + repo_build_path = abs_workdir / Path(repo_name) / Path(repo.get("build_path", ".")) if not args.skip_setup: print("Cloning repo") diff --git a/scripts/create_app_list/app_load_params_check.py b/scripts/create_app_list/app_load_params_check.py index 59c26b4..63636c2 100755 --- a/scripts/create_app_list/app_load_params_check.py +++ b/scripts/create_app_list/app_load_params_check.py @@ -99,8 +99,8 @@ def check_manifest(manifest: dict, database: dict) -> int: # Check that the params match with the one from the database for key in APP_LOAD_PARAMS_VALUE_CHECK: - app_params_ref_value = app_params_ref.get(key) - app_load_params_value = app_load_params.get(key) + app_params_ref_value = app_params_ref[key] + app_load_params_value = app_load_params[key] if key == "appName": if len(app_load_params_value) != 1: print("[ERROR] Expected a single value for 'appName' " diff --git a/scripts/create_app_list/parse_github.py b/scripts/create_app_list/parse_github.py index 063c6a1..a81b6bf 100755 --- a/scripts/create_app_list/parse_github.py +++ b/scripts/create_app_list/parse_github.py @@ -1,21 +1,26 @@ import requests -from typing import Dict, List, Union +from dataclasses import dataclass, asdict +from typing import Dict, List base_url = "https://api.github.com" org_name = "LedgerHQ" repos_endpoint = f"{base_url}/orgs/{org_name}/repos" -params: Dict[str, Union[str, int]] = { - "type": "public", - "archived": "false", - "sort": "full_name", - "page": 1, - "per_page": 100 -} +@dataclass +class Params: + type: str + archived: str + sort: str + page: int + per_page: int -def parse_github(access_token: str = "") -> List[Dict]: + +params = Params("public", "false", "full_name", 1, 100) + + +def parse_github(access_token: str = "") -> List[Dict[str, str]]: repos = [] headers = { "Authorization": f"Bearer {access_token}", @@ -23,7 +28,7 @@ def parse_github(access_token: str = "") -> List[Dict]: } while True: - response = requests.get(repos_endpoint, params=params, headers=headers) + response = requests.get(repos_endpoint, params=asdict(params), headers=headers) repos_data = response.json() if not repos_data: # No more repositories to fetch break @@ -45,6 +50,6 @@ def parse_github(access_token: str = "") -> List[Dict]: owner_name = parent_data["parent"]["owner"]["login"] repos.append({"name": repo_name, "owner": owner_name, "ref": ref, "url": repo_url}) - params["page"] += 1 + params.page += 1 return repos diff --git a/scripts/utils.py b/scripts/utils.py index 6bb9ac8..f2403d1 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -1,7 +1,7 @@ import subprocess import shutil from pathlib import Path -from typing import Tuple +from typing import List, Tuple def run_cmd(cmd: str, @@ -56,7 +56,7 @@ def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path) -> No return -def merge_json(json1: dict, json2: dict, key: str): +def merge_json(json1: dict, json2: dict, key: str) -> List: merged_data = [] for obj1 in json1: From d768d36e664a18336d39824de571b5964bcd9699 Mon Sep 17 00:00:00 2001 From: Sarah GLINER Date: Mon, 27 Nov 2023 11:31:08 +0100 Subject: [PATCH 06/10] build_app: run when variant exists --- scripts/build_and_test/build_app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_and_test/build_app.py b/scripts/build_and_test/build_app.py index 93b2b28..3381fdf 100644 --- a/scripts/build_and_test/build_app.py +++ b/scripts/build_and_test/build_app.py @@ -18,7 +18,7 @@ def build_variant(target: str, return True, f"Error: {app_build_path} does not exists\n" run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make clean", cwd=app_build_path, no_throw=True) - if variant_param is not None: + if variant_param: error, log = run_cmd(f"TARGET={target} BOLOS_SDK={sdk_path} make {variant_param}={variant_value} {extra_flags}", cwd=app_build_path, no_throw=True) else: From 32970fb637410624f982c63cde3972681d846142 Mon Sep 17 00:00:00 2001 From: Sarah GLINER Date: Mon, 27 Nov 2023 11:39:13 +0100 Subject: [PATCH 07/10] status : split into blacklisted and unselected --- scripts/build_and_test/build_app.py | 4 ++-- scripts/build_and_test/scan_app.py | 4 ++-- scripts/build_and_test/test_app.py | 4 ++-- scripts/output_scripts/convert.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/build_and_test/build_app.py b/scripts/build_and_test/build_app.py index 3381fdf..28645e5 100644 --- a/scripts/build_and_test/build_app.py +++ b/scripts/build_and_test/build_app.py @@ -59,10 +59,10 @@ def build_device(device: Device, error_log = "" if not device.selected: - return "Skipped - not selected", error_log + return "Unselected", error_log if device.model_name in blacklist: - return "Skipped - blacklisted", error_log + return "Blacklisted", error_log variants = app_json.get(f"variants_{device.model_name}", []) variant_output: Dict[str, str] = {} diff --git a/scripts/build_and_test/scan_app.py b/scripts/build_and_test/scan_app.py index 6d93a77..96affe6 100644 --- a/scripts/build_and_test/scan_app.py +++ b/scripts/build_and_test/scan_app.py @@ -54,10 +54,10 @@ def scan_device(device: Device, error_log = "" if not device.selected: - return "Skipped - not selected", error_log + return "Unselected", error_log if device.model_name not in whitelist: - return "Skipped - blacklisted", error_log + return "Blacklisted", error_log variants = app_json.get(f"variants_{device.model_name}", []) variant_output: Dict[str, str] = {} diff --git a/scripts/build_and_test/test_app.py b/scripts/build_and_test/test_app.py index 7d609a0..42cf39d 100644 --- a/scripts/build_and_test/test_app.py +++ b/scripts/build_and_test/test_app.py @@ -36,10 +36,10 @@ def test_device(device: Device, log = "" if not device.selected: - return "Skipped - not selected", log + return "Unselected", log if device.model_name in blacklist: - return "Skipped - blacklisted", log + return "Blacklisted", log error, log = install_dependencies(app_test_path) if error: diff --git a/scripts/output_scripts/convert.py b/scripts/output_scripts/convert.py index 118358a..d13dcaf 100644 --- a/scripts/output_scripts/convert.py +++ b/scripts/output_scripts/convert.py @@ -22,12 +22,12 @@ def json_to_markdown(json_list, key): if isinstance(build_data, dict): # nested structure status_icon = (":red_circle:" if "Fail" in build_data.values() else ":heavy_check_mark:" if "Success" in build_data.values() - else ":fast_forward:" if "Skipped" in build_data.values() + else ":fast_forward:" if "Blacklisted" in build_data.values() else "") else: status_icon = (":heavy_check_mark:" if build_data == "Success" else ":red_circle:" if build_data == "Fail" - else ":fast_forward:" if build_data == "Skipped" + else ":fast_forward:" if build_data == "Blacklisted" else "") row += " {} |".format(status_icon) From 4cb2c950b1fae5b91a61bb91cb242f20c8db17d3 Mon Sep 17 00:00:00 2001 From: Sarah GLINER Date: Mon, 27 Nov 2023 11:49:31 +0100 Subject: [PATCH 08/10] refresh_inputs: update base branch for PR creation --- .github/workflows/refresh_inputs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/refresh_inputs.yml b/.github/workflows/refresh_inputs.yml index 703a580..a1fc1bf 100644 --- a/.github/workflows/refresh_inputs.yml +++ b/.github/workflows/refresh_inputs.yml @@ -84,7 +84,7 @@ jobs: if: ${{ failure() && (github.event_name == 'schedule' || inputs.create_pr == true) }} run: | apk add github-cli - gh pr create --base "test-all-apps" --head "update-input" --title "[BOT] Update input file" --body "Input file has changed ! Please review changes !" + gh pr create --base "develop" --head "update-input" --title "[BOT] Update input file" --body "Input file has changed ! Please review changes !" env: GITHUB_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} From 6ca0c121d410850657c7abc728c7b46b1544face Mon Sep 17 00:00:00 2001 From: Sarah GLINER Date: Mon, 27 Nov 2023 13:33:27 +0100 Subject: [PATCH 09/10] output: write status only for selected device --- scripts/build_and_test/build_app.py | 8 ++++---- scripts/build_and_test/scan_app.py | 8 ++++---- scripts/build_and_test/test_app.py | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/build_and_test/build_app.py b/scripts/build_and_test/build_app.py index 28645e5..7f23ed6 100644 --- a/scripts/build_and_test/build_app.py +++ b/scripts/build_and_test/build_app.py @@ -94,13 +94,13 @@ def build_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: stax_output, stax_log = build_device(devices.stax, variant_param, app_build_path, sdk_path, app_json) - if nanos_output: + if nanos_output and devices.nanos.selected: output["build"]["nanos"] = nanos_output - if nanosp_output: + if nanosp_output and devices.nanosp.selected: output["build"]["nanosp"] = nanosp_output - if nanox_output: + if nanox_output and devices.nanox.selected: output["build"]["nanox"] = nanox_output - if stax_output: + if stax_output and devices.stax.selected: output["build"]["stax"] = stax_output log = nanos_log + nanosp_log + nanox_log + stax_log diff --git a/scripts/build_and_test/scan_app.py b/scripts/build_and_test/scan_app.py index 96affe6..618df26 100644 --- a/scripts/build_and_test/scan_app.py +++ b/scripts/build_and_test/scan_app.py @@ -89,13 +89,13 @@ def scan_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: stax_output, stax_log = scan_device(devices.stax, variant_param, app_build_path, sdk_path, app_json) - if nanos_output: + if nanos_output and devices.nanos.selected: output["scan"]["nanos"] = nanos_output - if nanosp_output: + if nanosp_output and devices.nanosp.selected: output["scan"]["nanosp"] = nanosp_output - if nanox_output: + if nanox_output and devices.nanox.selected: output["scan"]["nanox"] = nanox_output - if stax_output: + if stax_output and devices.stax.selected: output["scan"]["stax"] = stax_output log = nanos_log + nanosp_log + nanox_log + stax_log diff --git a/scripts/build_and_test/test_app.py b/scripts/build_and_test/test_app.py index 42cf39d..e0f00b0 100644 --- a/scripts/build_and_test/test_app.py +++ b/scripts/build_and_test/test_app.py @@ -87,13 +87,13 @@ def test_all_devices(devices: Devices, sdk_path: Path, app_json: dict, workdir: stax_output, stax_log = test_device(devices.stax, variant_param, app_build_path, app_test_path, sdk_path, extra_flags, blacklist, test_params) - if nanos_output: + if nanos_output and devices.nanos.selected: output["test"]["nanos"] = nanos_output - if nanosp_output: + if nanosp_output and devices.nanosp.selected: output["test"]["nanosp"] = nanosp_output - if nanox_output: + if nanox_output and devices.nanox.selected: output["test"]["nanox"] = nanox_output - if stax_output: + if stax_output and devices.stax.selected: output["test"]["stax"] = stax_output print(output) From fb3900d25291be2ff8f377da7d9f7d05e2968a07 Mon Sep 17 00:00:00 2001 From: Sarah GLINER Date: Mon, 27 Nov 2023 14:42:18 +0100 Subject: [PATCH 10/10] merge_json: fix mypy --- scripts/build_and_test/__init__.py | 2 +- scripts/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_and_test/__init__.py b/scripts/build_and_test/__init__.py index ad5ae69..5bc7c84 100644 --- a/scripts/build_and_test/__init__.py +++ b/scripts/build_and_test/__init__.py @@ -93,7 +93,7 @@ def main(args: Namespace) -> None: build_output.append(scan_app) logs += log - output = merge_json(build_output, {}, "name") + output = merge_json(build_output, [], "name") with open(args.output_file, 'w') as json_file: json.dump(output, json_file, indent=1) diff --git a/scripts/utils.py b/scripts/utils.py index f2403d1..896c940 100644 --- a/scripts/utils.py +++ b/scripts/utils.py @@ -56,7 +56,7 @@ def git_setup(repo_name: str, repo_ref: str, repo_url: str, workdir: Path) -> No return -def merge_json(json1: dict, json2: dict, key: str) -> List: +def merge_json(json1: list, json2: list, key: str) -> List: merged_data = [] for obj1 in json1: