Python script checkers

.github/workflows/build_all.yml

@@ -100,8 +100,8 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Split input into 10 files
-        run: | 
-          python3 scripts/split_input/split_input.py --input_file input_files/input.json 
+        run: |
+          python3 scripts/entrypoint.py split_input --input_file input_files/input.json
 
       - name: Archive output file
         uses: actions/upload-artifact@v3
@@ -186,17 +186,17 @@ jobs:
 
       - name: Setup repos
         if: ${{ inputs.use_sha1_from_live }}
-        run: | 
-          python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json --use_sha1_from_live --provider ${{ inputs.provider }} --device ${{ inputs.device }} --version ${{ inputs.version }} 
+        run: |
+          python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json --use_sha1_from_live --provider ${{ inputs.provider }} --device ${{ inputs.device }} --version ${{ inputs.version }}
 
       - name: Setup repos
         if: ${{ !inputs.use_sha1_from_live }}
-        run: | 
-          python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json
+        run: |
+          python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json
 
       - name: Launch build
-        run: | 
-          python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --build --${{ matrix.device }} --skip_setup --output_file build_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt
+        run: |
+          python3 scripts/entrypoint.py build_and_test --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --build --${{ matrix.device }} --skip_setup --output_file build_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt
 
       #- name: Push info to DB
       #  run: | 
@@ -534,8 +534,8 @@ jobs:
           name: build_${{ matrix.device }}_10.json
 
       - name: Merge output files
-        run: | 
-          python3 scripts/output_scripts/merge.py --input_pattern "build_"${{ matrix.device }}"_*.json" --output_file merged_build_${{ matrix.device }}.json --key "build"
+        run: |
+          python3 scripts/entrypoint.py merge_output --input_pattern "build_"${{ matrix.device }}"_*.json" --output_file merged_build_${{ matrix.device }}.json --key "build"
 
       - name: Archive output file
         uses: actions/upload-artifact@v3
@@ -577,20 +577,20 @@ jobs:
           name: merged_build_stax.json
 
       - name: Merge output files
-        run: | 
-          python3 scripts/output_scripts/merge.py --input_pattern "merged_build_*.json" --output_file full_build_output.json --key "build"
+        run: |
+          python3 scripts/entrypoint.py merge_output --input_pattern "merged_build_*.json" --output_file full_build_output.json --key "build"
 
       - name: Convert to markdown
         run: |
-          python3 scripts/output_scripts/convert.py --input_file full_build_output.json --output_file out.md --key build
+          python3 scripts/entrypoint.py convert_output --input_file full_build_output.json --output_file out.md --key build
           cat out.md >> $GITHUB_STEP_SUMMARY
 
       - name: Echo GHA url 
         run: echo "url"=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} >> $GITHUB_ENV
 
       - name: Convert to slack json
         run: |
-          python3 scripts/output_scripts/slack.py --input_file full_build_output.json --output_file slack.json --key build --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }}
+          python3 scripts/entrypoint.py slack_output --input_file full_build_output.json --output_file slack.json --key build --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }}
 
       - name: Send custom JSON data to Slack workflow
         if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }}
@@ -603,7 +603,7 @@ jobs:
 
       - name: Set job status
         run: |
-          python3 scripts/output_scripts/status.py --input_file full_build_output.json --key build
+          python3 scripts/entrypoint.py status_output --input_file full_build_output.json --key build
 
       - name: Archive output file
         if: always()

.github/workflows/fast-checks.yml

@@ -0,0 +1,48 @@
+name: Fast Python checks
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - develop
+      - master
+  pull_request:
+
+jobs:
+  lint:
+    strategy:
+      fail-fast: false
+    name: Linting
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone
+      uses: actions/checkout@v3
+    - run: pip install flake8
+    - name: Flake8 lint Python code
+      run: find scripts/ -type f -name '*.py' -exec flake8 --max-line-length=120 '{}' '+'
+
+  mypy:
+    strategy:
+      fail-fast: false
+    name: Type checking
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone
+      uses: actions/checkout@v3
+    - run: pip install mypy types-requests
+    - name: Mypy type checking
+      run: mypy scripts/
+
+  misspell:
+    strategy:
+      fail-fast: false
+    name: Check misspellings
+    runs-on: ubuntu-latest
+    steps:
+    - name: Clone
+      uses: actions/checkout@v3
+    - name: Check misspellings
+      uses: codespell-project/actions-codespell@v1
+      with:
+        builtin: clear,rare
+        check_filenames: true

.github/workflows/refresh_inputs.yml

@@ -20,14 +20,14 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: ghcr.io/ledgerhq/ledger-app-builder/ledger-app-builder:latest
-    
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-        
+
       - name: Build db
-        run: | 
-          python3 scripts/create_app_list/main.py --full_output_file out.json --access_token ${{ secrets.GH_ACCESS_TOKEN }} 
+        run: |
+          python3 scripts/entrypoint.py  create_app_list --full_output_file out.json --access_token ${{ secrets.GH_ACCESS_TOKEN }}
       - name: Upload file
         uses: actions/upload-artifact@v3
         with:
@@ -84,7 +84,7 @@ jobs:
         if: ${{ failure() && (github.event_name == 'schedule' || inputs.create_pr == true) }}
         run: |
           apk add github-cli
-          gh pr create --base "test-all-apps" --head "update-input" --title "[BOT] Update input file" --body "Input file has changed ! Please review changes !"
+          gh pr create --base "develop" --head "update-input" --title "[BOT] Update input file" --body "Input file has changed ! Please review changes !"
         env:
           GITHUB_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
 

.github/workflows/scan_all.yml

@@ -83,8 +83,8 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Split input into 10 files
-        run: | 
-          python3 scripts/split_input/split_input.py --input_file input_files/input.json 
+        run: |
+          python3 scripts/entrypoint.py split_input --input_file input_files/input.json
 
       - name: Archive output file
         uses: actions/upload-artifact@v3
@@ -168,12 +168,12 @@ jobs:
           name: input_${{ matrix.index }}.json
 
       - name: Setup repos
-        run: | 
-          python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json 
+        run: |
+          python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json
 
       - name: Launch scan
-        run: | 
-          python3 scripts/build_and_test/main.py --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --scan --${{ matrix.device }} --skip_setup --output_file scan_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt
+        run: |
+          python3 scripts/entrypoint.py build_and_test --sdk_ref ${{ inputs.sdk_ref || 'master' }} --input_file input_${{ matrix.index }}.json --scan --${{ matrix.device }} --skip_setup --output_file scan_${{ matrix.device }}_${{ matrix.index }}.json --logs_file log_${{ matrix.device }}_${{ matrix.index }}.txt
 
       #- name: Push info to DB
       #  run: | 
@@ -254,8 +254,8 @@ jobs:
           name: scan_${{ matrix.device }}_10.json
 
       - name: Merge output files
-        run: | 
-          python3 scripts/output_scripts/merge.py --input_pattern "scan_"${{ matrix.device }}"_*.json" --output_file merged_scan_${{ matrix.device }}.json --key "scan"
+        run: |
+          python3 scripts/entrypoint.py merge_output --input_pattern "scan_"${{ matrix.device }}"_*.json" --output_file merged_scan_${{ matrix.device }}.json --key "scan"
 
       - name: Archive output file
         uses: actions/upload-artifact@v3
@@ -554,20 +554,20 @@ jobs:
           name: merged_scan_stax.json
 
       - name: Merge output files
-        run: | 
-          python3 scripts/output_scripts/merge.py --input_pattern "merged_scan_*.json" --output_file full_scan_output.json --key "scan"
+        run: |
+          python3 scripts/entrypoint.py merge_output --input_pattern "merged_scan_*.json" --output_file full_scan_output.json --key "scan"
 
       - name: Convert to markdown
         run: |
-          python3 scripts/output_scripts/convert.py --input_file full_scan_output.json --output_file out.md --key scan
+          python3 scripts/entrypoint.py convert_output --input_file full_scan_output.json --output_file out.md --key scan
           cat out.md >> $GITHUB_STEP_SUMMARY
 
       - name: Echo GHA url 
         run: echo "url"=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} >> $GITHUB_ENV
 
       - name: Convert to slack json
         run: |
-          python3 scripts/output_scripts/slack.py --input_file full_scan_output.json --output_file slack.json --key scan --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }}
+          python3 scripts/entrypoint.py slack_output --input_file full_scan_output.json --output_file slack.json --key scan --devices ${{ needs.setup-devices.outputs.names }} --url ${{ env.url }}
 
       - name: Send custom JSON data to Slack workflow
         if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }}
@@ -580,7 +580,7 @@ jobs:
 
       - name: Set job status
         run: |
-          python3 scripts/output_scripts/status.py --input_file full_scan_output.json --key scan
+          python3 scripts/entrypoint.py status_output --input_file full_scan_output.json --key scan
 
       - name: Archive output file
         uses: actions/upload-artifact@v3

.github/workflows/test_all.yml

@@ -47,8 +47,8 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Split input into 10 files
-        run: | 
-          python3 scripts/split_input/split_input.py --input_file input_files/input.json 
+        run: |
+          python3 scripts/entrypoint.py split_input --input_file input_files/input.json
 
       - name: Archive output file
         uses: actions/upload-artifact@v3
@@ -136,8 +136,8 @@ jobs:
           name: input_${{ matrix.index }}.json
 
       - name: Setup repos
-        run: | 
-          python3 scripts/build_and_test/main.py --input_file input_${{ matrix.index }}.json 
+        run: |
+          python3 scripts/entrypoint.py build_and_test --input_file input_${{ matrix.index }}.json
 
       - name: Install tests dependencies
         run: |
@@ -167,22 +167,22 @@ jobs:
 
       - name: Launch test nanosp
         if: ${{ env.test_nanosp }}
-        run: | 
-          python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_nanosp }} --input_file input_${{ matrix.index }}.json --test --nanosp --skip_setup --output_file test_nanosp_${{ matrix.index }}.json --logs_file log_nanosp.txt
+        run: |
+          python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_nanosp }} --input_file input_${{ matrix.index }}.json --test --nanosp --skip_setup --output_file test_nanosp_${{ matrix.index }}.json --logs_file log_nanosp.txt
 
       - name: Launch test nanox
         if: ${{ env.test_nanox }}
-        run: | 
-          python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_nanox }} --input_file input_${{ matrix.index }}.json --test --nanox --skip_setup --output_file test_nanox_${{ matrix.index }}.json --logs_file log_nanox.txt
+        run: |
+          python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_nanox }} --input_file input_${{ matrix.index }}.json --test --nanox --skip_setup --output_file test_nanox_${{ matrix.index }}.json --logs_file log_nanox.txt
 
       - name: Launch test stax
         if: ${{ env.test_stax }}
-        run: | 
-          python scripts/build_and_test/main.py --sdk_ref ${{ env.api_level_stax }} --input_file input_${{ matrix.index }}.json --test --stax --skip_setup --output_file test_stax_${{ matrix.index }}.json --logs_file log_stax.txt
+        run: |
+          python scripts/entrypoint.py build_and_test --sdk_ref ${{ env.api_level_stax }} --input_file input_${{ matrix.index }}.json --test --stax --skip_setup --output_file test_stax_${{ matrix.index }}.json --logs_file log_stax.txt
 
       - name: Merge output files
-        run: | 
-          python scripts/output_scripts/merge.py --input_pattern "test_*.json" --output_file test_output_${{ matrix.index }}.json --key "test"
+        run: |
+          python scripts/entrypoint.py merge_output --input_pattern "test_*.json" --output_file test_output_${{ matrix.index }}.json --key "test"
       - name: Archive output file
         uses: actions/upload-artifact@v3
         with:
@@ -339,8 +339,8 @@ jobs:
           name: test_output_10.json
 
       - name: Merge output files
-        run: | 
-          python scripts/output_scripts/merge.py --input_pattern "test_output_*.json" --output_file test_output.json --key "test"
+        run: |
+          python scripts/entrypoint.py merge_output --input_pattern "test_output_*.json" --output_file test_output.json --key "test"
 
       - name: Archive output file
         uses: actions/upload-artifact@v3
@@ -351,15 +351,15 @@ jobs:
 
       - name: Convert to markdown
         run: |
-          python scripts/output_scripts/convert.py --input_file test_output.json --output_file out.md --key test
+          python scripts/entrypoint.py convert_output --input_file test_output.json --output_file out.md --key test
           cat out.md >> $GITHUB_STEP_SUMMARY
 
       - name: Echo GHA url 
         run: echo "url"=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} >> $GITHUB_ENV
 
       - name: Convert to slack json
         run: |
-          python scripts/output_scripts/slack.py --input_file test_output.json --output_file slack.json --key test --url ${{ env.url }}
+          python scripts/entrypoint.py slack_output --input_file test_output.json --output_file slack.json --key test --url ${{ env.url }}
 
       - name: Send custom JSON data to Slack workflow
         if: ${{ github.event_name == 'schedule' || inputs.send_to_slack == true }}
@@ -372,4 +372,4 @@ jobs:
 
       - name: Set job status
         run: |
-          python scripts/output_scripts/status.py --input_file test_output.json --key test
+          python scripts/entrypoint.py status_output --input_file test_output.json --key test

README.md

@@ -41,7 +41,7 @@ file generated previously.
 - `scan_all.yml`: Scans for selected devices
 - `refresh_inputs.yml`: Check whether the input list needs updating.
 
-To reduce CI run times, the input file is automatically splitted into 10 sub-inputs, and then all the inputs are run through a matrix strategy.
+To reduce CI run times, the input file is automatically split into 10 sub-inputs, and then all the inputs are run through a matrix strategy.
 ### 3. Planned Improvements
 
 - **Support for ZEMU Tests**
@@ -58,5 +58,5 @@ Alternatively you can run the script from the actions tab of the repo.
 You can view the result in the summary of the GH action: 
 :red_circle: means a fail.
 :heavy_check_mark: means success,
-:fast_forward: means the action was explicitely blacklisted in the input file.
+:fast_forward: means the action was explicitly blacklisted in the input file.
 nothing: means there was no variant associated to the device when running make listvariants.

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.mypy]
+ignore_missing_imports = true