From 652bb82603c5f54b611b8a705123bc1e526b07d1 Mon Sep 17 00:00:00 2001 From: Ramon Roche Date: Wed, 25 Sep 2024 07:41:52 -0700 Subject: [PATCH] ci: Upload Artifacts to S3 - replaces metadata and deploy workflows - move ci scripts under Tools/ci --- .github/workflows/build_all_targets.yml | 66 +++++++-- .github/workflows/deploy_all.yml | 56 -------- .github/workflows/metadata.yml | 133 ------------------ .../build_all_runner.sh} | 0 Tools/{ => ci}/generate_board_targets_json.py | 53 ++++--- Tools/ci/package_build_artifacts.sh | 46 ++++++ 6 files changed, 135 insertions(+), 219 deletions(-) delete mode 100644 .github/workflows/deploy_all.yml delete mode 100644 .github/workflows/metadata.yml rename Tools/{ci_build_all_runner.sh => ci/build_all_runner.sh} (100%) rename Tools/{ => ci}/generate_board_targets_json.py (86%) create mode 100755 Tools/ci/package_build_artifacts.sh diff --git a/.github/workflows/build_all_targets.yml b/.github/workflows/build_all_targets.yml index 4ace13de52a5..dd912872fd30 100644 --- a/.github/workflows/build_all_targets.yml +++ b/.github/workflows/build_all_targets.yml @@ -32,6 +32,7 @@ jobs: matrix: ${{ steps.set-matrix.outputs.matrix }} timestamp: ${{ steps.set-timestamp.outputs.timestamp }} tagname: ${{ steps.set-tag.outputs.tagname }} + branchname: ${{ steps.set-branch.outputs.branchname }} steps: - uses: actions/checkout@v4 @@ -41,7 +42,7 @@ jobs: path: "./Tools/setup/requirements.txt" - id: set-matrix - run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py --group)" + run: echo "::set-output name=matrix::$(./Tools/ci/generate_board_targets_json.py --group)" - id: set-timestamp run: echo "::set-output name=timestamp::$(date +"%Y%m%d%H%M%S")" @@ -60,6 +61,13 @@ jobs: if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' run: echo "::set-output name=tagname::${{ github.event_name == 'workflow_dispatch' && inputs.tag || github.ref_name }}" + - id: set-branch + run: echo "::set-output name=branchname::${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" + + - name: Debug Matrix Output + if: runner.debug == '1' + run: echo "$(./Tools/ci/generate_board_targets_json.py --group --verbose)" + setup: name: Build Group [${{ matrix.group }}] # runs-on: ubuntu-latest @@ -74,17 +82,17 @@ jobs: with: fetch-depth: 0 - - name: ownership workaround + - name: Git ownership workaround run: git config --system --add safe.directory '*' - - name: ccache setup keys + - name: Setup ccache uses: actions/cache@v4 with: path: ~/.ccache key: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }} restore-keys: ${{ matrix.group }}-ccache-${{ needs.group_targets.outputs.timestamp }} - - name: setup ccache + - name: Configure ccache run: | mkdir -p ~/.ccache echo "base_dir = ${GITHUB_WORKSPACE}" > ~/.ccache/ccache.conf @@ -95,24 +103,54 @@ jobs: ccache -s ccache -z - - name: building [${{ matrix.group }}] + - name: Building [${{ matrix.group }}] + run: | + ./Tools/ci/build_all_runner.sh ${{matrix.targets}} + + - name: Arrange Build Artifacts run: | - ./Tools/ci_build_all_runner.sh ${{matrix.targets}} + ./Tools/ci/package_build_artifacts.sh - - name: Upload px4 package + - name: Upload Build Artifacts uses: actions/upload-artifact@v4 with: name: px4_${{matrix.group}}_build_artifacts - path: | - build/**/*.px4 - build/**/*.bin - compression-level: 0 + path: artifacts/ - - name: ccache post-run + - name: Cache Save run: ccache -s + artifacts: + name: Upload Artifacts to S3 + # runs-on: ubuntu-latest + runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}"] + needs: [setup, group_targets] + if: contains(fromJSON('["main", "stable", "beta"]'), needs.group_targets.outputs.branchname) + steps: + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts/ + merge-multiple: true + + - name: Branch Name + run: | + echo "${{ needs.group_targets.outputs.branchname }}" + + - name: Uploading Artifacts to S3 [${{ needs.group_targets.outputs.branchname == 'main' && 'master' || needs.group_targets.outputs.branchname }}] + uses: jakejarvis/s3-sync-action@master + with: + args: --acl public-read + env: + AWS_S3_BUCKET: 'px4-travis' + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: 'us-west-1' + SOURCE_DIR: artifacts/ + DEST_DIR: Firmware/${{ needs.group_targets.outputs.branchname == 'main' && 'master' || needs.group_targets.outputs.branchname }}/ + release: - name: Release and Upload + name: Create Release and Upload Artifacts # runs-on: ubuntu-latest runs-on: [runs-on,runner=1cpu-linux-x64,image=ubuntu22-full-x64,"run-id=${{ github.run_id }}"] needs: [setup, group_targets] @@ -126,7 +164,7 @@ jobs: mkdir artifacts cp **/**/*.px4 artifacts/ - - name: Upload Binaries + - name: Upload Binaries to Release uses: softprops/action-gh-release@v2 with: name: ${{ needs.group_targets.outputs.tagname }} diff --git a/.github/workflows/deploy_all.yml b/.github/workflows/deploy_all.yml deleted file mode 100644 index 3f1a4c0049b1..000000000000 --- a/.github/workflows/deploy_all.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Deploy metadata for all targets - -on: - push: - branches: - - 'main' - - 'release/*' - - 'pr-metadata-test' - -jobs: - enumerate_targets: - runs-on: ubuntu-latest - container: px4io/px4-dev-base-focal:2021-09-08 - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v1 - with: - token: ${{secrets.ACCESS_TOKEN}} - - id: set-matrix - run: echo "::set-output name=matrix::$(./Tools/generate_board_targets_json.py)" - build: - runs-on: ubuntu-latest - needs: enumerate_targets - strategy: - matrix: ${{fromJson(needs.enumerate_targets.outputs.matrix)}} - container: ${{ matrix.container }} - steps: - - uses: actions/checkout@v1 - with: - token: ${{secrets.ACCESS_TOKEN}} - - - name: ownership workaround - run: git config --system --add safe.directory '*' - - - name: make ${{matrix.target}} - run: make ${{matrix.target}} - - - name: parameter & events metadata - run: | - make ${{matrix.target}} ver_gen events_json actuators_json - ./src/lib/version/get_git_tag_or_branch_version.sh build/${{ matrix.target }} >> $GITHUB_ENV - cd build/${{ matrix.target }} - mkdir _metadata || true - cp parameters.* events/*.xz actuators.json* _metadata - - - uses: jakejarvis/s3-sync-action@master - with: - args: --acl public-read - env: - AWS_S3_BUCKET: 'px4-travis' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: 'us-west-1' - SOURCE_DIR: 'build/${{ matrix.target }}/_metadata/' - DEST_DIR: 'Firmware/${{ env.version }}/${{ matrix.target }}/' diff --git a/.github/workflows/metadata.yml b/.github/workflows/metadata.yml deleted file mode 100644 index bdcf17859090..000000000000 --- a/.github/workflows/metadata.yml +++ /dev/null @@ -1,133 +0,0 @@ -name: Metadata - -on: - push: - branches: - - 'main' - - 'release/*' - - 'pr-metadata-test' - -jobs: - - airframe: - runs-on: ubuntu-latest - container: px4io/px4-dev-base-focal:2021-09-08 - steps: - - uses: actions/checkout@v1 - with: - token: ${{ secrets.ACCESS_TOKEN }} - - - name: airframe metadata - run: | - make airframe_metadata - ./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV - cd build/px4_sitl_default/docs - # TODO: deploy to userguide gitbook - - - uses: jakejarvis/s3-sync-action@master - with: - args: --acl public-read - env: - AWS_S3_BUCKET: 'px4-travis' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: 'us-west-1' - SOURCE_DIR: 'build/px4_sitl_default/docs/' - DEST_DIR: 'Firmware/${{ env.version }}/_general/' - - module: - runs-on: ubuntu-latest - container: px4io/px4-dev-base-focal:2021-09-08 - steps: - - uses: actions/checkout@v1 - with: - token: ${{ secrets.ACCESS_TOKEN }} - - - name: module documentation - run: | - make module_documentation - cd build/px4_sitl_default/docs - ls -ls * - # TODO: deploy to userguide gitbook and s3 - - parameter: - runs-on: ubuntu-latest - container: px4io/px4-dev-base-focal:2021-09-08 - steps: - - uses: actions/checkout@v1 - with: - token: ${{ secrets.ACCESS_TOKEN }} - - - name: parameter metadata - run: | - make parameters_metadata - ./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV - - - uses: jakejarvis/s3-sync-action@master - with: - args: --acl public-read - env: - AWS_S3_BUCKET: 'px4-travis' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: 'us-west-1' - SOURCE_DIR: 'build/px4_sitl_default/docs/' - DEST_DIR: 'Firmware/${{ env.version }}/_general/' - - events: - runs-on: ubuntu-latest - container: px4io/px4-dev-base-focal:2021-09-08 - steps: - - uses: actions/checkout@v1 - with: - token: ${{ secrets.ACCESS_TOKEN }} - - - name: events metadata - run: | - make extract_events - ./src/lib/version/get_git_tag_or_branch_version.sh build/px4_sitl_default >> $GITHUB_ENV - cd build/px4_sitl_default - mkdir _events_full || true - cp events/all_events_full.json.xz _events_full/all_events.json.xz - - - uses: jakejarvis/s3-sync-action@master - with: - args: --acl public-read - env: - AWS_S3_BUCKET: 'px4-travis' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_REGION: 'us-west-1' - SOURCE_DIR: 'build/px4_sitl_default/_events_full/' - DEST_DIR: 'Firmware/${{ env.version }}/_general/' - - uorb_graph: - runs-on: ubuntu-latest - container: px4io/px4-dev-nuttx-focal:2022-08-12 - steps: - - uses: actions/checkout@v1 - with: - token: ${{ secrets.ACCESS_TOKEN }} - - - name: uORB graph - run: | - make uorb_graphs - cd Tools/uorb_graph - ls -ls * - # TODO: deploy graph_px4_sitl.json to S3 px4-travis:Firmware/master/ - - ROS2_msgs: - runs-on: ubuntu-latest - container: px4io/px4-dev-base-focal:2021-09-08 - steps: - - uses: actions/checkout@v1 - with: - token: ${{ secrets.ACCESS_TOKEN }} - - - name: PX4 ROS2 msgs - run: | - git clone https://github.com/PX4/px4_msgs.git - rm px4_msgs/msg/*.msg - rm px4_msgs/srv/*.srv - cp msg/*.msg px4_msgs/msg/ - cp srv/*.srv px4_msgs/srv/ diff --git a/Tools/ci_build_all_runner.sh b/Tools/ci/build_all_runner.sh similarity index 100% rename from Tools/ci_build_all_runner.sh rename to Tools/ci/build_all_runner.sh diff --git a/Tools/generate_board_targets_json.py b/Tools/ci/generate_board_targets_json.py similarity index 86% rename from Tools/generate_board_targets_json.py rename to Tools/ci/generate_board_targets_json.py index 649b3fceefa6..f453d62c0797 100755 --- a/Tools/generate_board_targets_json.py +++ b/Tools/ci/generate_board_targets_json.py @@ -111,7 +111,19 @@ def process_target(px4board_file, target_name): print("= scanning for boards =") print("=======================") -for manufacturer in os.scandir(os.path.join(source_dir, 'boards')): +# We also need to build metadata +# includes: +# - Airframe +# - Parameters +# - Events +metadata_targets = ['airframe_metadata', 'parameters_metadata', 'extract_events'] +grouped_targets['base'] = {} +grouped_targets['base']['container'] = 'px4io/px4-dev-base-focal:2021-09-08' +grouped_targets['base']['manufacturers'] = {} +grouped_targets['base']['manufacturers']['px4'] = [] +grouped_targets['base']['manufacturers']['px4'] += metadata_targets + +for manufacturer in os.scandir(os.path.join(source_dir, '../boards')): if not manufacturer.is_dir(): continue if manufacturer.name in excluded_manufacturers: @@ -143,7 +155,6 @@ def process_target(px4board_file, target_name): grouped_targets[target['arch']]['container'] = target['container'] grouped_targets[target['arch']]['manufacturers'] = {} if(manufacturer.name not in grouped_targets[target['arch']]['manufacturers']): - grouped_targets[target['arch']]['manufacturers'][manufacturer.name] = {} grouped_targets[target['arch']]['manufacturers'][manufacturer.name] = [] grouped_targets[target['arch']]['manufacturers'][manufacturer.name].append(target_name) if target is not None: @@ -156,6 +167,11 @@ def process_target(px4board_file, target_name): print("============================") pprint.pp(grouped_targets) +if(verbose): + print("===================") + print("= Generating JSON =") + print("===================") + if (args.group): # if we are using this script for grouping builds # we loop trough the manufacturers list and split their targets @@ -179,16 +195,24 @@ def process_target(px4board_file, target_name): last_arch = '' SPLIT_LIMIT = 10 LOWER_LIMIT = 5 + if(verbose): + print(f'=:Architectures: [{grouped_targets.keys()}]') for arch in grouped_targets: + if(verbose): + print(f'=:Processing: [{arch}] Last: [{last_arch}]') + if(last_arch == ''): last_arch = arch if(arch not in group_number): group_number[arch] = 0 if(last_arch != arch and len(temp_group) > 0): + group_name = last_arch + "-" + str(group_number[last_arch]) group_number[last_arch] += 1 targets = comma_targets(temp_group) + if(verbose): + print(f'=:Orphan: [{arch}][{last_arch}][{targets}]') final_groups.append({ "container": grouped_targets[last_arch]['container'], "targets": targets, @@ -198,12 +222,22 @@ def process_target(px4board_file, target_name): }) last_arch = arch temp_group = [] + for man in grouped_targets[arch]['manufacturers']: + if(verbose): + print(f'=:Processing: [{arch}][{man}]') for tar in grouped_targets[arch]['manufacturers'][man]: + man_len = len(grouped_targets[arch]['manufacturers'][man]) + if(verbose): + print(f'=:Processing: [{arch}][{man}][{man_len}][{tar}]') if(last_man != man): - man_len = len(grouped_targets[arch]['manufacturers'][man]) + # if(verbose): + # print(f'=:Processing: [{arch}][{man}][{tar}][{man_len}]') if(man_len > LOWER_LIMIT and man_len < (SPLIT_LIMIT + 1)): # Manufacturers can have their own group + if(verbose): + print(f'=:Processing: ==Manufacturers can have their own group') + print(f'=:Processing: Limits[{LOWER_LIMIT}][{SPLIT_LIMIT}]') group_name = arch + "-" + man targets = comma_targets(grouped_targets[arch]['manufacturers'][man]) last_man = man @@ -234,19 +268,6 @@ def process_target(px4board_file, target_name): else: temp_group.append(tar) - if(last_arch != arch and len(temp_group) > 0): - group_name = last_arch + "-" + str(group_number[last_arch]) - group_number[last_arch] += 1 - targets = comma_targets(temp_group) - final_groups.append({ - "container": grouped_targets[last_arch]['container'], - "targets": targets, - "arch": last_arch, - "group": group_name, - "len": len(temp_group) - }) - last_arch = arch - temp_group = [] if(len(temp_group) > (LOWER_LIMIT - 1)): group_name = arch + "-" + str(group_number[arch]) last_arch = arch diff --git a/Tools/ci/package_build_artifacts.sh b/Tools/ci/package_build_artifacts.sh new file mode 100755 index 000000000000..a665e295df71 --- /dev/null +++ b/Tools/ci/package_build_artifacts.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +mkdir artifacts +cp **/**/*.px4 artifacts/ +cp **/**/*.elf artifacts/ +for build_dir_path in build/*/ ; do + build_dir=${build_dir_path#*/} + build_dir=${build_dir::${#build_dir}-1} + mkdir artifacts/$build_dir + find artifacts/ -maxdepth 1 -type f -name "*$build_dir*" + # Airframe + cp $build_dir_path/airframes.xml artifacts/$build_dir/ + # Parameters + cp $build_dir_path/parameters.xml artifacts/$build_dir/ + cp $build_dir_path/parameters.json artifacts/$build_dir/ + cp $build_dir_path/parameters.json.xz artifacts/$build_dir/ + # Actuators + cp $build_dir_path/actuators.json artifacts/$build_dir/ + cp $build_dir_path/actuators.json.xz artifacts/$build_dir/ + # Events + cp $build_dir_path/events/all_events.json.xz artifacts/$build_dir/ + # ROS 2 msgs + cp $build_dir_path/events/all_events.json.xz artifacts/$build_dir/ + # Module Docs + ls -la artifacts/$build_dir + echo "----------" +done + +# general metadata +mkdir artifacts/_general/ +cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/ +# Airframe +cp artifacts/px4_sitl_default/airframes.xml artifacts/_general/ +# Parameters +cp artifacts/px4_sitl_default/parameters.xml artifacts/_general/ +cp artifacts/px4_sitl_default/parameters.json artifacts/_general/ +cp artifacts/px4_sitl_default/parameters.json.xz artifacts/_general/ +# Actuators +cp artifacts/px4_sitl_default/actuators.json artifacts/_general/ +cp artifacts/px4_sitl_default/actuators.json.xz artifacts/_general/ +# Events +cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/ +# ROS 2 msgs +cp artifacts/px4_sitl_default/events/all_events.json.xz artifacts/_general/ +# Module Docs +ls -la artifacts/_general/