From 8786a0d7caf1650ab3ff22ae720b0204ddfc49d6 Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Fri, 10 May 2024 08:37:45 -0400 Subject: [PATCH] Update CI base image to pulp-ci-centos9 [noissue] --- .ci/scripts/tweet.py | 16 -- .github/template_gitref | 2 +- .github/workflows/build.yml | 4 +- .github/workflows/changelog.yml | 2 +- .github/workflows/ci.yml | 7 +- .github/workflows/create-branch.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/nightly.yml | 14 +- .github/workflows/publish.yml | 28 +-- .github/workflows/release.yml | 2 +- .../scripts/create_release_from_tag.sh | 11 -- .github/workflows/scripts/install.sh | 2 +- .github/workflows/scripts/release.py | 177 ------------------ .github/workflows/test.yml | 16 +- .github/workflows/update-labels.yml | 2 +- .github/workflows/update_ci.yml | 2 +- pulp_rpm/tests/functional/api/test_sync.py | 12 +- template_config.yml | 2 +- 18 files changed, 52 insertions(+), 251 deletions(-) delete mode 100755 .ci/scripts/tweet.py delete mode 100755 .github/workflows/scripts/create_release_from_tag.sh delete mode 100755 .github/workflows/scripts/release.py diff --git a/.ci/scripts/tweet.py b/.ci/scripts/tweet.py deleted file mode 100755 index 9959bcc15..000000000 --- a/.ci/scripts/tweet.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -import sys -from tweepy import Client - -release_version = sys.argv[1] -if release_version.endswith(".0"): - client = Client( - consumer_key=os.getenv("TWITTER_API_KEY"), - consumer_secret=os.getenv("TWITTER_API_KEY_SECRET"), - access_token=os.getenv("TWITTER_ACCESS_TOKEN"), - access_token_secret=os.getenv("TWITTER_ACCESS_TOKEN_SECRET"), - ) - link = "https://docs.pulpproject.org/pulp_rpm/changes.html" - msg = f"pulp_rpm-{release_version} - Check out for more details: {link}" - release_msg = f"Hey! We've just released {msg}" - client.create_tweet(text=release_msg) diff --git a/.github/template_gitref b/.github/template_gitref index 418b21cd4..2232d657d 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-327-g6a0d984 +2021.08.26-330-g0b7d0c1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e0022ca8..c97609c7d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,7 +23,7 @@ jobs: with: fetch-depth: 1 path: "pulp_rpm" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" @@ -36,7 +36,7 @@ jobs: python3 setup.py sdist bdist_wheel --python-tag py3 twine check dist/* - name: "Upload Package whl" - uses: "actions/upload-artifact@v3" + uses: "actions/upload-artifact@v4" with: name: "plugin_package" path: "pulp_rpm/dist/" diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 5059d7eb6..38c05d228 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -28,7 +28,7 @@ jobs: with: fetch-depth: 1 - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7a4a8f58..f9d441172 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: with: fetch-depth: 0 path: "pulp_rpm" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Install python dependencies" @@ -67,10 +67,11 @@ jobs: mkdir -p "pulp_rpm" working-directory: "." - name: "Download Deprecations" - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: - name: "deprecations" + pattern: "deprecations-*" path: "pulp_rpm" + merge-multiple: true - name: "Print deprecations" run: | cat deprecations-*.txt | sort -u diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 18cb5b44e..faa6a389c 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -26,7 +26,7 @@ jobs: fetch-depth: 0 path: "pulp_rpm" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 75975dd22..d8d920cbd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -24,7 +24,7 @@ jobs: fetch-depth: 1 path: "pulp_rpm" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0a37e5298..44409274a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -38,7 +38,7 @@ jobs: fetch-depth: 0 path: "pulp_rpm" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -76,12 +76,12 @@ jobs: fetch-depth: 1 path: "pulp_rpm" - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: "plugin_package" path: "pulp_rpm/dist/" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -96,13 +96,13 @@ jobs: echo "TEST=${{ matrix.env.TEST }}" >> $GITHUB_ENV - name: Download built docs - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: "docs.tar" path: "pulp_rpm" - name: Download Python client docs - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: "python-client-docs.tar" path: "pulp_rpm" @@ -145,12 +145,12 @@ jobs: repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: "plugin_package" path: "pulp_rpm/dist/" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c2ab82caa..c3bbeeb62 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,11 +39,11 @@ jobs: repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - - uses: "actions/download-artifact@v3" + - uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp_rpm/dist/" @@ -89,20 +89,20 @@ jobs: .github/workflows/scripts/install_ruby_client.sh shell: "bash" - name: "Upload python client packages" - uses: "actions/upload-artifact@v3" + uses: "actions/upload-artifact@v4" with: name: "python-client.tar" path: "pulp_rpm/rpm-python-client.tar" if-no-files-found: "error" - name: "Upload python client docs" - uses: "actions/upload-artifact@v3" + uses: "actions/upload-artifact@v4" with: name: "python-client-docs.tar" path: "pulp_rpm/rpm-python-client-docs.tar" if-no-files-found: "error" - name: "Upload ruby client packages" - uses: "actions/upload-artifact@v3" + uses: "actions/upload-artifact@v4" with: name: "ruby-client.tar" path: "pulp_rpm/rpm-ruby-client.tar" @@ -115,7 +115,7 @@ jobs: tar -cvf docs/docs.tar docs/_build - name: "Upload built docs" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "docs.tar" path: "pulp_rpm/docs/docs.tar" @@ -145,12 +145,12 @@ jobs: fetch-depth: 1 path: "pulp_rpm" - - uses: "actions/download-artifact@v3" + - uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp_rpm/dist/" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -184,7 +184,7 @@ jobs: path: "pulp_rpm" - name: "Download Python client" - uses: "actions/download-artifact@v3" + uses: "actions/download-artifact@v4" with: name: "python-client.tar" path: "pulp_rpm/" @@ -193,7 +193,7 @@ jobs: run: | tar -xvf rpm-python-client.tar - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -227,7 +227,7 @@ jobs: path: "pulp_rpm" - name: "Download Ruby client" - uses: "actions/download-artifact@v3" + uses: "actions/download-artifact@v4" with: name: "ruby-client.tar" path: "pulp_rpm/" @@ -263,7 +263,7 @@ jobs: fetch-depth: 1 path: "pulp_rpm" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" @@ -280,13 +280,13 @@ jobs: SECRETS_CONTEXT: "${{ toJson(secrets) }}" - name: "Download built docs" - uses: "actions/download-artifact@v3" + uses: "actions/download-artifact@v4" with: name: "docs.tar" path: "pulp_rpm/" - name: "Download Python client docs" - uses: "actions/download-artifact@v3" + uses: "actions/download-artifact@v4" with: name: "python-client-docs.tar" path: "pulp_rpm/" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 24d63f60f..a37e75411 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: path: "pulp_rpm" token: ${{ secrets.RELEASE_TOKEN }} - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" diff --git a/.github/workflows/scripts/create_release_from_tag.sh b/.github/workflows/scripts/create_release_from_tag.sh deleted file mode 100755 index 4de7777d7..000000000 --- a/.github/workflows/scripts/create_release_from_tag.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh -set -eu - -curl -s -X "POST https://api.github.com/repos/$GITHUB_REPOSITORY/releases" \ --H "Authorization: token $RELEASE_TOKEN" \ --d @- << EOF -{ - "tag_name": "$1", - "name": "$1" -} -EOF diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index bbf701e32..ec0eead39 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -79,7 +79,7 @@ cat >> vars/main.yaml << VARSYAML pulp_env: {} pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "orphan_protection_time": 0} pulp_scheme: https -pulp_default_container: ghcr.io/pulp/pulp-ci-centos:latest +pulp_default_container: ghcr.io/pulp/pulp-ci-centos9:latest VARSYAML SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "generate-bindings" "lowerbounds") diff --git a/.github/workflows/scripts/release.py b/.github/workflows/scripts/release.py deleted file mode 100755 index 656484baa..000000000 --- a/.github/workflows/scripts/release.py +++ /dev/null @@ -1,177 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_rpm' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -import argparse -import re -import os -import textwrap -import requests -import subprocess - -from git import Repo -from pathlib import Path - - -def get_package_from_pypi(version, plugin_path): - """ - Download a package from PyPI. - - :param version: version of the package to download from PyPI - :return: True/False if download was successful - """ - os.makedirs(os.path.join(plugin_path, "dist"), exist_ok=True) - r = requests.get(f"https://pypi.org/pypi/pulp-rpm/{version}/json") - if r.status_code == 200: - metadata = r.json() - for url_data in metadata["urls"]: - filename = url_data["filename"] - r2 = requests.get(url_data["url"]) - if r2.status_code != 200: - raise RuntimeError(f"Failed to download released artifact {filename}") - with open(os.path.join(plugin_path, "dist", filename), "wb") as f: - f.write(r2.content) - return True - return False - - -def create_release_commits(repo, release_version, plugin_path): - """Build changelog, set version, commit, bump to next dev version, commit.""" - issues_to_close = set() - for filename in Path(f"{plugin_path}/CHANGES").rglob("*"): - if filename.stem.isdigit(): - issue = filename.stem - issues_to_close.add(issue) - - issues = ",".join(issues_to_close) - # First commit: changelog - os.system(f"towncrier build --yes --version {release_version}") - git = repo.git - git.add("CHANGES.rst") - git.add("CHANGES/*") - git.commit("-m", f"Add changelog for {release_version}\n\n[noissue]") - - # Second commit: release version - os.system("bump2version release --allow-dirty") - - git.add(f"{plugin_path}") - git.commit("-m", f"Release {release_version}\nGH Issues: {issues}\n\n[noissue]") - sha = repo.head.object.hexsha - short_sha = git.rev_parse(sha, short=7) - - os.system("bump2version patch --allow-dirty") - - new_dev_version = None - with open(f"{plugin_path}/setup.py") as fp: - for line in fp.readlines(): - if "version=" in line: - new_dev_version = re.split("\"|'", line)[1] - if not new_dev_version: - raise RuntimeError("Could not detect new dev version ... aborting.") - - git.add(f"{plugin_path}") - git.commit("-m", f"Bump to {new_dev_version}\n\n[noissue]") - print(f"Release commit == {short_sha}") - print(f"All changes were committed on branch: release_{release_version}") - return sha - - -def create_tag_and_build_package(repo, desired_tag, commit_sha, plugin_path): - """Create a tag if one is needed and build a package if one is not on PyPI.""" - # Remove auth header config - with repo.config_writer() as conf: - conf.remove_section('http "https://github.com/"') - conf.release() - - # Determine if a tag exists and if it matches the specified commit sha - tag = None - for existing_tag in repo.tags: - if existing_tag.name == desired_tag: - if existing_tag.commit.hexsha == commit_sha: - tag = existing_tag - else: - raise RuntimeError( - f"The '{desired_tag}' tag already exists, but the commit sha does not match " - f"'{commit_sha}'." - ) - - # Create a tag if one does not exist - if not tag: - tag = repo.create_tag(desired_tag, ref=commit_sha) - - # Checkout the desired tag and reset the tree - repo.head.reference = tag.commit - repo.head.reset(index=True, working_tree=True) - - # Check if Package is available on PyPI - if not get_package_from_pypi(tag.name, plugin_path): - os.system("python3 setup.py sdist bdist_wheel --python-tag py3") - - -def main(): - helper = textwrap.dedent( - """\ - Start the release process. - - Example: - setup.py on plugin before script: - version="2.0.0.dev" - - $ python .ci/scripts/release.py - - setup.py on plugin after script: - version="2.0.1.dev" - - - """ - ) - parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, description=helper - ) - - parser.add_argument( - "release_version", - type=str, - help="The version string for the release.", - nargs="?", - ) - - args = parser.parse_args() - - release_version_arg = args.release_version - - release_path = os.path.dirname(os.path.abspath(__file__)) - plugin_path = release_path.split("/.github")[0] - - output = subprocess.check_output(["bump2version", "--dry-run", "--list", "release"]) - release_version = re.findall(r"\nnew_version=([0-9.]*)\n", output.decode())[0] - - print(f"\n\nRepo path: {plugin_path}") - repo = Repo(plugin_path) - - release_commit = None - if release_version_arg and release_version != release_version_arg: - # Look for a commit with the requested release version - for commit in repo.iter_commits(): - if f"Release {release_version_arg}\n" in commit.message: - release_commit = commit - release_version = release_version_arg - break - if not release_commit: - raise RuntimeError( - f"The release version {release_version_arg} does not match the .dev version at " - "HEAD. A release commit for such version does not exist." - ) - - if not release_commit: - release_commit_sha = create_release_commits(repo, release_version, plugin_path) - else: - release_commit_sha = release_commit.hexsha - create_tag_and_build_package(repo, release_version, release_commit_sha, plugin_path) - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67e08b222..8793eb54c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,11 +39,11 @@ jobs: repository: "pulp/pulp-openapi-generator" path: "pulp-openapi-generator" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - - uses: "actions/download-artifact@v3" + - uses: "actions/download-artifact@v4" with: name: "plugin_package" path: "pulp_rpm/dist/" @@ -119,15 +119,15 @@ jobs: docker logs pulp 2>&1 | grep -i pulpcore.deprecation | tee deprecations-${{ matrix.env.TEST }}.txt - name: "Upload Deprecations" - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: "deprecations" + name: "deprecations-${{ matrix.env.TEST }}" path: "pulp_rpm/deprecations-${{ matrix.env.TEST }}.txt" if-no-files-found: "error" retention-days: 5 - name: Upload python client packages if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "python-client.tar" path: "pulp_rpm/rpm-python-client.tar" @@ -136,7 +136,7 @@ jobs: - name: Upload python client docs if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "python-client-docs.tar" path: "pulp_rpm/rpm-python-client-docs.tar" @@ -144,7 +144,7 @@ jobs: retention-days: 5 - name: Upload Ruby client if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "ruby-client.tar" path: "pulp_rpm/rpm-ruby-client.tar" @@ -152,7 +152,7 @@ jobs: retention-days: 5 - name: Upload built docs if: ${{ env.TEST == 'docs' }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "docs.tar" path: "pulp_rpm/docs/docs.tar" diff --git a/.github/workflows/update-labels.yml b/.github/workflows/update-labels.yml index b0ebe132c..84415987c 100644 --- a/.github/workflows/update-labels.yml +++ b/.github/workflows/update-labels.yml @@ -19,7 +19,7 @@ jobs: update_backport_labels: runs-on: "ubuntu-latest" steps: - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" - name: "Configure Git with pulpbot name and email" diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 0c1f31289..e01e2f19c 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -29,7 +29,7 @@ jobs: repository: "pulp/plugin_template" path: "plugin_template" - - uses: "actions/setup-python@v4" + - uses: "actions/setup-python@v5" with: python-version: "3.11" diff --git a/pulp_rpm/tests/functional/api/test_sync.py b/pulp_rpm/tests/functional/api/test_sync.py index 9422a9f37..2215fd099 100644 --- a/pulp_rpm/tests/functional/api/test_sync.py +++ b/pulp_rpm/tests/functional/api/test_sync.py @@ -940,7 +940,9 @@ def test_core_metadata(init_and_sync, rpm_package_api): # TODO: figure out how to un-ignore "time_file" without breaking the tests diff = dictdiffer.diff( - package, RPM_COMPLEX_PACKAGE_DATA, ignore={"time_file", "pulp_created", "pulp_href"} + package, + RPM_COMPLEX_PACKAGE_DATA, + ignore={"time_file", "pulp_created", "pulp_last_updated", "pulp_href"}, ) assert list(diff) == [], list(diff) @@ -1034,15 +1036,17 @@ def module_obsolete_key(m): RPM_MODULEMD_OBSOLETES_DATA.sort(key=module_obsolete_key) for m1, m2 in zip(modules, RPM_MODULEMDS_DATA): - diff = dictdiffer.diff(m1, m2, ignore={"packages", "pulp_created", "pulp_href"}) + diff = dictdiffer.diff( + m1, m2, ignore={"packages", "pulp_created", "pulp_last_updated", "pulp_href"} + ) assert list(diff) == [], list(diff) for m1, m2 in zip(module_defaults, RPM_MODULEMD_DEFAULTS_DATA): - diff = dictdiffer.diff(m1, m2, ignore={"pulp_created", "pulp_href"}) + diff = dictdiffer.diff(m1, m2, ignore={"pulp_created", "pulp_last_updated", "pulp_href"}) assert list(diff) == [], list(diff) for m1, m2 in zip(module_obsoletes, RPM_MODULEMD_OBSOLETES_DATA): - diff = dictdiffer.diff(m1, m2, ignore={"pulp_created", "pulp_href"}) + diff = dictdiffer.diff(m1, m2, ignore={"pulp_created", "pulp_last_updated", "pulp_href"}) assert list(diff) == [], list(diff) # assert all package from modular repo is marked as modular diff --git a/template_config.yml b/template_config.yml index 2becdfe1e..fbed40810 100644 --- a/template_config.yml +++ b/template_config.yml @@ -9,7 +9,7 @@ check_commit_message: true check_gettext: true check_manifest: true check_stray_pulpcore_imports: true -ci_base_image: ghcr.io/pulp/pulp-ci-centos +ci_base_image: ghcr.io/pulp/pulp-ci-centos9 ci_env: {} ci_trigger: '{pull_request: {branches: [''*'']}}' ci_update_docs: false