From ba1139f495b155ef4dadf573399fd85037ad824e Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Tue, 5 Sep 2023 13:22:48 +0200 Subject: [PATCH] Update CI [noissue] --- .ci/run_container.sh | 1 + .ci/scripts/collect_changes.py | 94 +++++++++++++++++++ .ci/scripts/create_release_branch.sh | 28 ++++++ .ci/scripts/release.sh | 26 +++++ .github/dependabot.yml | 16 ---- .github/workflows/codeql.yml | 39 +++++--- .github/workflows/collect_changes.yml | 32 +++++++ .github/workflows/lint.yml | 11 ++- .github/workflows/nightly.yml | 8 +- .github/workflows/pr.yml | 8 +- .github/workflows/publish.yml | 32 +++++++ .github/workflows/release.yml | 47 +++++----- .github/workflows/release_branch.yml | 30 ++++++ .github/workflows/test.yml | 8 +- lint_requirements.txt | 22 ++--- ...ments.txt => lower_bounds_constraints.lock | 0 pulp-glue-gem/pulp_glue/gem/context.py | 12 +-- 17 files changed, 332 insertions(+), 82 deletions(-) create mode 100755 .ci/scripts/collect_changes.py create mode 100755 .ci/scripts/create_release_branch.sh create mode 100755 .ci/scripts/release.sh create mode 100644 .github/workflows/collect_changes.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/release_branch.yml rename tests/lower_bounds_requirements.txt => lower_bounds_constraints.lock (100%) diff --git a/.ci/run_container.sh b/.ci/run_container.sh index d7898dd..3730ff0 100755 --- a/.ci/run_container.sh +++ b/.ci/run_container.sh @@ -5,6 +5,7 @@ # - https://github.com/pulp/pulp-cli/blob/main/.ci/run_container.sh # - https://github.com/pulp/pulp-cli-deb/blob/main/.ci/run_container.sh # - https://github.com/pulp/pulp-cli-gem/blob/main/.ci/run_container.sh +# - https://github.com/pulp/pulp-cli-maven/blob/main/.ci/run_container.sh # - https://github.com/pulp/pulp-cli-ostree/blob/main/.ci/run_container.sh # - https://github.com/pulp/squeezer/blob/develop/tests/run_container.sh diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py new file mode 100755 index 0000000..d11b528 --- /dev/null +++ b/.ci/scripts/collect_changes.py @@ -0,0 +1,94 @@ +import itertools +import os +import re + +import toml +from git import GitCommandError, Repo +from pkg_resources import parse_version + +# Read Towncrier settings +tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"] + +CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") +START_STRING = tc_settings.get( + "start_string", + "\n" + if CHANGELOG_FILE.endswith(".md") + else ".. towncrier release notes start\n", +) +TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})") + + +NAME_REGEX = r".*" +VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)" +DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}" +TITLE_REGEX = ( + "(" + + re.escape( + TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX") + ) + .replace("NAME_REGEX", NAME_REGEX) + .replace("VERSION_REGEX", VERSION_REGEX) + .replace("DATE_REGEX", DATE_REGEX) + + ")" +) + + +def get_changelog(repo, branch): + return repo.git.show(f"{branch}:{CHANGELOG_FILE}") + "\n" + + +def _tokenize_changes(splits): + assert len(splits) % 3 == 0 + for i in range(len(splits) // 3): + title = splits[3 * i] + version = parse_version(splits[3 * i + 1]) + yield [version, title + splits[3 * i + 2]] + + +def split_changelog(changelog): + preamble, rest = changelog.split(START_STRING, maxsplit=1) + split_rest = re.split(TITLE_REGEX, rest) + return preamble + START_STRING + split_rest[0], list(_tokenize_changes(split_rest[1:])) + + +def main(): + repo = Repo(os.getcwd()) + remote = repo.remotes[0] + branches = [ref for ref in remote.refs if re.match(r"^([0-9]+)\.([0-9]+)$", ref.remote_head)] + branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True) + branches = [ref.name for ref in branches] + + with open(CHANGELOG_FILE, "r") as f: + main_changelog = f.read() + preamble, main_changes = split_changelog(main_changelog) + old_length = len(main_changes) + + for branch in branches: + print(f"Looking at branch {branch}") + try: + changelog = get_changelog(repo, branch) + except GitCommandError: + print("No changelog found on this branch.") + continue + dummy, changes = split_changelog(changelog) + new_changes = sorted(main_changes + changes, key=lambda x: x[0], reverse=True) + # Now remove duplicates (retain the first one) + main_changes = [new_changes[0]] + for left, right in itertools.pairwise(new_changes): + if left[0] != right[0]: + main_changes.append(right) + + new_length = len(main_changes) + if old_length < new_length: + print(f"{new_length - old_length} new versions have been added.") + with open(CHANGELOG_FILE, "w") as fp: + fp.write(preamble) + for change in main_changes: + fp.write(change[1]) + + repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/create_release_branch.sh b/.ci/scripts/create_release_branch.sh new file mode 100755 index 0000000..c76bd62 --- /dev/null +++ b/.ci/scripts/create_release_branch.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eu -o pipefail + +BRANCH="$(git branch --show-current)" + +if ! [[ "${BRANCH}" = "main" ]] +then + echo ERROR: This is not the main branch! + exit 1 +fi + +NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')" + +if [[ -z "${NEW_BRANCH}" ]] +then + echo ERROR: Could not parse new version. + exit 1 +fi + +git branch "${NEW_BRANCH}" + +# Clean changelog snippets. +find CHANGES/ \( -name "*.feature" -o -name "*.bugfix" -o -name "*.doc" -o -name "*.translation" -o -name "*.devel" -o -name "*.misc" \) -exec git rm -f \{\} + + +bumpversion minor --commit --message $'Bump version to {new_version}\n\n[noissue]' --allow-dirty + +git push origin "${NEW_BRANCH}" diff --git a/.ci/scripts/release.sh b/.ci/scripts/release.sh new file mode 100755 index 0000000..f4f3e52 --- /dev/null +++ b/.ci/scripts/release.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eu -o pipefail + +BRANCH=$(git branch --show-current) + +if ! [[ "${BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]] +then + echo ERROR: This is not a release branch! + exit 1 +fi + +NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')" +echo "Release ${NEW_VERSION}" + +if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] +then + echo ERROR: Version does not match release branch + exit 1 +fi + +towncrier build --yes --version "${NEW_VERSION}" +bumpversion release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bumpversion patch --commit + +git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 45af2fc..fba8440 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,22 +7,6 @@ updates: open-pull-requests-limit: 10 commit-message: prefix: "[noissue]" -- package-ecosystem: pip - directory: "/pulp-glue-gem" - schedule: - interval: daily - open-pull-requests-limit: 10 - commit-message: - prefix: "[noissue]" -- package-ecosystem: pip - directory: "/tests" - schedule: - interval: monthly - ignore: - - dependency-name: "*" - open-pull-requests-limit: 0 - commit-message: - prefix: "[noissue]" - package-ecosystem: github-actions directory: "/" schedule: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 420f757..6d529d4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,6 +1,9 @@ name: "CodeQL" on: + push: + branches: + - main workflow_call: jobs: @@ -13,15 +16,29 @@ jobs: security-events: write steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/*constraints.lock', '**/setup.py', '**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip- + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Manually install from sources + run: | + python -m pip install -e . -e ./pulp-glue-gem + echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: python + setup-python-dependencies: false - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: python - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:python" + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:python" diff --git a/.github/workflows/collect_changes.yml b/.github/workflows/collect_changes.yml new file mode 100644 index 0000000..0cfd5b4 --- /dev/null +++ b/.github/workflows/collect_changes.yml @@ -0,0 +1,32 @@ +name: Collect changes +on: + workflow_call: + workflow_dispatch: + +jobs: + collect-changes: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: "main" + fetch-depth: 0 + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Setup git + run: | + git config user.name pulpbot + git config user.email pulp-infra@redhat.com + - name: Collect changes + run: | + pip install GitPython toml + python3 .ci/scripts/collect_changes.py + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.RELEASE_TOKEN }} + title: "Update Changelog" + body: "" + branch: "update_changes" + delete-branch: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2eef624..9baa371 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,4 +1,4 @@ -name: CI +name: Lint on: workflow_call: @@ -14,12 +14,17 @@ jobs: - "3.11" steps: - uses: actions/checkout@v4 + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/*constraints.lock', '**/setup.py', '**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Set up Python uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - name: Install python dependencies - run: | - pip install -r lint_requirements.txt + run: pip install -r lint_requirements.txt - name: Lint code run: make lint diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e96770f..6aafe12 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,11 +1,15 @@ -name: Nightly +name: pulp-cli Nightly on: schedule: - - cron: '15 3 * * *' + - cron: "15 3 * * *" + workflow_dispatch: jobs: test: uses: "./.github/workflows/test.yml" codeql: uses: "./.github/workflows/codeql.yml" + collect_changes: + uses: "./.github/workflows/collect_changes.yml" + secrets: inherit diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0851f7a..378a1b1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,4 +1,4 @@ -name: Pull Request CI +name: pulp-cli CI on: pull_request: @@ -22,15 +22,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: | - git fetch --prune --unshallow + - run: git fetch --prune --unshallow - name: Set up Python uses: actions/setup-python@v4 with: python-version: "3.11" - name: Install python dependencies - run: | - pip install toml pygithub + run: pip install toml pygithub - name: Check commit message env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..5bb326c --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,32 @@ +name: pulp-cli Publish + +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]" + +jobs: + publish-pypi: + name: Publish to PyPI + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Build and publish + env: + TWINE_USERNAME: pulp + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + cd pulp-glue-gem + python setup.py sdist bdist_wheel + twine upload dist/* + cd .. + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4d41bf9..6009bc2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,32 +1,27 @@ -name: Pulp CLI Release +name: pulp-cli Release on: - push: - tags: - - "[0-9]+.[0-9]+.[0-9]" + workflow_dispatch jobs: - publish-pypi: - name: Publish to PyPI + release: + name: Release runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: pulp - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - cd pulp-glue-gem - python setup.py sdist bdist_wheel - twine upload dist/* - cd .. - python setup.py sdist bdist_wheel - twine upload dist/* + - uses: actions/checkout@v4 + with: + token: ${{ secrets.RELEASE_TOKEN }} + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install bump2version towncrier + - name: Setup git + run: | + git config user.name pulpbot + git config user.email pulp-infra@redhat.com + - name: Release + run: .ci/scripts/release.sh diff --git a/.github/workflows/release_branch.yml b/.github/workflows/release_branch.yml new file mode 100644 index 0000000..29f9fbd --- /dev/null +++ b/.github/workflows/release_branch.yml @@ -0,0 +1,30 @@ +--- +name: Create Release Branch +on: + workflow_dispatch: + +jobs: + create-release-branch: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Setup git + run: | + git config user.name pulpbot + git config user.email pulp-infra@redhat.com + - name: Install python dependencies + run: pip install bump2version + - name: Create Release Branch + run: .ci/scripts/create_release_branch.sh + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + token: ${{ secrets.RELEASE_TOKEN }} + title: "Bump dev-version" + body: "" + branch: "bump_version" + delete-branch: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 635035b..dbbd3ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,12 @@ jobs: lower_bounds: true steps: - uses: actions/checkout@v4 + - uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/*constraints.lock', '**/setup.py', '**/pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pip- - name: Set up Python uses: actions/setup-python@v4 with: @@ -32,7 +38,7 @@ jobs: run: | if [ "${{matrix.lower_bounds}}" ] then - pip install -r test_requirements.txt -r tests/lower_bounds_requirements.txt + pip install -r test_requirements.txt -c lower_bounds_constraints.lock else pip install -r test_requirements.txt fi diff --git a/lint_requirements.txt b/lint_requirements.txt index 895d4e9..08bb34d 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -1,18 +1,16 @@ # Lint requirements -black==23.7.0 ; python_version > "3.7" -flake8==5.0.4 ; python_version >= "3.6.1" and python_version < "3.8.1" -flake8==6.1.0 ; python_version >= "3.8.1" -isort==5.11.5 ; python_version > "3.6" and python_version < "3.8" -isort==5.12.0 ; python_version >= "3.8" -mypy==1.5.1 ; python_version > "3.6" -shellcheck-py==0.9.0.5 ; python_version > "3.6" +black==23.7.0 +flake8==6.1.0 +isort==5.12.0 +mypy==1.5.1 +shellcheck-py==0.9.0.5 # Type annotation stubs -types-pygments ; python_version > "3.6" -types-PyYAML ; python_version > "3.6" -types-requests ; python_version > "3.6" -types-setuptools ; python_version > "3.6" -types-toml ; python_version > "3.6" +types-pygments +types-PyYAML +types-requests +types-setuptools +types-toml # Install the actual bits for mypy -r test_requirements.txt diff --git a/tests/lower_bounds_requirements.txt b/lower_bounds_constraints.lock similarity index 100% rename from tests/lower_bounds_requirements.txt rename to lower_bounds_constraints.lock diff --git a/pulp-glue-gem/pulp_glue/gem/context.py b/pulp-glue-gem/pulp_glue/gem/context.py index 1700d31..8a612b5 100644 --- a/pulp-glue-gem/pulp_glue/gem/context.py +++ b/pulp-glue-gem/pulp_glue/gem/context.py @@ -20,7 +20,7 @@ class PulpGemContentContext(PulpContentContext): HREF = "gem_gem_content_href" ID_PREFIX = "content_gem_gem" CAPABILITIES = {"upload": []} - NEEDS_PLUGINS = [PluginRequirement("gem", min="0.0.0")] + NEEDS_PLUGINS = [PluginRequirement("gem")] class PulpGemDistributionContext(PulpDistributionContext): @@ -30,7 +30,7 @@ class PulpGemDistributionContext(PulpDistributionContext): ENTITIES = _("gem distributions") HREF = "gem_gem_distribution_href" ID_PREFIX = "distributions_gem_gem" - NEEDS_PLUGINS = [PluginRequirement("gem", min="0.0.0")] + NEEDS_PLUGINS = [PluginRequirement("gem")] def preprocess_body(self, body: EntityDefinition) -> EntityDefinition: body = super().preprocess_body(body) @@ -48,7 +48,7 @@ class PulpGemPublicationContext(PulpPublicationContext): ENTITIES = _("gem publications") HREF = "gem_gem_publication_href" ID_PREFIX = "publications_gem_gem" - NEEDS_PLUGINS = [PluginRequirement("gem", min="0.0.0")] + NEEDS_PLUGINS = [PluginRequirement("gem")] def preprocess_entity(self, body: EntityDefinition, partial: bool = False) -> EntityDefinition: body = super().preprocess_entity(body, partial=partial) @@ -66,7 +66,7 @@ class PulpGemRemoteContext(PulpRemoteContext): ENTITIES = _("gem remotes") HREF = "gem_gem_remote_href" ID_PREFIX = "remotes_gem_gem" - NEEDS_PLUGINS = [PluginRequirement("gem", min="0.0.0")] + NEEDS_PLUGINS = [PluginRequirement("gem")] class PulpGemRepositoryVersionContext(PulpRepositoryVersionContext): @@ -74,7 +74,7 @@ class PulpGemRepositoryVersionContext(PulpRepositoryVersionContext): RESOURCE_TYPE = "gem" HREF = "gem_gem_repository_version_href" ID_PREFIX = "repositories_gem_gem_versions" - NEEDS_PLUGINS = [PluginRequirement("gem", min="0.0.0")] + NEEDS_PLUGINS = [PluginRequirement("gem")] class PulpGemRepositoryContext(PulpRepositoryContext): @@ -83,5 +83,5 @@ class PulpGemRepositoryContext(PulpRepositoryContext): HREF = "gem_gem_repository_href" ID_PREFIX = "repositories_gem_gem" VERSION_CONTEXT = PulpGemRepositoryVersionContext - NEEDS_PLUGINS = [PluginRequirement("gem", min="0.0.0")] + NEEDS_PLUGINS = [PluginRequirement("gem")] NULLABLES = PulpRepositoryContext.NULLABLES | {"remote"}