Update CI

[noissue]

.ci/run_container.sh

@@ -5,6 +5,7 @@
 #   - https://github.com/pulp/pulp-cli/blob/main/.ci/run_container.sh
 #   - https://github.com/pulp/pulp-cli-deb/blob/main/.ci/run_container.sh
 #   - https://github.com/pulp/pulp-cli-gem/blob/main/.ci/run_container.sh
+#   - https://github.com/pulp/pulp-cli-maven/blob/main/.ci/run_container.sh
 #   - https://github.com/pulp/pulp-cli-ostree/blob/main/.ci/run_container.sh
 #   - https://github.com/pulp/squeezer/blob/develop/tests/run_container.sh
 

.ci/scripts/collect_changes.py

@@ -0,0 +1,94 @@
+import itertools
+import os
+import re
+
+import toml
+from git import GitCommandError, Repo
+from pkg_resources import parse_version
+
+# Read Towncrier settings
+tc_settings = toml.load("pyproject.toml")["tool"]["towncrier"]
+
+CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst")
+START_STRING = tc_settings.get(
+    "start_string",
+    "<!-- towncrier release notes start -->\n"
+    if CHANGELOG_FILE.endswith(".md")
+    else ".. towncrier release notes start\n",
+)
+TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})")
+
+
+NAME_REGEX = r".*"
+VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)"
+DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}"
+TITLE_REGEX = (
+    "("
+    + re.escape(
+        TITLE_FORMAT.format(name="NAME_REGEX", version="VERSION_REGEX", project_date="DATE_REGEX")
+    )
+    .replace("NAME_REGEX", NAME_REGEX)
+    .replace("VERSION_REGEX", VERSION_REGEX)
+    .replace("DATE_REGEX", DATE_REGEX)
+    + ")"
+)
+
+
+def get_changelog(repo, branch):
+    return repo.git.show(f"{branch}:{CHANGELOG_FILE}") + "\n"
+
+
+def _tokenize_changes(splits):
+    assert len(splits) % 3 == 0
+    for i in range(len(splits) // 3):
+        title = splits[3 * i]
+        version = parse_version(splits[3 * i + 1])
+        yield [version, title + splits[3 * i + 2]]
+
+
+def split_changelog(changelog):
+    preamble, rest = changelog.split(START_STRING, maxsplit=1)
+    split_rest = re.split(TITLE_REGEX, rest)
+    return preamble + START_STRING + split_rest[0], list(_tokenize_changes(split_rest[1:]))
+
+
+def main():
+    repo = Repo(os.getcwd())
+    remote = repo.remotes[0]
+    branches = [ref for ref in remote.refs if re.match(r"^([0-9]+)\.([0-9]+)$", ref.remote_head)]
+    branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True)
+    branches = [ref.name for ref in branches]
+
+    with open(CHANGELOG_FILE, "r") as f:
+        main_changelog = f.read()
+    preamble, main_changes = split_changelog(main_changelog)
+    old_length = len(main_changes)
+
+    for branch in branches:
+        print(f"Looking at branch {branch}")
+        try:
+            changelog = get_changelog(repo, branch)
+        except GitCommandError:
+            print("No changelog found on this branch.")
+            continue
+        dummy, changes = split_changelog(changelog)
+        new_changes = sorted(main_changes + changes, key=lambda x: x[0], reverse=True)
+        # Now remove duplicates (retain the first one)
+        main_changes = [new_changes[0]]
+        for left, right in itertools.pairwise(new_changes):
+            if left[0] != right[0]:
+                main_changes.append(right)
+
+    new_length = len(main_changes)
+    if old_length < new_length:
+        print(f"{new_length - old_length} new versions have been added.")
+        with open(CHANGELOG_FILE, "w") as fp:
+            fp.write(preamble)
+            for change in main_changes:
+                fp.write(change[1])
+
+        repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/create_release_branch.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+BRANCH="$(git branch --show-current)"
+
+if ! [[ "${BRANCH}" = "main" ]]
+then
+  echo ERROR: This is not the main branch!
+  exit 1
+fi
+
+NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')"
+
+if [[ -z "${NEW_BRANCH}" ]]
+then
+  echo ERROR: Could not parse new version.
+  exit 1
+fi
+
+git branch "${NEW_BRANCH}"
+
+# Clean changelog snippets.
+find CHANGES/ \( -name "*.feature" -o -name "*.bugfix" -o -name "*.doc" -o -name "*.translation" -o -name "*.devel" -o -name "*.misc" \) -exec git rm -f \{\} +
+
+bumpversion minor --commit --message $'Bump version to {new_version}\n\n[noissue]' --allow-dirty
+
+git push origin "${NEW_BRANCH}"

.ci/scripts/release.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+BRANCH=$(git branch --show-current)
+
+if ! [[ "${BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]]
+then
+  echo ERROR: This is not a release branch!
+  exit 1
+fi
+
+NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')"
+echo "Release ${NEW_VERSION}"
+
+if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]]
+then
+  echo ERROR: Version does not match release branch
+  exit 1
+fi
+
+towncrier build --yes --version "${NEW_VERSION}"
+bumpversion release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty
+bumpversion patch --commit
+
+git push origin "${BRANCH}" "${NEW_VERSION}"

.github/dependabot.yml

@@ -7,22 +7,6 @@ updates:
   open-pull-requests-limit: 10
   commit-message:
     prefix: "[noissue]"
-- package-ecosystem: pip
-  directory: "/pulp-glue-gem"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
-  commit-message:
-    prefix: "[noissue]"
-- package-ecosystem: pip
-  directory: "/tests"
-  schedule:
-    interval: monthly
-  ignore:
-    - dependency-name: "*"
-  open-pull-requests-limit: 0
-  commit-message:
-    prefix: "[noissue]"
 - package-ecosystem: github-actions
   directory: "/"
   schedule:

.github/workflows/codeql.yml

@@ -1,6 +1,9 @@
 name: "CodeQL"
 
 on:
+  push:
+    branches:
+      - main
   workflow_call:
 
 jobs:
@@ -13,15 +16,29 @@ jobs:
       security-events: write
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/*constraints.lock', '**/setup.py', '**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Manually install from sources
+        run: |
+          python -m pip install -e . -e ./pulp-glue-gem
+          echo "CODEQL_PYTHON=$(which python)" >> $GITHUB_ENV
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: python
+          setup-python-dependencies: false
 
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: python
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
-      with:
-        category: "/language:python"
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2
+        with:
+          category: "/language:python"

.github/workflows/collect_changes.yml

@@ -0,0 +1,32 @@
+name: Collect changes
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  collect-changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: "main"
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Setup git
+        run: |
+          git config user.name pulpbot
+          git config user.email [email protected]
+      - name: Collect changes
+        run: |
+          pip install GitPython toml
+          python3 .ci/scripts/collect_changes.py
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          token: ${{ secrets.RELEASE_TOKEN }}
+          title: "Update Changelog"
+          body: ""
+          branch: "update_changes"
+          delete-branch: true

.github/workflows/lint.yml

@@ -1,4 +1,4 @@
-name: CI
+name: Lint
 
 on:
   workflow_call:
@@ -14,12 +14,17 @@ jobs:
           - "3.11"
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/*constraints.lock', '**/setup.py', '**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
       - name: Install python dependencies
-        run: |
-          pip install -r lint_requirements.txt
+        run: pip install -r lint_requirements.txt
       - name: Lint code
         run: make lint

.github/workflows/nightly.yml

@@ -1,11 +1,15 @@
-name: Nightly
+name: pulp-cli Nightly
 
 on:
   schedule:
-    - cron: '15 3 * * *'
+    - cron: "15 3 * * *"
+  workflow_dispatch:
 
 jobs:
   test:
     uses: "./.github/workflows/test.yml"
   codeql:
     uses: "./.github/workflows/codeql.yml"
+  collect_changes:
+    uses: "./.github/workflows/collect_changes.yml"
+    secrets: inherit

.github/workflows/pr.yml

@@ -1,4 +1,4 @@
-name: Pull Request CI
+name: pulp-cli CI
 
 on:
   pull_request:
@@ -22,15 +22,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - run: |
-          git fetch --prune --unshallow
+      - run: git fetch --prune --unshallow
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: "3.11"
       - name: Install python dependencies
-        run: |
-          pip install toml pygithub
+        run: pip install toml pygithub
       - name: Check commit message
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/publish.yml

@@ -0,0 +1,32 @@
+name: pulp-cli Publish
+
+on:
+  push:
+    tags:
+      - "[0-9]+.[0-9]+.[0-9]"
+
+jobs:
+  publish-pypi:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: pulp
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          cd pulp-glue-gem
+          python setup.py sdist bdist_wheel
+          twine upload dist/*
+          cd ..
+          python setup.py sdist bdist_wheel
+          twine upload dist/*