WIP: Add release automation

* Create release branch from main and bump minor version
* Release from a release branch and bump patch version
* Collect changelogs from all branches in main

[noissue]

.ci/scripts/collect_changes.py

@@ -0,0 +1,84 @@
+import itertools
+import re
+import os
+from pkg_resources import parse_version
+from git import Repo, GitCommandError
+
+
+# TODO read Towncrier settings here
+CHANGELOG_FILE = "docs/CHANGES.md"
+START_STRING = "[//]: # (towncrier release notes start)\n"
+TITLE_FORMAT = "## {version} ({project_date})"
+
+
+VERSION_REGEX = r"([0-9]+\.[0-9]+\.[0-9][0-9ab]*)"
+DATE_REGEX = r"[0-9]{4}-[0-9]{2}-[0-9]{2}"
+TITLE_REGEX = (
+    "("
+    + re.escape(TITLE_FORMAT.format(version="VERSION_REGEX", project_date="DATE_REGEX"))
+    .replace("VERSION_REGEX", VERSION_REGEX)
+    .replace("DATE_REGEX", DATE_REGEX)
+    + ")"
+)
+
+
+def get_changelog(repo, branch):
+    return repo.git.show(f"{branch}:{CHANGELOG_FILE}") + "\n"
+
+
+def _tokenize_changes(splits):
+    assert len(splits) % 3 == 0
+    for i in range(len(splits) // 3):
+        title = splits[3 * i]
+        version = parse_version(splits[3 * i + 1])
+        yield [version, title + splits[3 * i + 2]]
+
+
+def split_changelog(changelog):
+    preamble, rest = changelog.split(START_STRING, maxsplit=1)
+    split_rest = re.split(TITLE_REGEX, rest)
+    return preamble + START_STRING + split_rest[0], list(_tokenize_changes(split_rest[1:]))
+
+
+def main():
+    repo = Repo(os.getcwd())
+    remote = repo.remotes[0]
+    branches = [
+        ref for ref in remote.refs if re.match(r"^([0-9]+)\.([0-9]+)$", ref.remote_head)
+    ]
+    branches.sort(key=lambda ref: parse_version(ref.remote_head), reverse=True)
+    branches = [ref.name for ref in branches]
+
+    with open(CHANGELOG_FILE, "r") as f:
+        main_changelog = f.read()
+    preamble, main_changes = split_changelog(main_changelog)
+    old_length = len(main_changes)
+
+    for branch in branches:
+        print(f"Looking at branch {branch}")
+        try:
+            changelog = get_changelog(repo, branch)
+        except GitCommandError:
+            print("No changelog found on this branch.")
+            continue
+        dummy, changes = split_changelog(changelog)
+        new_changes = sorted(main_changes + changes, key=lambda x: x[0], reverse=True)
+        # Now remove duplicates (retain the first one)
+        main_changes = [new_changes[0]]
+        for left, right in itertools.pairwise(new_changes):
+            if left[0] != right[0]:
+                main_changes.append(right)
+
+    new_length = len(main_changes)
+    if old_length < new_length:
+        print(f"{new_length - old_length} new versions have been added.")
+        with open(CHANGELOG_FILE, "w") as f:
+            f.write(preamble)
+            for change in main_changes:
+                f.write(change[1])
+
+        repo.git.commit("-m", "Update Changelog", "-m" "[noissue]", CHANGELOG_FILE)
+
+
+if __name__ == "__main__":
+    main()

.ci/scripts/create_release_branch.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+BRANCH=$(git branch --show-current)
+
+if ! [[ "${BRANCH}" = "main" ]]
+then
+  echo ERROR: This is not the main branch!
+  exit 1
+fi
+
+NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')"
+
+if [[ -z "${NEW_BRANCH}" ]]
+then
+  echo ERROR: Could not parse new version.
+  exit 1
+fi
+
+git branch "${NEW_BRANCH}"
+
+# Clean changelog snippets.
+find CHANGES/ \( -name "*.feature" -o -name "*.bugfix" -o -name "*.doc" -o -name "*.translation" -o -name "*.devel" -o -name "*.misc" \) -exec git rm -f \{\} +
+
+bumpversion minor --commit --allow-dirty
+
+git push origin main "${NEW_BRANCH}"

.ci/scripts/release.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+BRANCH=$(git branch --show-current)
+
+if ! [[ "${BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]]
+then
+  echo ERROR: This is not a release branch!
+  exit 1
+fi
+
+NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')"
+echo "Release ${NEW_VERSION}"
+
+if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]]
+then
+  echo ERROR: Version does not match release branch
+  exit 1
+fi
+
+exit
+towncrier --yes
+bumpversion release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty
+bumpversion patch --commit
+
+
+# TODO Push changes; upload to pypi; publish docs
+
+exit 1

.github/workflows/changes.yml

@@ -0,0 +1,28 @@
+name: Collect changes
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  collect-changes:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Setup git
+        run: |
+          git config user.name pulpbot
+          git config user.email [email protected]
+      - name: Collect changes
+        run: |
+          pip install GitPython
+          python3 .ci/scripts/collect_changes.py
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-reuqest@v5
+        with:
+          title: "Update Changelog"
+          branch: "update_changes"

.github/workflows/nightly.yml

@@ -2,10 +2,12 @@ name: Nightly
 
 on:
   schedule:
-    - cron: '15 3 * * *'
+    - cron: "15 3 * * *"
 
 jobs:
   test:
     uses: "./.github/workflows/test.yml"
   codeql:
     uses: "./.github/workflows/codeql.yml"
+  collect_changes:
+    uses: "./.github/workflows/changes.yml"

.github/workflows/pr.yml

@@ -8,6 +8,8 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  experiment:
+    uses: "./.github/workflows/changes.yml"
   lint:
     uses: "./.github/workflows/lint.yml"
   test:

.github/workflows/release_branch.yml

@@ -0,0 +1,24 @@
+---
+name: Create Release Branch
+on:
+  workflow_dispatch:
+
+jobs:
+  create-release-branch:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+      - name: Setup git
+        run: |
+          git config user.name pulpbot
+          git config user.email [email protected]
+      - name: Install python dependencies
+        run: |
+          pip install bump2version
+      - name: Create Release Branch
+        run: |
+          .ci/scripts/create_release_branch.sh