Automation of github release and tag, and publish to pypi

.github/workflows/publish_main.yml

@@ -0,0 +1,122 @@
+name: Publish to PyPI
+on:
+  push:
+    branches: ['main']
+
+jobs:
+  pypi_release:
+    runs-on: ubuntu-latest
+    env:
+      POETRY_VERSION:  1.3.0
+    steps:
+      - name: Get list of tags
+        uses: octokit/[email protected]
+        id: get_latest_release
+        with:
+          route: GET /repos/${{ github.repository }}/git/matching-refs/tags
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #----------------------------------------------
+      #       check-out repo and set-up python     
+      #----------------------------------------------
+      - name: Check out repository
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      #----------------------------------------------
+      #          install & configure poetry
+      #----------------------------------------------
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org \
+            | python3 - --version ${{ env.POETRY_VERSION }};
+          poetry config virtualenvs.create true;
+          poetry config virtualenvs.in-project true;
+
+      #----------------------------------------------
+      #       load cached venv if cache exists      
+      #----------------------------------------------
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v2
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
+
+      #----------------------------------------------
+      # install dependencies and root project
+      #----------------------------------------------
+      - name: Install dependencies and root project
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --all-extras
+
+      #----------------------------------------------
+      #    Construct release version
+      #----------------------------------------------      
+      - name: Construct release version
+        id: release_version
+        run: |
+           VAR=$(poetry run python3.10 github_scripts/construct_release_version.py)
+           echo "RELEASE_VERSION=$VAR" >> "$GITHUB_OUTPUT"
+        shell: sh
+        env:
+          TAGS: ${{ steps.get_latest_release.outputs.data }}
+
+      #----------------------------------------------
+      #    Create release
+      #----------------------------------------------  
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ steps.release_version.outputs.RELEASE_VERSION }}
+          release_name: ${{ steps.release_version.outputs.RELEASE_VERSION }}
+          draft: false
+          prerelease: false
+
+      #----------------------------------------------
+      #    Set release version
+      #----------------------------------------------      
+      - name: Set release version
+        run: |
+          poetry run python3 github_scripts/update_toml_version.py
+        shell: sh
+        env:
+          RELEASE_VERSION: ${{ steps.release_version.outputs.RELEASE_VERSION }}
+
+      #----------------------------------------------
+      #    publish to pypi
+      #----------------------------------------------  
+      - name: Publish package to Pypi
+        id: publish-to-pypi
+        env:
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+          PYPI_USERNAME: __token__
+        run: |
+          poetry publish --build --username $PYPI_USERNAME --password $PYPI_TOKEN
+
+      #----------------------------------------------
+      #    post a message to slack
+      #----------------------------------------------  
+
+      - name: Post to a Slack channel
+        if: steps.publish-to-pypi.outcome == 'success'
+        id: slack
+        uses: slackapi/[email protected]
+        with:
+          # Slack channel id, channel name, or user id to post message.
+          # See also: https://api.slack.com/methods/chat.postMessage#channels
+          # You can pass in multiple channels to post to by providing a comma-delimited list of channel IDs.
+          # ibc-fair-data channel and data-curator-schematic channel
+          channel-id: 'C050YD75QRL,C01ANC02U59'
+          # For posting a simple plain text message
+          slack-message: "Schematic has just been released. Check out new version: ${{ github.ref_name }}"
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/publish.yml → .github/workflows/publish_tag.yml

@@ -2,7 +2,7 @@ name: Publish to PyPI
 on:
   push:
     tags:
-      - 'v[1-9][0-9].[0-9]+.[0-9]+'
+      - 'v[1-9][0-9].[0-9]+.[0-9].[a-z]+'
     branches: [main]
 
 jobs:
@@ -114,4 +114,4 @@ jobs:
           # For posting a simple plain text message
           slack-message: "Schematic has just been released. Check out new version: ${{ github.ref_name }}"
         env:
-          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

github_scripts/construct_release_version.py

@@ -0,0 +1,24 @@
+"""
+This is a command line script meant to be used by a github action.
+It constructs the current release tag using todays date, and the number of releases 
+  already this month.
+For example the first release of January 2024 would be 24.1.1 and the second would be 24.1.2
+It assumes there is an environmental variable set called TAGS that contains the tag 
+ information from the repo.
+"""
+import os
+
+from github_scripts.utils import (
+    get_date_strings,
+    get_number_of_versions_this_month, 
+    get_version_list_from_tags,
+)
+
+tag_string = os.getenv("TAGS")
+assert isinstance(tag_string, str)
+
+year, month = get_date_strings()
+version_list = get_version_list_from_tags(tag_string)
+num_tags_this_month = get_number_of_versions_this_month(version_list, year, month)
+
+print (f"{year}.{month}.{num_tags_this_month + 1}")

github_scripts/update_toml_version.py

@@ -0,0 +1,20 @@
+"""
+This is a command line script meant to be used by a github action.
+It modifies the pyproject.toml file to have the current release version.
+Poetry use the toml version as the version it publishes to pypi.
+It assumes there is an environmental variable set called RELEASE_VERSION 
+  that contains the release version to use.
+"""
+
+import os
+import toml
+
+RELEASE_VERSION = os.getenv("RELEASE_VERSION")
+assert isinstance(RELEASE_VERSION, str)
+
+data = toml.load("pyproject.toml")
+data['tool']['poetry']['version']=RELEASE_VERSION
+print('the version number of this release is: ', RELEASE_VERSION)
+
+with open("pyproject.toml",'w', encoding="utf-8") as file:
+    toml.dump(data, file)

github_scripts/utils.py

@@ -0,0 +1,75 @@
+"""Util functions for github scripts"""
+
+import os
+from datetime import date
+import re
+from typing import Optional
+import json
+
+
+def get_date_strings(todays_date:Optional[date]=None) -> tuple[str, str]:
+    """Gets todays year and month in form as strings
+
+    Args:
+        todays_date (Optional[date], optional): Defaults to None.
+
+    Returns:
+        tuple[str, str]: The year and month in two digit form ie. ("24", "12")
+    """
+    if todays_date is None:
+        todays_date = date.today()
+    todays_year = str(todays_date.year)[-2:]
+    todays_month = str(todays_date.month)
+    return todays_year, todays_month
+
+def get_version_list_from_tags(tag_string:str) -> list[str]:
+    """Gets the versions from a list fo tags in string form
+
+    Args:
+        tag_string (str): A json in string form that contians tag information such as:
+            '[
+                {
+                    "ref": "refs/tags/v24.2.1",
+                },
+                    "ref": "refs/tags/v24.2.1-beta",
+                }
+            ]'
+
+    Returns:
+        list[str]: A list of versions from each tag
+    """
+    tag_dict_list = json.loads(tag_string)
+    assert isinstance(tag_dict_list, list)
+    for tag in tag_dict_list:
+        assert isinstance(tag, dict)
+        assert "ref" in tag
+        assert isinstance(tag["ref"], str)
+    ref_list: list[str] = [tag_dict["ref"] for tag_dict in tag_dict_list]
+    version_list = [os.path.basename(ref) for ref in ref_list]
+    return version_list
+
+
+def get_number_of_versions_this_month(
+    version_list: list[str],
+    todays_year: str,
+    todays_month: str
+) -> int:
+    """
+    This takes a list of version and returns the number of tags from this month.
+
+    Args:
+        version_list (list[str]): A list of tag version such as "v24.2.1"
+        todays_year(str): todays year as a two digit string ie. 2024: "24"
+        todays_month(str): todays month as a one or two digit string ie. December: "12"
+
+    Returns:
+        int: Ther number of tags already released this month
+    """
+    tags_this_month:int = 0
+    for version in version_list:
+        if re.match("^v[0-9]+\.[0-9]+\.[0-9]+$", version):
+            version = version[1:]
+            version_year, version_month, version_number = version.split(".")
+            if version_year == todays_year and version_month == todays_month:
+                tags_this_month = max(int(version_number), tags_this_month)
+    return tags_this_month

pyproject.toml

@@ -166,4 +166,4 @@ markers = [
     empty_token: marks api tests that \
     send empty credentials in the request
     """
-]
+]

tests/test_github_scripts.py

@@ -0,0 +1,45 @@
+""" Tests for github scripts"""
+
+from datetime import date
+
+from github_scripts.utils import (
+    get_date_strings,
+    get_version_list_from_tags,
+    get_number_of_versions_this_month,
+)
+
+def test_get_date_string() -> None:
+    """Tests get_date_string"""
+    year, month = get_date_strings()
+    assert len(year) == 2
+    assert 100 > int(year) >= 0
+    assert 3 > len(month) > 0
+    assert 13 > int(month) > 0
+
+    year, month = get_date_strings(date(2020, 1, 1))
+    year = "20"
+    month = "1"
+
+    year, month = get_date_strings(date(1999, 12, 10))
+    year = "99"
+    month = "12"
+
+def test_get_version_list_from_tags() -> None:
+    """Tests get_version_list_from_tags"""
+    tags = (
+        '['
+            '{"ref": "x/x/v24.1.1"},'
+            '{"ref": "x/x/v24.1.2"},'
+            '{"ref": "x/x/v24.1.3"}'
+        ']'
+    )
+    assert get_version_list_from_tags(tags) == ["v24.1.1", "v24.1.2", "v24.1.3"]
+
+def test_get_number_of_versions_this_month() -> None:
+    """Tests get_number_of_versions_this_month"""
+    versions = ["v24.1.1", "v24.1.2", "v24.1.3"]
+    assert get_number_of_versions_this_month(versions, "24", "1") == 3
+    assert get_number_of_versions_this_month(versions, "23", "1") == 0
+    assert get_number_of_versions_this_month(versions, "24", "0") == 0
+    versions2 = ["24.1.1", "v24.1.1-dev"]
+    assert get_number_of_versions_this_month(versions2, "24", "1") == 0