From ab86d21e30ff92787a5d7f88787000d3da0b27a8 Mon Sep 17 00:00:00 2001 From: Lukasz Dynowski Date: Fri, 23 Apr 2021 13:06:42 +0200 Subject: [PATCH 1/3] adding first action --- .github/workflows/ci.yaml | 107 ++++++++++++++++++++++++ scripts/notification.sh | 37 ++++++++ scripts/travis_telegram_notification.sh | 0 tox.ini | 7 ++ 4 files changed, 151 insertions(+) create mode 100644 .github/workflows/ci.yaml create mode 100755 scripts/notification.sh mode change 100644 => 100755 scripts/travis_telegram_notification.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..484a29c --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,107 @@ +name: Continuous Integration +on: + push: + branches: [ actions ] + pull_request: + branches: [ master, actions ] + workflow_dispatch: + branches: [ master, actions ] +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - python: 3.6 + django: 2.2 + toxenv: py36-django22 + - python: 3.6 + django: 3.0 + toxenv: py36-django30 + - python: 3.7 + django: 2.2 + toxenv: py37-django22 + - python: 3.7 + django: 3.0 + toxenv: py37-django30 + - python: 3.8 + django: 2.2 + toxenv: py38-django22 + - python: 3.8 + django: 3.0 + toxenv: py38-django30 + - python: 3.9 + django: 2.2 + toxenv: py39-django22 + - python: 3.9 + django: 3.0 + toxenv: py39-django30 + - python: 3.6 + toxenv: quality + - python: 3.6 + toxenv: security + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + name: Set up Python ${{ matrix.python }} ${{ matrix.django }} + with: + python-version: ${{ matrix.python }} + - name: Install pip packages + run: | + pip install pip --upgrade + pip install codecov poetry tox + - name: "Run tox - testenv: ${{ matrix.toxenv }}" + env: + DJANGO: ${{ matrix.django }} + TOXENV: ${{ matrix.toxenv }} + run: tox + - name: Run notification script + env: + GITHUB_JOB_STATUS: ${{ job.status }} + GITHUB_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + GITHUB_RUN_URL: ${{ github.event.repository.url }}/actions/runs/${{ github.run_id }} + run: ./scripts/notification.sh + + quality: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - python: 3.6 + toxenv: quality + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + name: Set up Python ${{ matrix.python }} + with: + python-version: ${{ matrix.python }} + - name: Install pip packages + run: | + pip install pip --upgrade + pip install poetry tox + - name: "Run tox - testenv: ${{ matrix.toxenv }}" + env: + TOXENV: ${{ matrix.toxenv }} + run: tox + + security: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - python: 3.6 + toxenv: security + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + name: Set up Python ${{ matrix.python }} + with: + python-version: ${{ matrix.python }} + - name: Install pip packages + run: | + pip install pip --upgrade + pip install poetry tox + - name: "Run tox - testenv: ${{ matrix.toxenv }}" + env: + TOXENV: ${{ matrix.toxenv }} + run: tox \ No newline at end of file diff --git a/scripts/notification.sh b/scripts/notification.sh new file mode 100755 index 0000000..b9c5052 --- /dev/null +++ b/scripts/notification.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +# TODO: Update/missing GitHub Actions environment variables with TELEGRAM_TOKEN +# TODO: Update/missing GitHub Actions environment variables with TELEGRAM_CHAT_ID +# TODO: Once this script is working file `travis_telegram_notification.sh` should be deleted! + +# Get the token from Travis environment vars and build the bot URL: +BOT_URL="https://api.telegram.org/bot${TELEGRAM_TOKEN}/sendMessage" + +# Set formatting for the message. Can be either "Markdown" or "HTML" +PARSE_MODE="Markdown" + +# Define send message function. parse_mode can be changed to +# HTML, depending on how you want to format your message: +send_msg () { + curl -s -X POST ${BOT_URL} \ + -d chat_id=$TELEGRAM_CHAT_ID \ + -d text="$1" \ + -d parse_mode=${PARSE_MODE} +} + +# Send message to the bot with some pertinent details about the job +# Note that for Markdown, you need to escape any backtick (inline-code) +# characters, since they're reserved in bash +send_msg " +---------------------------------------------------- +GitHub Actions build *${GITHUB_JOB_STATUS}!* +\`Repository: ${GITHUB_REPOSITORY}\` +\`Branch: ${GITHUB_REF}\` +\`Environment: ${TOXENV}\` +\`Run Number/Run ID: ${GITHUB_RUN_NUMBER}/${GITHUB_RUN_ID}\` +*Commit Msg:* +${GITHUB_COMMIT_MESSAGE} + +[See complete job log here](${GITHUB_RUN_URL}) +----------------------------------------------------- +" \ No newline at end of file diff --git a/scripts/travis_telegram_notification.sh b/scripts/travis_telegram_notification.sh old mode 100644 new mode 100755 diff --git a/tox.ini b/tox.ini index 21bc23b..515beba 100644 --- a/tox.ini +++ b/tox.ini @@ -94,4 +94,11 @@ commands = poetry install -vvv poetry run black --check . poetry run flake8 . + +[testenv:security] +whitelist_externals = poetry +skip_install = true +commands = + poetry update --lock + poetry install -vvv poetry run bandit . From a71b7ce350fb1f51cddc5930ecb2b14386461efd Mon Sep 17 00:00:00 2001 From: Lukasz Dynowski Date: Thu, 29 Apr 2021 16:02:06 +0200 Subject: [PATCH 2/3] Adding bump.sh script --- scripts/bump.py | 149 ++++++++++++++++++++++++++++++++++++++++++++++++ tox.ini | 2 +- 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100755 scripts/bump.py diff --git a/scripts/bump.py b/scripts/bump.py new file mode 100755 index 0000000..85629fb --- /dev/null +++ b/scripts/bump.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import argparse +import re +import toml + + +def get_args(): + """Set up CLI args""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--major", + type=int, + nargs="*", + help="Major version bump. Empty flag will auto bump current version.", + ) + parser.add_argument( + "--minor", + type=int, + nargs="*", + help="Minor version bump. Empty flag will auto bump current version.", + ) + parser.add_argument( + "--patch", + type=int, + nargs="*", + help="Patch version bump. Empty flag will auto bump current version.", + ) + parser.add_argument( + "--update", + type=str, + default="yes", + choices=["yes", "no"], + help="Make update to config file.", + ) + parser.add_argument( + "--dry-run", + type=str, + default="yes", + choices=["yes", "no"], + help="Print versions.", + ) + + return parser.parse_args() + + +def get_version(pyproject_file_path): + """Function returns tuple that elements follow semantic versioning order""" + with open(pyproject_file_path) as file: + pyproject = toml.loads(file.read()) + current_version = pyproject["tool"]["poetry"]["version"] + + validate_version(current_version) + + # Normalize version to tuple + if current_version.count(".") == 0: + return tuple(int(current_version)) + + return tuple(int(v) for v in current_version.split(".")) + + +def validate_version(current_version, pattern=r"^\d+\.\d+\.\d+$"): + """ + Validate that extracted version follows "MAJOR.MINOR.PATCH" pattern + """ + match = re.search(pattern, current_version) + if not match: + print( + f'Error: Package version {current_version} is not following semantic versioning "MAJOR.MINOR.PATCH"' + ) + exit(1) + + +def bump_major(version): + if not version: + return CURRENT_VERSION[0] + 1 + + return version[0] + + +def bump_minor(version): + if not version: + return CURRENT_VERSION[1] + 1 + + return version[0] + + +def bump_patch(version): + if not version: + return CURRENT_VERSION[2] + 1 + + return version[0] + + +def bump_version(major, minor, patch): + if major or type(major) == list: + major = bump_major(major) + + if minor or type(minor) == list: + minor = bump_minor(minor) + + if (type(patch) == list) or all(v is None for v in (major, minor, patch)): + patch = bump_patch(patch) + + # Construct bump from new version and current version + bump = [] + new_versions = (major, minor, patch) + for index in range(len(new_versions)): + if new_versions[index] is None: + bump.append(CURRENT_VERSION[index]) + else: + bump.append(new_versions[index]) + + return tuple(bump) + + +if __name__ == "__main__": + args = get_args() + + # Obtain 'pyproject.toml' file path + current_file_path = str(Path(__file__).resolve()) + pyproject_file_path = current_file_path.replace(current_file_path, "pyproject.toml") + + # Bump and normalize current version + CURRENT_VERSION = get_version(pyproject_file_path) + NEW_VERSION = bump_version(args.major, args.minor, args.patch) + CURRENT_VERSION = ".".join(map(str, CURRENT_VERSION)) + NEW_VERSION = ".".join(map(str, NEW_VERSION)) + + # Print version check + if args.dry_run == "yes": + print(f"Current version: {CURRENT_VERSION}") + print(f"New version: {NEW_VERSION}") + exit(0) + + # Update package version + if args.update == "yes": + with open(pyproject_file_path, "r") as file: + content = file.read() + + with open(pyproject_file_path, "w") as file: + CV = f'version = "{CURRENT_VERSION}"' + NV = f'version = "{NEW_VERSION}"' + content = content.replace(CV, NV) + file.write(content) + print( + f"Successfully updated package version from {CURRENT_VERSION} to {NEW_VERSION}" + ) diff --git a/tox.ini b/tox.ini index 515beba..31d1883 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] isolated_build = true -envlist = py{36,37,38,39}-django{22,30}, quality +envlist = py{36,37,38,39}-django{22,30}, quality, security [travis:env] DJANGO = From 7ca14c582dc8e34c6f88e861f1583ce60ee00bda Mon Sep 17 00:00:00 2001 From: Lukasz Dynowski Date: Mon, 3 May 2021 14:56:46 +0200 Subject: [PATCH 3/3] Finishing CICD for GitHub Actions --- .github/workflows/ci.yaml | 107 ----------------- .github/workflows/cicd.yaml | 149 +++++++++++++++++++++++- .travis.yml | 46 -------- scripts/bump.py | 18 +-- scripts/notification.sh | 4 - scripts/travis_telegram_notification.sh | 37 ------ tox.ini | 10 +- 7 files changed, 156 insertions(+), 215 deletions(-) delete mode 100644 .github/workflows/ci.yaml delete mode 100644 .travis.yml delete mode 100755 scripts/travis_telegram_notification.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index 484a29c..0000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,107 +0,0 @@ -name: Continuous Integration -on: - push: - branches: [ actions ] - pull_request: - branches: [ master, actions ] - workflow_dispatch: - branches: [ master, actions ] -jobs: - test: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python: 3.6 - django: 2.2 - toxenv: py36-django22 - - python: 3.6 - django: 3.0 - toxenv: py36-django30 - - python: 3.7 - django: 2.2 - toxenv: py37-django22 - - python: 3.7 - django: 3.0 - toxenv: py37-django30 - - python: 3.8 - django: 2.2 - toxenv: py38-django22 - - python: 3.8 - django: 3.0 - toxenv: py38-django30 - - python: 3.9 - django: 2.2 - toxenv: py39-django22 - - python: 3.9 - django: 3.0 - toxenv: py39-django30 - - python: 3.6 - toxenv: quality - - python: 3.6 - toxenv: security - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - name: Set up Python ${{ matrix.python }} ${{ matrix.django }} - with: - python-version: ${{ matrix.python }} - - name: Install pip packages - run: | - pip install pip --upgrade - pip install codecov poetry tox - - name: "Run tox - testenv: ${{ matrix.toxenv }}" - env: - DJANGO: ${{ matrix.django }} - TOXENV: ${{ matrix.toxenv }} - run: tox - - name: Run notification script - env: - GITHUB_JOB_STATUS: ${{ job.status }} - GITHUB_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} - GITHUB_RUN_URL: ${{ github.event.repository.url }}/actions/runs/${{ github.run_id }} - run: ./scripts/notification.sh - - quality: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python: 3.6 - toxenv: quality - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - name: Set up Python ${{ matrix.python }} - with: - python-version: ${{ matrix.python }} - - name: Install pip packages - run: | - pip install pip --upgrade - pip install poetry tox - - name: "Run tox - testenv: ${{ matrix.toxenv }}" - env: - TOXENV: ${{ matrix.toxenv }} - run: tox - - security: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python: 3.6 - toxenv: security - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v1 - name: Set up Python ${{ matrix.python }} - with: - python-version: ${{ matrix.python }} - - name: Install pip packages - run: | - pip install pip --upgrade - pip install poetry tox - - name: "Run tox - testenv: ${{ matrix.toxenv }}" - env: - TOXENV: ${{ matrix.toxenv }} - run: tox \ No newline at end of file diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 98b8309..483d2fb 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -1,12 +1,153 @@ -name: WIP +name: Continuous Integration & Delivery on: + pull_request: + branches: [ master ] workflow_dispatch: branches: [ master, actions ] jobs: - demo: + test: runs-on: ubuntu-latest + strategy: + matrix: + include: + - python: 3.6 + django: 2.2 + toxenv: py36-django22 + - python: 3.6 + django: 3.0 + toxenv: py36-django30 + - python: 3.7 + django: 2.2 + toxenv: py37-django22 + - python: 3.7 + django: 3.0 + toxenv: py37-django30 + - python: 3.8 + django: 2.2 + toxenv: py38-django22 + - python: 3.8 + django: 3.0 + toxenv: py38-django30 + - python: 3.9 + django: 2.2 + toxenv: py39-django22 + - python: 3.9 + django: 3.0 + toxenv: py39-django30 steps: - uses: actions/checkout@v2 - - name: Run a one-line script + - uses: actions/setup-python@v1 + name: Set up Python ${{ matrix.python }} ${{ matrix.django }} + with: + python-version: ${{ matrix.python }} + - name: Install pip packages run: | - echo Hello, world 1! + pip install pip --upgrade + pip install codecov poetry tox + - name: "Run tox - testenv: ${{ matrix.toxenv }}" + env: + DJANGO: ${{ matrix.django }} + TOXENV: ${{ matrix.toxenv }} + run: tox + - name: Run notification script + env: + GITHUB_COMMIT_MESSAGE: ${{ github.event.head_commit.message }} + GITHUB_JOB_STATUS: ${{ job.status }} + GITHUB_RUN_URL: ${{ github.event.repository.url }}/actions/runs/${{ github.run_id }} + TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }} + TELEGRAM_TOKEN: ${{ secrets.TELEGRAM_TOKEN }} + run: ./scripts/notification.sh + + quality: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - python: 3.6 + toxenv: quality + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + name: Set up Python ${{ matrix.python }} + with: + python-version: ${{ matrix.python }} + - name: Install pip packages + run: | + pip install pip --upgrade + pip install poetry tox + - name: "Run tox - testenv: ${{ matrix.toxenv }}" + env: + TOXENV: ${{ matrix.toxenv }} + run: tox + + security: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - python: 3.6 + toxenv: security + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v1 + name: Set up Python ${{ matrix.python }} + with: + python-version: ${{ matrix.python }} + - name: Install pip packages + run: | + pip install pip --upgrade + pip install poetry tox + - name: "Run tox - testenv: ${{ matrix.toxenv }}" + env: + TOXENV: ${{ matrix.toxenv }} + run: tox + + # # AUTHOR: Lukasz Dynowski + # # TODO: Uncomment this code only if we agreed for Continuous Delivery. + # # TODO: Update PYPI envars to production values! + # publish: + # needs: + # - test + # - security + # - quality + # runs-on: ubuntu-latest + # strategy: + # matrix: + # include: + # - python: 3.6 + # toxenv: build + # steps: + # - uses: actions/checkout@v2 + # - uses: actions/setup-python@v1 + # name: Set up Python ${{ matrix.python }} + # with: + # python-version: ${{ matrix.python }} + # - name: Install pip packages + # run: | + # pip install pip --upgrade + # pip install poetry tox + # - name: Bump package version + # run: ./scripts/bump.py --dry-run no + # - name: "Build package - testenv: ${{ matrix.toxenv }}" + # env: + # TOXENV: ${{ matrix.toxenv }} + # run: tox + # - name: "Publish - testenv: ${{ matrix.toxenv }}" + # env: + # PYPI_ACCESS_TOKEN: ${{ secrets.PYPI_ACCESS_TOKEN }} + # # Values production 'https://pypi.org/', testing 'https://test.pypi.org/legacy/' + # PYPI_REPOSITORY_URL: https://test.pypi.org/legacy/ + # # Values production 'pypi', testing 'testpypi' + # PYPI_REPOSITORY: testpypi + # PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }} + # run: | + # poetry config repositories.$PYPI_REPOSITORY $PYPI_REPOSITORY_URL + # poetry config pypi-token.$PYPI_REPOSITORY $PYPI_ACCESS_TOKEN + # poetry publish -r $PYPI_REPOSITORY -u $PYPI_USERNAME + # - name: Create bump commit + # uses: EndBug/add-and-commit@latest + # with: + # author_name: github_actions + # author_email: github_actions + # message: PR Auto bumping package version + # add: pyproject.toml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f1223c3..0000000 --- a/.travis.yml +++ /dev/null @@ -1,46 +0,0 @@ -language: python -dist: focal - -stages: - - quality - - test - -cache: - pip: true - directories: - - $HOME/.cache/pypoetry - -matrix: - fast_finish: true - include: - - { python: "3.6", env: DJANGO=2.2 TOXENV=py36-django22 } - - { python: "3.6", env: DJANGO=3.0 TOXENV=py36-django30 } - - { python: "3.7", env: DJANGO=2.2 TOXENV=py37-django22 } - - { python: "3.7", env: DJANGO=3.0 TOXENV=py37-django30 } - - { python: "3.8", env: DJANGO=2.2 TOXENV=py38-django22 } - - { python: "3.8", env: DJANGO=3.0 TOXENV=py38-django30 } - - { python: "3.9", env: DJANGO=2.2 TOXENV=py39-django22 } - - { python: "3.9", env: DJANGO=3.0 TOXENV=py39-django30 } - - { python: "3.7", env: TOXENV=quality } - -before_install: - - pip install codecov - -install: - - pip install pip -U - - pip install tox-travis - - curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python - - source $HOME/.poetry/env - -script: - - tox - -after_script: - - bash ./scripts/travis_telegram_notification.sh - -after_success: - - pip install codecov - - codecov -e TOXENV,DJANGO - -notifications: - email: false diff --git a/scripts/bump.py b/scripts/bump.py index 85629fb..f7b5cee 100755 --- a/scripts/bump.py +++ b/scripts/bump.py @@ -28,18 +28,7 @@ def get_args(): help="Patch version bump. Empty flag will auto bump current version.", ) parser.add_argument( - "--update", - type=str, - default="yes", - choices=["yes", "no"], - help="Make update to config file.", - ) - parser.add_argument( - "--dry-run", - type=str, - default="yes", - choices=["yes", "no"], - help="Print versions.", + "--dry-run", type=str, default="yes", choices=["yes", "no"], help="Dry run" ) return parser.parse_args() @@ -132,10 +121,7 @@ def bump_version(major, minor, patch): if args.dry_run == "yes": print(f"Current version: {CURRENT_VERSION}") print(f"New version: {NEW_VERSION}") - exit(0) - - # Update package version - if args.update == "yes": + else: with open(pyproject_file_path, "r") as file: content = file.read() diff --git a/scripts/notification.sh b/scripts/notification.sh index b9c5052..9c8ed35 100755 --- a/scripts/notification.sh +++ b/scripts/notification.sh @@ -1,9 +1,5 @@ #!/bin/sh -# TODO: Update/missing GitHub Actions environment variables with TELEGRAM_TOKEN -# TODO: Update/missing GitHub Actions environment variables with TELEGRAM_CHAT_ID -# TODO: Once this script is working file `travis_telegram_notification.sh` should be deleted! - # Get the token from Travis environment vars and build the bot URL: BOT_URL="https://api.telegram.org/bot${TELEGRAM_TOKEN}/sendMessage" diff --git a/scripts/travis_telegram_notification.sh b/scripts/travis_telegram_notification.sh deleted file mode 100755 index bd137bc..0000000 --- a/scripts/travis_telegram_notification.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh - -# Get the token from Travis environment vars and build the bot URL: -BOT_URL="https://api.telegram.org/bot${TELEGRAM_TOKEN}/sendMessage" - -# Set formatting for the message. Can be either "Markdown" or "HTML" -PARSE_MODE="Markdown" - -# Use built-in Travis variables to check if all previous steps passed: -if [ $TRAVIS_TEST_RESULT -ne 0 ]; then - build_status="failed" -else - build_status="succeeded" -fi - -# Define send message function. parse_mode can be changed to -# HTML, depending on how you want to format your message: -send_msg () { - curl -s -X POST ${BOT_URL} -d chat_id=$TELEGRAM_CHAT_ID \ - -d text="$1" -d parse_mode=${PARSE_MODE} -} - -# Send message to the bot with some pertinent details about the job -# Note that for Markdown, you need to escape any backtick (inline-code) -# characters, since they're reserved in bash -send_msg " ----------------------------------------------------- -Travis build *${build_status}!* -\`Repository: ${TRAVIS_REPO_SLUG}\` -\`Branch: ${TRAVIS_BRANCH}\` -\`Environment: ${TOXENV}\` -*Commit Msg:* -${TRAVIS_COMMIT_MESSAGE} - -[See complete job log here](${TRAVIS_JOB_WEB_URL}) ------------------------------------------------------ -" \ No newline at end of file diff --git a/tox.ini b/tox.ini index 31d1883..58b3f00 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] isolated_build = true -envlist = py{36,37,38,39}-django{22,30}, quality, security +envlist = py{36,37,38,39}-django{22,30}, quality, security, build [travis:env] DJANGO = @@ -102,3 +102,11 @@ commands = poetry update --lock poetry install -vvv poetry run bandit . + +[testenv:build] +whitelist_externals = poetry +skip_install = true +commands = + poetry update --lock + poetry install -vvv + poetry build