From c3f433b3d3acfa87e424f64433e1bb54da7c418c Mon Sep 17 00:00:00 2001 From: pulpbot Date: Thu, 30 Nov 2023 14:42:00 +0000 Subject: [PATCH] Update CI files [noissue] --- .github/workflows/build.yml | 8 +- .github/workflows/release.yml | 237 +++++++++++++----- .../workflows/scripts/publish_client_gem.sh | 25 +- .../workflows/scripts/publish_client_pypi.sh | 6 +- .../workflows/scripts/publish_plugin_pypi.sh | 26 +- 5 files changed, 215 insertions(+), 87 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6d75d3fbcff..ed94af69910 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,11 +29,13 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging wheel + pip install packaging twine wheel echo ::endgroup:: - name: "Build package" - run: "python3 setup.py sdist bdist_wheel --python-tag py3" - - name: 'Upload Package whl' + run: | + python3 setup.py sdist bdist_wheel --python-tag py3 + twine check + - name: "Upload Package whl" uses: "actions/upload-artifact@v3" with: name: "plugin_package" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a966918e27b..1198d3e351d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,29 +51,38 @@ jobs: env: SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: Create the release commit, tag it, create a post-release commit, and build plugin package - run: python .github/workflows/scripts/release.py ${{ github.event.inputs.release }} + - name: "Create the release commit, tag it, create a post-release commit, and build plugin package" + run: | + python .github/workflows/scripts/release.py ${{ github.event.inputs.release }} + + - name: "Upload Package whl" + uses: "actions/upload-artifact@v3" + with: + name: "plugin_package" + path: "pulpcore/dist/" + if-no-files-found: "error" - name: "Tar files" run: | tar -cvf pulpcore.tar . - name: "Upload Artifact" - uses: actions/upload-artifact@v3 + uses: "actions/upload-artifact@v3" with: name: "pulpcore.tar" path: "pulpcore/pulpcore.tar" build-bindings-docs: - needs: build-artifacts - runs-on: ubuntu-latest + needs: + - "build-artifacts" + runs-on: "ubuntu-latest" # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests env: - TEST: docs + TEST: "docs" steps: - - uses: actions/download-artifact@v3 + - uses: "actions/download-artifact@v3" with: name: "pulpcore.tar" path: "pulpcore/" @@ -114,16 +123,15 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - name: Install + - name: "Install" run: | - export PLUGIN_VERSION=${{ github.event.inputs.release }} .github/workflows/scripts/install.sh + shell: "bash" env: PY_COLORS: "1" ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - shell: bash - name: "Install Python client" run: | @@ -134,20 +142,20 @@ jobs: .github/workflows/scripts/install_ruby_client.sh shell: "bash" - name: "Upload python client packages" - uses: actions/upload-artifact@v3 + uses: "actions/upload-artifact@v3" with: name: "python-client.tar" path: "pulpcore/core-python-client.tar" if-no-files-found: "error" - name: "Upload python client docs" - uses: actions/upload-artifact@v3 + uses: "actions/upload-artifact@v3" with: name: "python-client-docs.tar" path: "pulpcore/core-python-client-docs.tar" if-no-files-found: "error" - name: "Upload ruby client packages" - uses: actions/upload-artifact@v3 + uses: "actions/upload-artifact@v3" with: name: "ruby-client.tar" path: "pulpcore/core-ruby-client.tar" @@ -178,36 +186,61 @@ jobs: docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" - publish: - runs-on: ubuntu-latest - needs: build-bindings-docs + publish-tag: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" env: - TEST: publish - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" steps: - - uses: actions/download-artifact@v3 + - uses: "actions/download-artifact@v3" with: name: "pulpcore.tar" path: "pulpcore/" - - uses: "actions/setup-python@v4" - with: - python-version: "3.8" - - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" + - name: "Untar repository" + run: | + tar -xf pulpcore.tar - name: "Configure Git with pulpbot name and email" run: | git config --global user.name 'pulpbot' git config --global user.email 'pulp-infra@redhat.com' - - name: Untar repository + - name: "Push branch and tag to GitHub" run: | - tar -xf pulpcore.tar + bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} + publish-package: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + - "publish-tag" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulpcore" + + - uses: "actions/download-artifact@v3" + with: + name: "plugin_package" + path: "pulpcore/dist/" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.8" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install twine + echo ::endgroup:: - name: "Setting secrets" run: | @@ -215,34 +248,25 @@ jobs: env: SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: "Install python dependencies" + - name: "Deploy plugin to pypi" run: | - echo ::group::PYDEPS - pip install twine gitpython requests 'packaging~=21.3' mkdocs pymdown-extensions 'Jinja2<3.1' - echo ::endgroup:: - - - name: Push branch and tag to GitHub - run: bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} - - name: "Download built docs" - uses: actions/download-artifact@v3 - with: - name: "docs.tar" - path: "pulpcore/" + .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.event.inputs.release }} + publish-python-bindings: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + - "publish-tag" + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - name: "Download Python client docs" - uses: actions/download-artifact@v3 + steps: + - uses: "actions/checkout@v4" with: - name: "python-client-docs.tar" - path: "pulpcore/" + fetch-depth: 1 + path: "pulpcore" - - name: Publish docs to pulpproject.org - run: | - tar -xvf docs.tar - .github/workflows/scripts/publish_docs.sh tag ${{ github.event.inputs.release }} - - name: Deploy plugin to pypi - run: bash .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.event.inputs.release }} - name: "Download Python client" - uses: actions/download-artifact@v3 + uses: "actions/download-artifact@v3" with: name: "python-client.tar" path: "pulpcore/" @@ -251,11 +275,42 @@ jobs: run: | tar -xvf core-python-client.tar + - uses: "actions/setup-python@v4" + with: + python-version: "3.8" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install twine + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + - name: "Publish client to pypi" run: | - bash .github/workflows/scripts/publish_client_pypi.sh + bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.event.inputs.release }} + publish-ruby-bindings: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + - "publish-tag" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulpcore" + - name: "Download Ruby client" - uses: actions/download-artifact@v3 + uses: "actions/download-artifact@v3" with: name: "ruby-client.tar" path: "pulpcore/" @@ -264,14 +319,82 @@ jobs: run: | tar -xvf core-ruby-client.tar + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "2.6" + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + - name: "Publish client to rubygems" run: | - bash .github/workflows/scripts/publish_client_gem.sh + bash .github/workflows/scripts/publish_client_gem.sh ${{ github.event.inputs.release }} + publish-docs: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + - "publish-tag" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - name: Create release on GitHub - uses: actions/github-script@v7 + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulpcore" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.8" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install 'packaging~=21.3' requests + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Download built docs" + uses: "actions/download-artifact@v3" + with: + name: "docs.tar" + path: "pulpcore/" + + - name: "Download Python client docs" + uses: "actions/download-artifact@v3" + with: + name: "python-client-docs.tar" + path: "pulpcore/" + + - name: "Publish docs to pulpproject.org" + run: | + tar -xvf docs.tar + .github/workflows/scripts/publish_docs.sh tag ${{ github.event.inputs.release }} + + create-gh-release: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + - "publish-tag" + - "publish-package" + - "publish-python-bindings" + - "publish-ruby-bindings" + - "publish-docs" + + steps: + - name: "Create release on GitHub" + uses: "actions/github-script@v7" env: - TAG_NAME: ${{ inputs.release }} + TAG_NAME: "${{ inputs.release }}" with: script: | const { TAG_NAME } = process.env; diff --git a/.github/workflows/scripts/publish_client_gem.sh b/.github/workflows/scripts/publish_client_gem.sh index 3a22f0e7472..41f9ad770b9 100755 --- a/.github/workflows/scripts/publish_client_gem.sh +++ b/.github/workflows/scripts/publish_client_gem.sh @@ -12,27 +12,24 @@ set -euv # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. - -mkdir ~/.gem || true -touch ~/.gem/credentials -echo "--- -:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials -sudo chmod 600 ~/.gem/credentials - -export VERSION=$(ls pulpcore_client* | sed -rn 's/pulpcore_client-(.*)\.gem/\1/p') +VERSION="$1" if [[ -z "$VERSION" ]]; then - echo "No client package found." - exit + echo "No version specified." + exit 1 fi -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/pulpcore_client/versions/$VERSION) +RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://rubygems.org/gems/pulpcore_client/versions/$VERSION")" -if [ "$response" == "200" ]; +if [ "$RESPONSE" == "200" ]; then echo "pulpcore client $VERSION has already been released. Skipping." exit fi -GEM_FILE="$(ls pulpcore_client-*)" -gem push ${GEM_FILE} +mkdir -p ~/.gem +touch ~/.gem/credentials +echo "--- +:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials +sudo chmod 600 ~/.gem/credentials +gem push "pulpcore_client-${VERSION}.gem" diff --git a/.github/workflows/scripts/publish_client_pypi.sh b/.github/workflows/scripts/publish_client_pypi.sh index c7e4e166696..3593bfd1178 100755 --- a/.github/workflows/scripts/publish_client_pypi.sh +++ b/.github/workflows/scripts/publish_client_pypi.sh @@ -12,11 +12,11 @@ set -euv # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")/../../.." -VERSION="$(ls dist | sed -rn 's/pulpcore-client-(.*)\.tar.gz/\1/p')" +VERSION="$1" if [[ -z "$VERSION" ]]; then - echo "No client package found." - exit + echo "No version specified." + exit 1 fi RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulpcore-client/$VERSION/")" diff --git a/.github/workflows/scripts/publish_plugin_pypi.sh b/.github/workflows/scripts/publish_plugin_pypi.sh index da9b6111dd8..2f1f167aa5f 100755 --- a/.github/workflows/scripts/publish_plugin_pypi.sh +++ b/.github/workflows/scripts/publish_plugin_pypi.sh @@ -7,21 +7,27 @@ # # For more info visit https://github.com/pulp/plugin_template +set -euv + # make sure this script runs at the repo root cd "$(dirname "$(realpath -e "$0")")"/../../.. -set -euv +VERSION="$1" -export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/pulpcore/$1/) -if [ "$response" == "200" ]; +if [[ -z "$VERSION" ]]; then + echo "No version specified." + exit 1 +fi + +RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/pulpcore/$VERSION/")" + +if [ "$RESPONSE" == "200" ]; then - echo "pulpcore $1 has already been released. Skipping." + echo "pulpcore $VERSION has already been released. Skipping." exit fi -twine check dist/pulpcore-$1-py3-none-any.whl || exit 1 -twine check dist/pulpcore-$1.tar.gz || exit 1 -twine upload dist/pulpcore-$1-py3-none-any.whl -u pulp -p $PYPI_PASSWORD -twine upload dist/pulpcore-$1.tar.gz -u pulp -p $PYPI_PASSWORD - -exit $? +twine upload -u pulp -p "$PYPI_PASSWORD" \ +"dist/pulpcore-$VERSION-py3-none-any.whl" \ +"dist/pulpcore-$VERSION.tar.gz" \ +;