diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..54eeeb3 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +FROM ghcr.io/chinigorg/bosh-package-cf-cli-release-ci:187042013-release-workflow-latest + +RUN apt update && apt install --yes \ + entr \ + fzf \ + htop \ + neovim + +RUN curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..fea1b01 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,25 @@ +{ + "name": "Cloudfoundry CLI Bosh Release", + "dockerFile": "Dockerfile", + "features": { + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}, + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/sshd:1": {} + }, + "customizations": { + "vscode": { + "settings": {}, + "extensions": [ + "DavidAnson.vscode-markdownlint", + "GitHub.copilot", + "GitHub.vscode-pull-request-github", + "editorconfig.editorconfig", + "github.vscode-github-actions", + "ms-vscode.makefile-tools", + "ms-vsliveshare.vsliveshare", + "timonwong.shellcheck", + "vmware.vscode-concourse" + ] + } + } +} diff --git a/.github/.secret b/.github/.secret new file mode 100644 index 0000000..2bd1a61 --- /dev/null +++ b/.github/.secret @@ -0,0 +1,2 @@ +AWS_ACCESS_KEY_ID: fake-aws-access-key +AWS_SECRET_ACCESS_KEY: fake-aws-secret-access-key \ No newline at end of file diff --git a/.github/.vars b/.github/.vars new file mode 100644 index 0000000..eee66b7 --- /dev/null +++ b/.github/.vars @@ -0,0 +1 @@ +AWS_REGION: us-west-2 \ No newline at end of file diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..7fa835c --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,19 @@ +# GHA Workflows + +## [Create Bosh Release](create-bosh-release.yml) + +Why? To create a new cf cli bosh release including major cli versions. + +### Resources +- [Old Concourse implementation of the release pipeline](https://ci.cli.fun/teams/main/pipelines/cf-cli-release-toolsmiths) + - [pipeline definition](../../ci/pipeline-toolsmiths.yml) + +### Plan + +- Acquire cf cli linux binaries for v6, v7, and v8 from s3 +- Detect latest tag under each major version + +- ... + +- Upload (where?) newly created cf cli bosh release. +- Update Releases section on GitHub https://github.com/cloudfoundry/bosh-package-cf-cli-release/releases \ No newline at end of file diff --git a/.github/workflows/create-bosh-release.yml b/.github/workflows/create-bosh-release.yml new file mode 100644 index 0000000..acf1039 --- /dev/null +++ b/.github/workflows/create-bosh-release.yml @@ -0,0 +1,93 @@ +name: Create Release +on: + workflow_dispatch: + push: + +defaults: + run: + shell: bash + # working-directory: ${{ }} + +jobs: + bosh_release_create_candidate: + runs-on: ubuntu-latest + container: + image: "ghcr.io/${{ github.actor }}/bosh-package-cf-cli-release-ci:187042013-release-workflow-latest" + + steps: + - name: Checkout cli bosh release repo + uses: actions/checkout@v4 + + - name: Acquire latest CF CLI V8 + run: | + mkdir -p v8-cli-binary + cd v8-cli-binary + wget --trust-server-names "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v8&source=bosh-package-cf-cli-release-workflow" + tar -xvzf cf8-cli_*_linux_x86-64.tgz --wildcards 'cf*' + ./cf version + + - name: Create bosh release candidate + id: create-bosh-release-candidate-step + env: + TRACE: ${{ runner.debug }} + # S3 final release config to upload + # ACCESS_KEY_ID: + # SECRET_KEY: + run: | + source ./ci/scripts/create-bosh-release-candidate.sh + create_bosh_release_candidate 8 + + mkdir -p candidate-release-output + bosh create-release --timestamp-version --tarball=./candidate-release-output/cf-cli-dev-release.tgz + + ## Store tgz with bosh release candidate + + - name: Push changes + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: ${{ github.ref }} + + ## Claim Shepherd env + ## Deploy cli bosh release candidate + ## Unclaim shepherd env + + + ## Finalize bosh release + ## Get final release from assets + ## Push commit with final release + - name: Finalize bosh release + # if: steps.create-bosh-release-candidate-step.outputs.blobs_updated == 'yes' + # env: + # TRACE: ${{ runner.debug }} + # S3 final release config to upload + # ACCESS_KEY_ID: + # SECRET_KEY: + run: | + set -o errexit -o nounset -o pipefail + [[ "${TRACE:-0}" == "1" ]] && set -o xtrace + + # if [[ "${{ steps.create-bosh-release-candidate-step.outputs.blobs_updated }}" == "yes" ]]; then + + # echo "Bosh Blobs: initial state" + # bosh blobs + + ## Determine next release version number + + # RELEASE_VERSION=$(cat release-version/version) + # git config --global --add safe.directory "$(pwd)" + # git config user.name "github-actions[bot]" + # git config user.email "41898282+github-actions[bot]@users.noreply.github.com " + + # bosh create-release --final --version="${RELEASE_VERSION}" --tarball="./cf-cli-v${RELEASE_VERSION}.tgz" + + # git add --all + # git status + # git commit -m "create final release ${RELEASE_VERSION}" + + ls -lah ./candidate-release-output/cf-cli-dev-release.tgz + + - name: Setup tmate session + if: ${{ always() && runner.debug }} + uses: mxschmitt/action-tmate@v3 + timeout-minutes: 15 diff --git a/.github/workflows/ensure-ci-image.yml b/.github/workflows/ensure-ci-image.yml new file mode 100644 index 0000000..08e6b4c --- /dev/null +++ b/.github/workflows/ensure-ci-image.yml @@ -0,0 +1,77 @@ +name: Ensure CI image + +on: + workflow_dispatch: + +env: + IMAGE_REGISTRY: ghcr.io + CI_DOCKERFILE_DIR: ./ci # Relative to project root + CI_DOCKERFILE_PATH: Dockerfile # Relative to CI_DOCKERFILE_DIR + CI_DOCKERFILE_MOST_RECENT_SHA: # Determined dynamically later on + +jobs: + calculate-latest-label: + runs-on: ubuntu-latest + + permissions: + contents: read + + outputs: + ci_dockerfile_latest_sha: ${{ steps.calculate_latest_sha.outputs.ci_dockerfile_latest_sha }} + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Calculate label for CI image + id: calculate_latest_sha + env: + IMAGE_NAME: ${{ github.repository }} + run: | + dockerfile_path=${CI_DOCKERFILE_DIR}/${CI_DOCKERFILE_PATH} + + [[ ! -f ${dockerfile_path} ]] && echo "Could not find Dockerfile at ${dockerfile_path}" 1>&2 && exit 1 + + echo "ci_dockerfile_latest_sha=$(git log --max-count 1 --pretty=format:%H "${dockerfile_path}")" >> $GITHUB_OUTPUT + + build-and-push-ci-image: + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write + + needs: + - calculate-latest-label + + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: Login to GitHub container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.IMAGE_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract metadata + uses: docker/metadata-action@v5 + id: meta + with: + images: ${{ env.IMAGE_REGISTRY }}/${{ github.repository }}-ci + tags: | + type=raw,value=${{ needs.calculate-latest-label.outputs.ci_dockerfile_latest_sha }} + type=raw,value=${{ github.ref_name }}-latest + + - name: Build and push CI image + uses: docker/build-push-action@v5 + with: + push: true + context: ${{ env.CI_DOCKERFILE_DIR }} + file: ${{ env.CI_DOCKERFILE_DIR }}/${{ env.CI_DOCKERFILE_PATH }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..15d5a8f --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,15 @@ +name: Lint shell scripts +on: + workflow_dispatch: + +jobs: + lint: + runs-on: ubuntu-latest + container: ghcr.io/chinigorg/bosh-package-cf-cli-release:187042013-release-workflow-latest + + steps: + - name: Checkout cli bosh release repo + uses: actions/checkout@v4 + + - name: lint + run: find ./ci -type f -name '*.sh' | xargs -t shellcheck \ No newline at end of file diff --git a/.github/workflows/manual-github-release.yml b/.github/workflows/manual-github-release.yml deleted file mode 100644 index f345430..0000000 --- a/.github/workflows/manual-github-release.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Manual GH release -on: - workflow_dispatch: - inputs: - release_version: - description: 'The version number of the release you want to create. Do not include the v. (Example: 1.38.0)' - required: true - -jobs: - create_github_release: - runs-on: ubuntu-latest - steps: - - name: Download Release from S3 - uses: keithweaver/aws-s3-github-action@v1.0.0 - with: - command: cp - source: s3://cf-cli-bosh-release/cf-cli-v${{ github.event.inputs.release_version }}.tgz - destination: ./cf-cli-v${{ github.event.inputs.release_version }}.tgz - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: us-west-1 - - name: Upload Release to Github - uses: pivotalsoftware/action-gh-release@v1 - with: - draft: true - name: "DRAFT v${{ github.event.inputs.release_version }}" - tag_name: "v${{ github.event.inputs.release_version }}" - repository: bosh-packages/cf-cli-release # repo to draft a release under - fail_on_unmatched_files: true - files: ./cf-cli-v${{ github.event.inputs.release_version }}.tgz diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7d629ad --- /dev/null +++ b/Makefile @@ -0,0 +1,32 @@ +ifndef GITHUB_USER + $(error GITHUB_USER is not set) +endif + +create-bosh-release: + act \ + --actor "${GITHUB_USER}" \ + --secret GITHUB_TOKEN="${GITHUB_TOKEN}" \ + --workflows .github/workflows/create-bosh-release.yml + +ensure-ci-image: + act \ + --actor "${GITHUB_USER}" \ + --secret GITHUB_TOKEN="${GITHUB_TOKEN}" \ + --workflows .github/workflows/ensure-ci-image.yml + +lint: + act \ + --actor "${GITHUB_USER}" \ + --secret GITHUB_TOKEN="${GITHUB_TOKEN}" \ + --workflows .github/workflows/lint.yml + +run: + ls **/* | entr -c \ + act \ + --actor "${GITHUB_USER}" \ + --secret GITHUB_TOKEN="${GITHUB_TOKEN}" \ + --workflows .github/workflows/create-bosh-release.yml \ + --job bosh_release_create_candidate + +hijack-act: + ./ci/scripts/hijack-act.sh diff --git a/ci/Dockerfile b/ci/Dockerfile new file mode 100644 index 0000000..bd638a8 --- /dev/null +++ b/ci/Dockerfile @@ -0,0 +1,93 @@ +FROM summerwind/actions-runner:latest + +ENV bbl_version 8.4.111 +ENV bosh_cli_version 7.2.3 +ENV NODE_VERSION 22.2.0 +ENV terraform_version 0.11.5 + +USER root +RUN usermod -a -G sudo root + +RUN \ + apt-get update && \ + apt-get -y install \ + shellcheck \ + yamllint && \ + apt list --installed + +RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + ppc64el) ARCH='ppc64le';; \ + s390x) ARCH='s390x';; \ + arm64) ARCH='arm64';; \ + armhf) ARCH='armv7l';; \ + i386) ARCH='x86';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac \ + # use pre-existing gpg directory, see https://github.com/nodejs/docker-node/pull/1895#issuecomment-1550389150 + && export GNUPGHOME="$(mktemp -d)" \ + # gpg keys listed at https://github.com/nodejs/node#release-keys + && set -ex \ + && for key in \ + 4ED778F539E3634C779C87C6D7062848A1AB005C \ + 141F07595B7B3FFE74309A937405533BE57C7D57 \ + 74F12602B6F1C4E913FAA37AD3A89613643B6201 \ + DD792F5973C6DE52C432CBDAC77ABFA00DDBF2B7 \ + 61FC681DFB92A079F1685E77973F295594EC4689 \ + 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + 890C08DB8579162FEE0DF9DB8BEAB4DFCF555EF4 \ + C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ + 108F52B48DB57BB0CC439B2997B01419BD92F80A \ + A363A499291CBBC940DD62E41F10027AF002F8B0 \ + CC68F5A3106FF448322E48ED27F5E38D5B0A215F \ + ; do \ + gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" || \ + gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && gpgconf --kill all \ + && rm -rf "$GNUPGHOME" \ + && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ + # smoke tests + && node --version \ + && npm --version + +# bosh-cli +RUN \ + wget --no-verbose https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-${bosh_cli_version}-linux-amd64 --output-document="/usr/local/bin/bosh" && \ + chmod +x /usr/local/bin/bosh + +# RUN latest_bosh_url=$(curl -s https://api.github.com/repos/cloudfoundry/bosh-cli/releases/latest \ +# | jq -r ".assets[] | select(.name | test(\"^bosh-cli-.*-linux-amd64$\")) | .browser_download_url") && \ +# curl ${latest_bosh_url:?} --silent --output /usr/local/bin/bosh --location && \ +# chmod +x /usr/local/bin/bosh + + # bbl and dependencies +RUN \ + wget --no-verbose https://github.com/cloudfoundry/bosh-bootloader/releases/download/v${bbl_version}/bbl-v${bbl_version}_linux_x86-64 -P /tmp && \ + mv /tmp/bbl-* /usr/local/bin/bbl && \ + cd /usr/local/bin && \ + chmod +x bbl + +RUN \ + wget --no-verbose https://github.com/cloudfoundry/bosh-bootloader/archive/v${bbl_version}.tar.gz -P /tmp && \ + mkdir -p /var/repos/bosh-bootloader && \ + tar xvf /tmp/v${bbl_version}.tar.gz --strip-components=1 -C /var/repos/bosh-bootloader && \ + rm -rf /tmp/* + +RUN \ + wget --no-verbose "https://releases.hashicorp.com/terraform/${terraform_version}/terraform_${terraform_version}_linux_amd64.zip" -P /tmp && \ + cd /tmp && \ + curl https://releases.hashicorp.com/terraform/${terraform_version}/terraform_${terraform_version}_SHA256SUMS | grep linux_amd64 | shasum -c - && \ + unzip "/tmp/terraform_${terraform_version}_linux_amd64.zip" -d /tmp && \ + mv /tmp/terraform /usr/local/bin/terraform && \ + cd /usr/local/bin && \ + chmod +x terraform && \ + rm -rf /tmp/* \ No newline at end of file diff --git a/ci/cli-release-base/Dockerfile b/ci/cli-release-base/Dockerfile deleted file mode 100644 index 34c06af..0000000 --- a/ci/cli-release-base/Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -FROM ubuntu:trusty - -# Last updated on 2020-7-16, if updated again please add concourse automation -# to bump versions of these binaries: https://www.pivotaltracker.com/n/projects/2450704/stories/173838043 -ENV bosh_cli_version 7.2.3 -ENV bbl_version 8.4.111 -ENV terraform_version 0.11.5 - -RUN \ - apt-get update && \ - apt-get -y install \ - build-essential \ - git \ - libreadline6 \ - libreadline6-dev \ - libsqlite3-dev \ - libssl-dev \ - libxml2-dev \ - libxslt-dev \ - libyaml-dev \ - openssl \ - software-properties-common \ - sqlite \ - unzip \ - wget \ - curl \ - zlib1g-dev \ - zlibc && \ - add-apt-repository ppa:brightbox/ruby-ng -y && \ - apt-get update && \ - apt-get -y install \ - ruby2.3 \ - ruby2.3-dev && \ - apt-get remove -y --purge software-properties-common - -# bosh-cli -RUN \ - wget https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-${bosh_cli_version}-linux-amd64 --output-document="/usr/bin/bosh" && \ - chmod +x /usr/bin/bosh - - # bbl and dependencies -RUN \ - wget https://github.com/cloudfoundry/bosh-bootloader/releases/download/v${bbl_version}/bbl-v${bbl_version}_linux_x86-64 -P /tmp && \ - mv /tmp/bbl-* /usr/local/bin/bbl && \ - cd /usr/local/bin && \ - chmod +x bbl - -RUN \ - wget https://github.com/cloudfoundry/bosh-bootloader/archive/v${bbl_version}.tar.gz -P /tmp && \ - mkdir -p /var/repos/bosh-bootloader && \ - tar xvf /tmp/v${bbl_version}.tar.gz --strip-components=1 -C /var/repos/bosh-bootloader && \ - rm -rf /tmp/* - -RUN \ - wget "https://releases.hashicorp.com/terraform/${terraform_version}/terraform_${terraform_version}_linux_amd64.zip" -P /tmp && \ - cd /tmp && \ - curl https://releases.hashicorp.com/terraform/${terraform_version}/terraform_${terraform_version}_SHA256SUMS | grep linux_amd64 | shasum -c - && \ - unzip "/tmp/terraform_${terraform_version}_linux_amd64.zip" -d /tmp && \ - mv /tmp/terraform /usr/local/bin/terraform && \ - cd /usr/local/bin && \ - chmod +x terraform && \ - rm -rf /tmp/* diff --git a/ci/scripts/create-bosh-release-candidate.sh b/ci/scripts/create-bosh-release-candidate.sh new file mode 100755 index 0000000..9a5b9a5 --- /dev/null +++ b/ci/scripts/create-bosh-release-candidate.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -o errexit -o nounset -o pipefail +[[ "${TRACE:-0}" == "1" ]] && set -o xtrace + +git config --global --add safe.directory "$(pwd)" + +create_bosh_release_candidate() { + cli_version_major=$1 + + latest_cli_version=$(ls -1 v${cli_version_major}-cli-binary | grep "^cf${cli_version_major}-cli" | cut -d_ -f2) + old_blob_path=$(bosh blobs --column=path | grep "cf${cli_version_major}-cli" | tr -d '[:space:]') + old_cli_version=$(echo "${old_blob_path:?}" | cut -d_ -f2) + + if [[ "${old_cli_version:?}" != "${latest_cli_version:?}" ]]; then + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com " + + echo "Bosh Blobs: initial state" + bosh blobs + bosh remove-blob "${old_blob_path}" + + echo "Bosh Blobs: without old blob" + bosh blobs + bosh add-blob "v${cli_version_major}-cli-binary/cf${cli_version_major}-cli_${latest_cli_version}_linux_x86-64.tgz" "cf${cli_version_major}-cli_${latest_cli_version}_linux_x86-64.tgz" + + echo "Bosh Blobs: added new blob" + bosh blobs + #TODO: add bosh upload-blobs + + git status + git add config/blobs.yml + git status + git commit -m "bump v${cli_version_major} cli from ${old_cli_version} to ${latest_cli_version}" + + echo "blobs_updated=yes" >> $GITHUB_OUTPUT + else + echo "Release has latest v${cli_version_major} CLI version, skipping bump." + + # echo "blobs_updated=no" >> $GITHUB_OUTPUT + fi + + git log -3 +} \ No newline at end of file diff --git a/ci/scripts/hijack-act.sh b/ci/scripts/hijack-act.sh new file mode 100755 index 0000000..fce4278 --- /dev/null +++ b/ci/scripts/hijack-act.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +image_id=$(docker ps --format=json | jq --slurp '.[] | select(.Image == "cfcli/cli-release-base") | .ID' --raw-output) +docker exec --interactive --tty "${image_id}" /bin/bash \ No newline at end of file diff --git a/config/blobs.yml b/config/blobs.yml index a388293..8a5559a 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -2,11 +2,10 @@ cf-cli_6.53.0_linux_x86-64.tgz: size: 8973024 object_id: ae691825-0baa-438f-5765-fa781d5f9982 sha: sha256:09664d1469fb8a0ddba804343121aba7d8f64ba6bfde75a53f6e29d6600b2342 -cf7-cli_7.7.10_linux_x86-64.tgz: +cf7-cli_7.99.99_linux_x86-64.tgz: size: 7654446 object_id: 19a7ec07-4a09-4acb-545e-4c72140f2d28 sha: sha256:9d8207b3a3e7f1d38c79ceab71fd6c110b9fa6f580f65fb3b8067d697cf42115 -cf8-cli_8.99.99_linux_x86-64.tgz: - size: 8966421 - object_id: fb7941f6-596b-4ca2-4c9f-4b987a088e72 - sha: sha256:bfda5c48cd281985f3005f584c7034227ebe814c51fc35211ff3819f05ac2c58 +cf8-cli_8.7.10_linux_x86-64.tgz: + size: 8966909 + sha: sha256:3ecb1cf3d677aa314f6cb638518dc01832426b4f60f795dee648aa78c8bdc480