diff --git a/.changeset/config.json b/.changeset/config.json new file mode 100644 index 0000000..8639ae5 --- /dev/null +++ b/.changeset/config.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://unpkg.com/@changesets/config@3.0.1/schema.json", + "changelog": "@changesets/cli/changelog", + "commit": false, + "fixed": [], + "linked": [], + "access": "restricted", + "baseBranch": "main", + "updateInternalDependencies": "patch", + "ignore": [] +} diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..9bba0e8 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,9 @@ +FROM ghcr.io/cloudfoundry/bosh-package-cf-cli-release-ci:main-latest + +RUN apt update && apt install --yes \ + entr \ + fzf \ + htop \ + neovim + +RUN curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..71a0ac8 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,24 @@ +{ + "name": "Cloudfoundry CLI Bosh Release", + "dockerFile": "Dockerfile", + "features": { + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}, + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/sshd:1": {} + }, + "customizations": { + "vscode": { + "settings": {}, + "extensions": [ + "DavidAnson.vscode-markdownlint", + "GitHub.vscode-pull-request-github", + "editorconfig.editorconfig", + "github.vscode-github-actions", + "ms-vscode.makefile-tools", + "ms-vsliveshare.vsliveshare", + "timonwong.shellcheck", + "vmware.vscode-concourse" + ] + } + } +} diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..6e20168 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +root = true + +[*] +end_of_line = lf +indent_size = 2 +indent_style = space +insert_final_newline = true + +[Makefile] +indent_style = tab diff --git a/.github/.secrets b/.github/.secrets new file mode 100644 index 0000000..b6d925c --- /dev/null +++ b/.github/.secrets @@ -0,0 +1,5 @@ +AWS_S3_ACCESS_KEY_ID: fake-aws-s3-access-key-id +AWS_S3_ASSUME_ROLE_ARN: fake-aws-s3-assume-role-arn +AWS_S3_SECRET_ACCESS_KEY: fake-aws-s3-secret-access-key +SHEPHERD_API_ENDPOINT: fake-shepherd-api-endpoint +SHEPHERD_API_TOKEN: fake-shepherd-api-token diff --git a/.github/.vars b/.github/.vars new file mode 100644 index 0000000..e149561 --- /dev/null +++ b/.github/.vars @@ -0,0 +1,6 @@ +AWS_S3_REGION: us-east-2 +AWS_S3_BUCKET: cf-cli-bosh-release-dev +PRODUCTION: true +# BRANCH_SUFFIX: -shadow +# SKIP_TESTS: false +# ENV_ID: diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..ed54df9 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,19 @@ +# GHA Workflows + +## [Create Bosh Release](create-bosh-release.yml) + +Why? To create a new cf cli bosh release including major cli versions. + +### Resources +- [Old Concourse implementation of the release pipeline](https://ci.cli.fun/teams/main/pipelines/cf-cli-release-toolsmiths) + - [pipeline definition](../../ci/pipeline-toolsmiths.yml) + +### Plan + +- Acquire cf cli linux binaries for v7, and v8 from s3 +- Detect latest tag under each major version + +- ... + +- Upload (where?) newly created cf cli bosh release. +- Update Releases section on GitHub https://github.com/cloudfoundry/bosh-package-cf-cli-release/releases \ No newline at end of file diff --git a/.github/workflows/create-bosh-release.yml b/.github/workflows/create-bosh-release.yml index ccdb80a..31c6942 100644 --- a/.github/workflows/create-bosh-release.yml +++ b/.github/workflows/create-bosh-release.yml @@ -1,18 +1,326 @@ -name: Create Release - +name: Create Bosh Release on: workflow_dispatch: - push: - tags: - - "v9.*" - - "v8.*" - - "v7.*" + inputs: + version_bump_type: + type: choice + default: minor + description: Make a choice + options: + - patch + - minor + - major + +defaults: + run: + shell: bash + +env: + BOSH_DEPLOYMENT: cf-cli-test + BOSH_NON_INTERACTIVE: true + PAGER: cat jobs: - acquire_binaries: + create_bosh_release: + name: Create Bosh Release runs-on: ubuntu-latest + container: + image: "ghcr.io/${{ github.repository }}-ci:${{ github.ref_name }}-latest" + permissions: + contents: write + steps: - - name: Setup upterm session - if: always() - uses: lhotari/action-upterm@v1 - timeout-minutes: 60 + - name: Checkout cli bosh release repo + uses: actions/checkout@v4 + + - name: Acquire latest CF CLI binaries + run: | + ./ci/scripts/download-cf-cli-binary.sh --major-version 7 --output-dir ./build/cf-cli-binaries + ./ci/scripts/download-cf-cli-binary.sh --major-version 8 --output-dir ./build/cf-cli-binaries + + - name: Overwrite config/final.yml with the destination bucket + run: | + cat << EOF > config/final.yml + name: cf-cli + blobstore: + provider: s3 + options: + region: ${{ vars.AWS_S3_REGION }} + bucket_name: ${{ vars.AWS_S3_BUCKET }} + endpoint: http://s3-us-west-1.amazonaws.com + EOF + + - name: Configure S3 backend for bosh in config/private.yml + if: ${{ vars.PRODUCTION == 'true' }} + run: | + cat << EOF > config/private.yml + blobstore: + options: + access_key_id: "${{ secrets.AWS_S3_ACCESS_KEY_ID }}" + assume_role_arn: "${{ secrets.AWS_S3_ASSUME_ROLE_ARN }}" + secret_access_key: "${{ secrets.AWS_S3_SECRET_ACCESS_KEY }}" + EOF + + - name: Create bosh release candidate + run: | + ./ci/scripts/create-bosh-release-candidate.sh \ + --downloaded-binaries-dir ./build/cf-cli-binaries \ + --git-username "github-actions[bot]" \ + --git-email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Store bosh release artifact + uses: actions/upload-artifact@v4 + with: + name: bosh-release-candidate + path: ./cf-cli-dev-release.tgz + + - name: Upload bosh blobs to blobstore + if: ${{ vars.PRODUCTION == 'true' }} + run: | + bosh upload-blobs + + - name: Push changes + if: ${{ vars.PRODUCTION == 'true' }} + uses: ad-m/github-push-action@9870d48124da805820c70ebc6ba563c715551019 + with: + branch: ${{ format('{0}{1}', github.ref, vars.BRANCH_SUFFIX) }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Create test environment + id: create-env + if: ${{ !(vars.ENV_ID || vars.SKIP_TESTS == 'true') }} + uses: a-b/gha-shepherd@latest + with: + api_endpoint: ${{ secrets.SHEPHERD_API_ENDPOINT }} + api_token: ${{ secrets.SHEPHERD_API_TOKEN }} + command: create + debug: ${{ runner.debug }} + duration: 1h + namespace: tas-devex + pool_name: cfd + pool_namespace: official + + - name: Get running env metadata + if: ${{ vars.SKIP_TESTS != 'true' }} + uses: a-b/gha-shepherd@latest + with: + api_endpoint: ${{ secrets.SHEPHERD_API_ENDPOINT }} + api_token: ${{ secrets.SHEPHERD_API_TOKEN }} + command: get + debug: ${{ runner.debug }} + env_file_path: lease.json + env_id: ${{ vars.ENV_ID || steps.create-env.outputs.env-id }} + namespace: tas-devex + + - name: Store lease data + if: ${{ vars.SKIP_TESTS != 'true' }} + uses: actions/upload-artifact@v4 + with: + name: lease-json + path: lease.json + + - name: Deploy and run tests + if: ${{ vars.SKIP_TESTS != 'true' }} + run: | + rm lease.json + + # use previous lease data to reuse leased environment + - name: Retrieve lease data + if: ${{ vars.SKIP_TESTS != 'true' }} + uses: actions/download-artifact@v4 + with: + name: lease-json + + - name: Deploy and run tests + if: ${{ vars.SKIP_TESTS != 'true' }} + run: | + setup_bosh_environment() { + + jq -r .output lease.json > metadata.json + + env_name=$(jq -r .name metadata.json) + jq -r .bosh.jumpbox_private_key metadata.json > /tmp/${env_name}.priv + eval "$(bbl print-env --metadata-file metadata.json)" + + } + + setup_bosh_environment + + bosh delete-deployment --force + bosh deployments + + echo "::group::Bosh deploy" + bosh deploy ./manifests/test.yml + echo "::endgroup::" + + echo "::group::Bosh run-errand cf-cli-7-linux-test" + bosh run-errand cf-cli-7-linux-test \ + | tee /tmp/cf-cli-7-linux-test.out + echo "::endgroup::" + + echo "::group::Bosh run-errand cf-cli-8-linux-test" + bosh run-errand cf-cli-8-linux-test \ + | tee /tmp/cf-cli-8-linux-test.out + echo "::endgroup::" + + bosh delete-deployment + bosh clean-up --all + + - name: Upload artifacts with test results + if: ${{ vars.SKIP_TESTS != 'true' }} + uses: actions/upload-artifact@v4 + with: + name: test-results + path: /tmp/cf-cli-*-test.out + + - name: Retrieve test results + if: ${{ vars.SKIP_TESTS != 'true' }} + uses: actions/download-artifact@v4 + with: + name: test-results + path: /tmp + + - name: Verify test results + if: ${{ vars.SKIP_TESTS != 'true' }} + run: | + set -x -o errexit -o nounset -o pipefail + for test_result in /tmp/cf-cli-*-test.out; do + cf_version=$(grep -e 'cf version' $test_result | sed 's/cf version //') + echo "result_${test_result%.*}=${cf_version}" >> $GITHUB_OUTPUT + done + + - name: Generate next release version + id: generate-next-release-version + env: + # BUMP_TYPE: ${{ inputs.version_bump_type }} + BUMP_TYPE: minor + run: | + _last_cf_cli_bosh_release_version=$(find releases/cf-cli -name 'cf-cli-*.yml' | sort -V | tail -1 | sed 's/.*cf-cli-\([[:digit:]].*\).yml/\1/') + echo "Last CF CLI bosh release version: ${_last_cf_cli_bosh_release_version}" + + if ! [[ "$_last_cf_cli_bosh_release_version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Release version must be in the format X.Y.Z" + exit 1 + fi + + IFS='.' read -r -a version_parts <<< "$_last_cf_cli_bosh_release_version" + major="${version_parts[0]}" + minor="${version_parts[1]}" + patch="${version_parts[2]}" + + case "$BUMP_TYPE" in + major) + major=$((major + 1)) + minor=0 + patch=0 + ;; + minor) + minor=$((minor + 1)) + patch=0 + ;; + patch) + patch=$((patch + 1)) + ;; + *) + echo "BUMP_TYPE must be one of: major, minor, patch" + exit 1 + ;; + esac + + _new_cf_cli_bosh_release_version="${major}.${minor}.${patch}" + + echo "New CF CLI bosh release version: ${_new_cf_cli_bosh_release_version}" + echo "version=${_new_cf_cli_bosh_release_version}" >> $GITHUB_OUTPUT + + + # Finalize bosh release + # Get final release from assets + # Push commit with final release + - name: Finalize bosh release + env: + TRACE: ${{ runner.debug }} + RELEASE_VERSION: ${{ steps.generate-next-release-version.outputs.version }} + run: | + set -o errexit -o nounset -o pipefail + [[ "${TRACE:-0}" == "1" ]] && set -o xtrace + + echo "Releasing version: ${RELEASE_VERSION:?}" + echo "::group::Blobs included in cf-cli release: v${RELEASE_VERSION}" + bosh blobs + echo "::endgroup::" + + git config --global --add safe.directory "$(pwd)" + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com " + + echo "::group::Git state before final release" + git diff --patch + git status + echo "::endgroup::" + + git update-index --skip-worktree config/blobs.yml + git update-index --skip-worktree config/final.yml + + bosh create-release --final --version="${RELEASE_VERSION}" --tarball="./cf-cli-v${RELEASE_VERSION}.tgz" + + echo "::group::Git state after final release" + git diff --patch + git status + echo "::endgroup::" + + git add \ + .final_builds/packages/cf-cli-7-linux/index.yml \ + .final_builds/packages/cf-cli-8-linux/index.yml \ + releases + + echo "::group::Git before the commit" + git diff --patch + git status + echo "::endgroup::" + + _message="create final release ${RELEASE_VERSION}" + git commit --message "${_message}" + git log --pretty=full --max-count=3 + + _git_tag="v${RELEASE_VERSION:?}" + git tag $_git_tag + + - name: Push changes + if: ${{ vars.PRODUCTION == 'true' }} + uses: ad-m/github-push-action@9870d48124da805820c70ebc6ba563c715551019 + with: + branch: ${{ format('{0}{1}', github.ref, vars.BRANCH_SUFFIX) }} + github_token: ${{ secrets.GITHUB_TOKEN }} + tags: true + + - name: Generate Release Notes + run: | + _cf_cli_versions=$(bosh blobs | cut -d_ -f2) + + cat << EOF > release_notes + ### Included CF CLI versions: + + ${_cf_cli_versions:?} + EOF + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + body_path: release_notes + fail_on_unmatched_files: true + make_latest: true + name: v${{ steps.generate-next-release-version.outputs.version }} + tag_name: v${{ steps.generate-next-release-version.outputs.version }} + files: | + ./cf-cli-v${{ steps.generate-next-release-version.outputs.version }}.tgz + + - name: Delete lease with provided env_id and namespace + if: ${{ always() && !vars.ENV_ID && steps.create-env.outcome == 'success' && !runner.debug }} + uses: a-b/gha-shepherd@latest + with: + api_endpoint: ${{ secrets.SHEPHERD_API_ENDPOINT }} + api_token: ${{ secrets.SHEPHERD_API_TOKEN }} + command: delete + debug: ${{ runner.debug }} + env_id: ${{ steps.create-env.outputs.env-id }} + namespace: tas-devex diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..dbb03b5 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,15 @@ +name: Lint shell scripts +on: + workflow_dispatch: + +jobs: + lint: + runs-on: ubuntu-latest + container: ghcr.io/chinigorg/bosh-package-cf-cli-release-ci:187042013-release-workflow-latest + + steps: + - name: Checkout cli bosh release repo + uses: actions/checkout@v4 + + - name: lint + run: find ./ci -type f -name '*.sh' | xargs -t shellcheck -x diff --git a/.github/workflows/manual-github-release.yml b/.github/workflows/manual-github-release.yml deleted file mode 100644 index f345430..0000000 --- a/.github/workflows/manual-github-release.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Manual GH release -on: - workflow_dispatch: - inputs: - release_version: - description: 'The version number of the release you want to create. Do not include the v. (Example: 1.38.0)' - required: true - -jobs: - create_github_release: - runs-on: ubuntu-latest - steps: - - name: Download Release from S3 - uses: keithweaver/aws-s3-github-action@v1.0.0 - with: - command: cp - source: s3://cf-cli-bosh-release/cf-cli-v${{ github.event.inputs.release_version }}.tgz - destination: ./cf-cli-v${{ github.event.inputs.release_version }}.tgz - aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws_region: us-west-1 - - name: Upload Release to Github - uses: pivotalsoftware/action-gh-release@v1 - with: - draft: true - name: "DRAFT v${{ github.event.inputs.release_version }}" - tag_name: "v${{ github.event.inputs.release_version }}" - repository: bosh-packages/cf-cli-release # repo to draft a release under - fail_on_unmatched_files: true - files: ./cf-cli-v${{ github.event.inputs.release_version }}.tgz diff --git a/.gitignore b/.gitignore index 0d9f52d..20235c2 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,8 @@ config/dev.yml config/private.yml dev_releases **/*.tgz +.vars* +.secrets* +node_modules +lease.json +metadata.json diff --git a/.gitpod.yml b/.gitpod.yml deleted file mode 100644 index 297aaa9..0000000 --- a/.gitpod.yml +++ /dev/null @@ -1,35 +0,0 @@ -tasks: - - name: Setup Workspace - before: > - echo before done - - sudo apt install --yes - icdiff - tldr - fzf - lastpass-cli - tmux - - brew install - asdf - - echo ". $HOME/.asdf/asdf.sh" >> $HOME/.bashrc - - echo ". $HOME/.asdf/completions/asdf.bash" >> $HOME/.bashrc - - echo "export GIT_EDITOR=vim" >> $HOME/.bashrc - - . $HOME/.bashrc - - asdf plugin add fly - - asdf install fly latest - init: > - echo init done - tldr --update - command: echo command done - -vscode: - extensions: - - Pivotal.vscode-concourse - - eamodio.gitlens diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000..8226afb --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1 @@ +external-sources=true diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..ab989bb --- /dev/null +++ b/Makefile @@ -0,0 +1,65 @@ +GH_ARGS=--repo cloudfoundry/bosh-package-cf-cli-release +GITHUB_TOKEN:=$(shell gh auth token || (gh auth login --scopes "write:packages, workflow" && gh auth token)) +GITHUB_USER:=$(gh api user | jq -r '.login') +PAGER=cat + +repo-context-setup: repo-context-set-vars repo-context-set-secrets + +repo-context-cleanup: repo-context-cleanup-vars repo-context-cleanup-secrets + +repo-context-cleanup-vars: + gh variable ${GH_ARGS} list --json name --jq '.[].name' \ + | xargs -n1 echo gh variable ${GH_ARGS} delete + +repo-context-cleanup-secrets: + gh secret ${GH_ARGS} list --json name --jq '.[].name' \ + | xargs -n1 echo gh secret ${GH_ARGS} delete + +repo-context-set-vars: + gh variable ${GH_ARGS} list + gh variable ${GH_ARGS} set -f .vars + gh variable ${GH_ARGS} list + +repo-context-set-secrets: + gh secret ${GH_ARGS} list + gh secret ${GH_ARGS} set -f .secrets + gh secret ${GH_ARGS} list + +create-bosh-release: + act \ + --actor "${GITHUB_USER}" \ + --secret GITHUB_TOKEN="${GITHUB_TOKEN}" \ + --workflows .github/workflows/create-bosh-release.yml + +ensure-ci-image: + act \ + --actor "${GITHUB_USER}" \ + --secret GITHUB_TOKEN="${GITHUB_TOKEN}" \ + --workflows .github/workflows/ensure-ci-image.yml + +lint: + act \ + --actor "${GITHUB_USER}" \ + --secret GITHUB_TOKEN="${GITHUB_TOKEN}" \ + --workflows .github/workflows/lint.yml + +run: + @echo "Running make with arguments after -- : $(MAKECMDGOALS)" + + # find . -name '.git' -prune -o -type f -print | entr -c \ + act \ + --actor "${GITHUB_USER}" \ + --secret GITHUB_TOKEN="${GITHUB_TOKEN}" \ + --workflows .github/workflows/create-bosh-release.yml \ + --secret-file .secrets \ + --var-file .vars \ + --job create_bosh_release \ + --rm \ + --artifact-server-path /tmp/artifacts \ + $(MAKECMDGOALS) + +hijack-act: + ./ci/scripts/hijack-act.sh + +bosh: + ./ci/scripts/bosh-connect.sh diff --git a/README.md b/README.md index 75b8a72..307aeb4 100644 --- a/README.md +++ b/README.md @@ -15,15 +15,28 @@ To co-locate the Linux CF CLI BOSH job on your target VM, follow these steps: Behind the scenes, the CF CLI binary is installed on the target machine at compile time via the `cf-cli-7-linux` or `cf-cli-8-linux` BOSH package (dependency of the `cf-cli-(7 | 8)-linux` BOSH job). The binary will be located at `/var/vcap/packages/cf-cli-(7 | 8)-linux/bin/cf`. ### Warning -Before consuming the release, ensure you've removed all previous `cf` CLI from either your blobs or packages. +Before consuming the release, ensure you've removed all previous `cf` CLI from either your blobs or packages. ## Development +1. Open with VisualStudion Code + - Check if [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension is installed. +1. By default [.vars](.github/.vars) and [.secrets](.github/.secrets) located in [.github](.github) folder. To use them please copy to the root of the project and update values. `cp .github/.vars .github/.secrets .` +1. Update [.secrets](.secrets) file with real API token. + - `echo "API_TOKEN: $(shepherd create service-account gha-shepherd --json | jq -r .secret)" >> .secrets` + - Local workflow dev runner [act](https://github.com/nektos/act) injects content of [.vars](.vars) and [.secrets](.secrets) into workflow execution context. +1. Open project inside the dev container. +1. Run `make run` to start. + +## Deployment + +1. To upload variables and secrets to the default remote repo for the current branch. **PROCEED WITH CARE** use `make repo-context-setup`. This will overwrite remote vaules with local from [.vars](.vars) and [.secrets](.secrets) + To test installation of the CF CLI binary via BOSH job co-location, run: -``` -$ ./tests/run.sh +```sh +./tests/run.sh ``` This will create a deployment using your currently targeted BOSH Director. diff --git a/ci/bin/reconfigure-pipeline b/ci/bin/reconfigure-pipeline deleted file mode 100755 index 806345b..0000000 --- a/ci/bin/reconfigure-pipeline +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -set -e -set -o pipefail - -check_installed() { - if ! command -v $1 > /dev/null 2>&1; then - printf "$1 must be installed before running this script!" - exit 1 - fi -} - -configure_pipeline() { - local name=$1 - local pipeline=$2 - - printf "configuring the $name pipeline...\n" - - fly -t ci set-pipeline \ - -p $name \ - -c $pipeline \ - -l <(lpass show "Shared-Dev-ex/toolsmiths-api-token" --notes) \ - -l <(lpass show "cf-cli-release Concourse Credentials" --notes) -} - - -check_installed lpass -check_installed fly - -# Make sure we're up to date and that we're logged in. -lpass sync - -pipelines_path=$(cd $(dirname $0)/.. && pwd) - -configure_pipeline cf-cli-release $pipelines_path/pipeline.yml -configure_pipeline cf-cli-release-toolsmiths $pipelines_path/pipeline-toolsmiths.yml \ No newline at end of file diff --git a/ci/cli-release-base/Dockerfile b/ci/cli-release-base/Dockerfile deleted file mode 100644 index 34c06af..0000000 --- a/ci/cli-release-base/Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -FROM ubuntu:trusty - -# Last updated on 2020-7-16, if updated again please add concourse automation -# to bump versions of these binaries: https://www.pivotaltracker.com/n/projects/2450704/stories/173838043 -ENV bosh_cli_version 7.2.3 -ENV bbl_version 8.4.111 -ENV terraform_version 0.11.5 - -RUN \ - apt-get update && \ - apt-get -y install \ - build-essential \ - git \ - libreadline6 \ - libreadline6-dev \ - libsqlite3-dev \ - libssl-dev \ - libxml2-dev \ - libxslt-dev \ - libyaml-dev \ - openssl \ - software-properties-common \ - sqlite \ - unzip \ - wget \ - curl \ - zlib1g-dev \ - zlibc && \ - add-apt-repository ppa:brightbox/ruby-ng -y && \ - apt-get update && \ - apt-get -y install \ - ruby2.3 \ - ruby2.3-dev && \ - apt-get remove -y --purge software-properties-common - -# bosh-cli -RUN \ - wget https://s3.amazonaws.com/bosh-cli-artifacts/bosh-cli-${bosh_cli_version}-linux-amd64 --output-document="/usr/bin/bosh" && \ - chmod +x /usr/bin/bosh - - # bbl and dependencies -RUN \ - wget https://github.com/cloudfoundry/bosh-bootloader/releases/download/v${bbl_version}/bbl-v${bbl_version}_linux_x86-64 -P /tmp && \ - mv /tmp/bbl-* /usr/local/bin/bbl && \ - cd /usr/local/bin && \ - chmod +x bbl - -RUN \ - wget https://github.com/cloudfoundry/bosh-bootloader/archive/v${bbl_version}.tar.gz -P /tmp && \ - mkdir -p /var/repos/bosh-bootloader && \ - tar xvf /tmp/v${bbl_version}.tar.gz --strip-components=1 -C /var/repos/bosh-bootloader && \ - rm -rf /tmp/* - -RUN \ - wget "https://releases.hashicorp.com/terraform/${terraform_version}/terraform_${terraform_version}_linux_amd64.zip" -P /tmp && \ - cd /tmp && \ - curl https://releases.hashicorp.com/terraform/${terraform_version}/terraform_${terraform_version}_SHA256SUMS | grep linux_amd64 | shasum -c - && \ - unzip "/tmp/terraform_${terraform_version}_linux_amd64.zip" -d /tmp && \ - mv /tmp/terraform /usr/local/bin/terraform && \ - cd /usr/local/bin && \ - chmod +x terraform && \ - rm -rf /tmp/* diff --git a/ci/pipeline-toolsmiths.yml b/ci/pipeline-toolsmiths.yml deleted file mode 100644 index 7935076..0000000 --- a/ci/pipeline-toolsmiths.yml +++ /dev/null @@ -1,173 +0,0 @@ ---- -resource_types: -- name: pcf-pool - type: docker-image - source: - repository: cftoolsmiths/toolsmiths-envs-resource - -- name: semver - type: docker-image - source: - repository: concourse/semver-resource - tag: latest - -- name: s3 - type: docker-image - source: - repository: concourse/s3-resource - tag: latest - -resources: -- name: cf-cli-release - type: git - source: - uri: git@github.com:bosh-packages/cf-cli-release - private_key: ((release-repo-github-key)) - branch: main - -- name: cf-cli-release-final - type: git - source: - uri: git@github.com:bosh-packages/cf-cli-release - private_key: ((release-repo-github-key)) - branch: main - -- name: gcp-env - type: pcf-pool - source: - api_token: ((token)) - hostname: environments.toolsmiths.cf-app.com - pool_name: cf-deployment - -- name: v6-cli-binary-linux-64 - type: s3 - source: - bucket: cf-cli-releases - regexp: releases/v(.*)/cf-cli_(.*)_linux_x86-64.tgz - region_name: us-west-1 - -- name: v7-cli-binary-linux-64 - type: s3 - source: - bucket: v7-cf-cli-releases - regexp: releases/v(.*)/cf7-cli_(.*)_linux_x86-64.tgz - region_name: us-west-1 - -- name: v8-cli-binary-linux-64 - type: s3 - source: - bucket: v8-cf-cli-releases - regexp: releases/v(.*)/cf8-cli_(.*)_linux_x86-64.tgz - region_name: us-west-1 - -- name: candidate-release - type: s3 - source: - bucket: cf-cli-bosh-release - versioned_file: cf-cli-dev-release.tgz - access_key_id: ((release-bucket-access-key-id)) - secret_access_key: ((release-bucket-secret-key)) - aws_role_arn: ((release-bucket-role-arn)) - region_name: us-west-1 - -- name: release-version - type: semver - source: - access_key_id: ((release-bucket-access-key-id)) - secret_access_key: ((release-bucket-secret-key)) - assume_role_arn: ((release-bucket-role-arn)) - initial_version: 1.1.0 - bucket: cf-cli-bosh-release - key: version - region_name: us-west-1 - -- name: cf-cli-release-final-tarball - type: s3 - source: - bucket: cf-cli-bosh-release - regexp: cf-cli-v(1\.\d+\.\d+)\.tgz - access_key_id: {{release-bucket-access-key-id}} - secret_access_key: {{release-bucket-secret-key}} - aws_role_arn: ((release-bucket-role-arn)) - region_name: us-west-1 - -jobs: - - name: update-cli - plan: - - in_parallel: - - get: cf-cli-release - - get: v6-cli-binary-linux-64 - trigger: true - - get: v7-cli-binary-linux-64 - trigger: true - - get: v8-cli-binary-linux-64 - trigger: true - - task: bump-cli - file: cf-cli-release/ci/tasks/bump-cli-release.yml - input_mapping: - v6-cli-binary: v6-cli-binary-linux-64 - v7-cli-binary: v7-cli-binary-linux-64 - v8-cli-binary: v8-cli-binary-linux-64 - cf-cli-release-input: cf-cli-release - params: - ACCESS_KEY_ID: ((release-bucket-access-key-id)) - SECRET_KEY: ((release-bucket-secret-key)) - AWS_ROLE_ARN: ((release-bucket-role-arn)) - - put: candidate-release - params: - file: "candidate-release-output/cf-cli-dev-release.tgz" - - put: cf-cli-release - params: - repository: cf-cli-release-output - rebase: true - - - name: run-tests - serial: true - plan: - - in_parallel: - - get: cf-cli-release - passed: [update-cli] - trigger: true - - put: gcp-env - params: - action: claim - - do: - - task: run-tests - file: cf-cli-release/ci/tasks/run-tests-toolsmiths.yml - ensure: - put: gcp-env - params: - action: unclaim - env_file: gcp-env/metadata - inputs: - - gcp-env - - - name: finalize-release - serial: true - plan: - - in_parallel: - - get: cf-cli-release - passed: [run-tests] - trigger: true - - get: release-version - params: - bump: minor - - task: finalize-release - file: cf-cli-release/ci/tasks/finalize-release.yml - input_mapping: - cf-cli-release-input: cf-cli-release - params: - ACCESS_KEY_ID: ((release-bucket-access-key-id)) - SECRET_KEY: ((release-bucket-secret-key)) - AWS_ROLE_ARN: ((release-bucket-role-arn)) - - put: release-version - params: - file: release-version/version - - put: cf-cli-release-final - params: - repository: cf-cli-release-output - rebase: true - tag: release-version/version - - put: cf-cli-release-final-tarball - params: - file: cf-cli-release-output/cf-cli-v1.*.tgz diff --git a/ci/pipeline.yml b/ci/pipeline.yml deleted file mode 100644 index faa0557..0000000 --- a/ci/pipeline.yml +++ /dev/null @@ -1,157 +0,0 @@ ---- -resources: -- name: bbl-store - type: git - source: - uri: git@github.com:cloudfoundry/cli-private - private_key: {{private-repo-github-key}} - branch: main - -- name: cf-cli-release - type: git - source: - uri: git@github.com:bosh-packages/cf-cli-release - private_key: {{release-repo-github-key}} - branch: main - -- name: cf-cli-release-final - type: git - source: - uri: git@github.com:bosh-packages/cf-cli-release - private_key: {{release-repo-github-key}} - branch: main - -- name: gcp-bosh-pool - type: pool - source: - uri: git@github.com:cloudfoundry/cli-pools - private_key: {{pools-repo-github-key}} - branch: master - pool: v8-pool - -- name: v6-cli-binary-linux-64 - type: s3 - source: - bucket: cf-cli-releases - regexp: releases/v(.*)/cf-cli_(.*)_linux_x86-64.tgz - region_name: us-west-1 - -- name: v7-cli-binary-linux-64 - type: s3 - source: - bucket: v7-cf-cli-releases - regexp: releases/v(.*)/cf7-cli_(.*)_linux_x86-64.tgz - region_name: us-west-1 - -- name: v8-cli-binary-linux-64 - type: s3 - source: - bucket: v8-cf-cli-releases - regexp: releases/v(.*)/cf8-cli_(.*)_linux_x86-64.tgz - region_name: us-west-1 - -- name: candidate-release - type: s3 - source: - bucket: cf-cli-bosh-release - versioned_file: cf-cli-dev-release.tgz - access_key_id: {{release-bucket-access-key-id}} - secret_access_key: {{release-bucket-secret-key}} - region_name: us-west-1 - -- name: cf-cli-release-final-tarball - type: s3 - source: - bucket: cf-cli-bosh-release - regexp: cf-cli-v(1\.\d+\.\d+)\.tgz - access_key_id: {{release-bucket-access-key-id}} - secret_access_key: {{release-bucket-secret-key}} - region_name: us-west-1 - -- name: release-version - type: semver - source: - access_key_id: {{release-bucket-access-key-id}} - secret_access_key: {{release-bucket-secret-key}} - initial_version: 1.1.0 - bucket: cf-cli-bosh-release - key: version - region_name: us-west-1 - -jobs: - - name: update-cli - plan: - - in_parallel: - - get: cf-cli-release - - get: v6-cli-binary-linux-64 - trigger: true - - get: v7-cli-binary-linux-64 - trigger: true - - get: v8-cli-binary-linux-64 - trigger: true - - task: bump-cli - file: cf-cli-release/ci/tasks/bump-cli-release.yml - input_mapping: - v6-cli-binary: v6-cli-binary-linux-64 - v7-cli-binary: v7-cli-binary-linux-64 - v8-cli-binary: v8-cli-binary-linux-64 - cf-cli-release-input: cf-cli-release - params: - ACCESS_KEY_ID: {{release-bucket-access-key-id}} - SECRET_KEY: {{release-bucket-secret-key}} - - put: candidate-release - params: - file: "candidate-release-output/cf-cli-dev-release.tgz" - - put: cf-cli-release - params: - repository: cf-cli-release-output - rebase: true - - - name: run-tests - serial: true - plan: - - in_parallel: - - get: cf-cli-release - passed: [update-cli] - trigger: true - - get: bbl-store - - put: bosh-lite-lock - resource: gcp-bosh-pool - params: - acquire: true - - do: - - task: run-tests - file: cf-cli-release/ci/tasks/run-tests.yml - ensure: - put: gcp-bosh-pool - params: - release: bosh-lite-lock - - - name: finalize-release - serial: true - plan: - - in_parallel: - - get: cf-cli-release - passed: [run-tests] - trigger: true - - get: release-version - params: - bump: minor - - task: finalize-release - file: cf-cli-release/ci/tasks/finalize-release.yml - input_mapping: - cf-cli-release-input: cf-cli-release - params: - ACCESS_KEY_ID: {{release-bucket-access-key-id}} - SECRET_KEY: {{release-bucket-secret-key}} - - put: release-version - params: - file: release-version/version - - put: cf-cli-release-final - params: - repository: cf-cli-release-output - rebase: true - tag: release-version/version - - put: cf-cli-release-final-tarball - params: - file: cf-cli-release-output/cf-cli-v1.*.tgz diff --git a/ci/scripts/bosh-connect.sh b/ci/scripts/bosh-connect.sh new file mode 100755 index 0000000..54b2728 --- /dev/null +++ b/ci/scripts/bosh-connect.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -o errexit -o nounset -o pipefail + +artifacts_dir="/tmp/artifacts/1" +artifacts_file="${artifacts_dir}/lease-json/lease-json.zip" +env_file=lease.json + +pushd "$(mktemp -d)" + unzip "${artifacts_file}" + jq -r .output ${env_file} > bosh_metadata.json + env_name=$(jq -r .name bosh_metadata.json) + jq -r .bosh.jumpbox_private_key bosh_metadata.json > "/tmp/${env_name:?}.priv" + eval "$(bbl print-env --metadata-file bosh_metadata.json)" +popd + +bosh deployments +bash diff --git a/ci/scripts/create-bosh-release-candidate.sh b/ci/scripts/create-bosh-release-candidate.sh new file mode 100755 index 0000000..f59aff9 --- /dev/null +++ b/ci/scripts/create-bosh-release-candidate.sh @@ -0,0 +1,188 @@ +#!/bin/bash +set -o errexit -o nounset -o pipefail +[[ "${TRACE:-0}" == "1" ]] && set -o xtrace + +# TODO: How do we configure shellcheck to find the right file? +# See https://www.shellcheck.net/wiki/SC1091 +# shellcheck source=/dev/null +source "$(dirname "${BASH_SOURCE[0]}")"/stdlib.sh + +# Used to find downloaded tarballs and extract semver +tarball_regex_sed="^.*cf[[:digit:]]\?-cli_\([[:digit:]]\+\).\([[:digit:]]\+\).\([[:digit:]]\+\)_linux_x86-64.tgz$" + +diff_and_commit_with_message() { + _message=${1} + + git diff --patch config/blobs.yml + git add config/blobs.yml + git commit --message "${_message}" + git log --pretty=full --max-count=1 +} + +add_and_commit_blob() { + _downloaded_tarball=${1} + _downloaded_tarball_basename=$(basename "${_downloaded_tarball}") + + # shellcheck disable=SC2001 + _major_version=$(echo "${_downloaded_tarball_basename}" | sed "s/${tarball_regex_sed}/\1/") + # shellcheck disable=SC2001 + _full_version=$(echo "${_downloaded_tarball_basename}" | sed "s/${tarball_regex_sed}/\1.\2.\3/") + + echo "::group::Adding blob for v${_major_version} - ${_downloaded_tarball_basename}" + bosh add-blob "${_downloaded_tarball}" "${_downloaded_tarball_basename}" + diff_and_commit_with_message "Setting CF CLI v${_major_version} to ${_full_version}" + echo "::endgroup::" +} + +remove_and_commit_blob() { + _published_blob_name=${1} + + # shellcheck disable=SC2001 + _major_version=$(echo "${_published_blob_name}" | sed "s/${tarball_regex_sed}/\1/") + + echo "::group::Removing blob for v${_major_version}" + bosh remove-blob "${_published_blob_name}" + diff_and_commit_with_message "Removing CF CLI v${_major_version}" + echo "::endgroup::" +} + +update_and_commit_blob() { + _published_blob_name=${1} + _downloaded_tarball=${2} + + _downloaded_tarball_basename=$(basename "${_downloaded_tarball}") + # shellcheck disable=SC2001 + _major_version=$(echo "${_downloaded_tarball_basename}" | sed "s/${tarball_regex_sed}/\1/") + # shellcheck disable=SC2001 + _published_version=$(echo "${_published_blob_name}" | sed "s/${tarball_regex_sed}/\1.\2.\3/") + # shellcheck disable=SC2001 + _new_version=$(echo "${_downloaded_tarball_basename}" | sed "s/${tarball_regex_sed}/\1.\2.\3/") + + echo "::group::Adding blob for v${_major_version} - ${_downloaded_tarball_basename}" + bosh add-blob "${_downloaded_tarball}" "${_downloaded_tarball_basename}" + diff_and_commit_with_message "Updating CF CLI v${_major_version} from ${_published_version} to ${_new_version}" + echo "::endgroup::" +} + +create_bosh_release_candidate() { + ## Parse and validate arguments + while [ ${#} -gt 0 ] ; do + case "${1}" in + --downloaded-binaries-dir) + case "${2:-}" in + "") fail_with "Must provide value for --downloaded-binaries-dir";; + *) _downloaded_binaries_dir="${2}"; shift 2;; + esac;; + --git-email) + case "${2:-}" in + "") fail_with "Must provide value for --git-email";; + *) _git_email="${2}"; shift 2;; + esac;; + --git-username) + case "${2:-}" in + "") fail_with "Must provide value for --git-username";; + *) _git_username="${2}"; shift 2;; + esac;; + (-*) + fail_with "Unrecognized option ${1}";; + (*) + fail_with "Unexpected argument ${1}";; + esac + done + + + [[ -z "${_downloaded_binaries_dir:-}" ]] && fail_with "Must provide --downloaded-binaries-dir" + [[ -z "${_git_email:-}" ]] && fail_with "Must provide --git-email" + [[ -z "${_git_username:-}" ]] && fail_with "Must provide --git-username" + + # Start off assuming no updates + _blobs_updated=false + + # Configure git + git config --global --add safe.directory "$(pwd)" + git config --global user.name "${_git_username}" + git config --global user.email "${_git_email}" + + # Remember current blobs + echo "::group::Blobs in most recent Bosh release:" + bosh blobs + _published_blobs=$(bosh blobs --json | jq --compact-output '.Tables[0].Rows[] | {path, digest}') + echo "::endgroup::" + + ## STEP 1: Prune or replace mismatched blobs from current Bosh release + echo "Replacing blobs from Bosh release that do not match downloaded binaries." + + for _published_blob in ${_published_blobs}; do + _published_blob_name=$(echo "${_published_blob}" | jq --raw-output '.path') + _resolved_tarball=$(find "${_downloaded_binaries_dir}"/* -type f -name "${_published_blob_name}") + [[ $(echo "${_resolved_tarball}" | wc -l ) -gt 1 ]] && \ + ls -laR "${_downloaded_binaries_dir}" && \ + fail_with "Found multiple tarballs with name ${_published_blob_name} in ${_downloaded_binaries_dir}" + + # Failed to find named tarball - Prune + if [[ -z "${_resolved_tarball}" ]]; then + _blobs_updated=true + + echo "Published blob ${_published_blob_name} does not have corresponding downloaded binary. Removing from new release." + remove_and_commit_blob "${_published_blob_name}" + + # Found named tarball - Compare digests + else + _published_blob_digest=$(echo "${_published_blob}" | jq --raw-output '.digest') + _downloaded_tarball_digest="sha256:$(sha256sum "${_resolved_tarball}" | cut --delimiter ' ' --field 1)" + + # Digest mismatch - Update + if [[ "${_published_blob_digest}" != "${_downloaded_tarball_digest}" ]]; then + _blobs_updated=true + + echo "Downloaded binary ${_published_blob_name} does not match blob in published Bosh release (published release specifies digest ${_published_blob_digest}, downloaded binary has digest ${_downloaded_tarball_digest}). Removing from new release." + + bosh remove-blob "${_published_blob}" + else + echo "Downloaded binary ${_published_blob_name} has same digest as blob in published release: ${_published_blob_digest}. Disregarding." + fi + fi + done + + # Update so that subsequent operations use newly-pruned blobs + _updated_published_blobs=$(bosh blobs --json | jq --compact-output '.Tables[0].Rows[] | {path, digest}') + + + ## STEP 2: Add new blobs to bosh release + echo "Adding downloaded binaries to the Bosh release." + + # Find tarballs + _downloaded_tarballs=$(find "${_downloaded_binaries_dir}"/* \ + -type f \ + -regextype sed \ + -regex "${tarball_regex_sed}") + + for _downloaded_tarball in ${_downloaded_tarballs}; do + _downloaded_tarball_basename=$(basename "${_downloaded_tarball}") + _published_blob=$(echo "${_updated_published_blobs}" | jq ". | select(.path == \"${_downloaded_tarball_basename}\")") + + if [[ -z "${_published_blob}" ]]; then + # Does downloaded tarball have a corresponding blob in the published Bosh release? + _blobs_updated=true + + echo "Downloaded binary ${_downloaded_tarball_basename} has no corresponding blob in published Bosh release. Adding to new release." + add_and_commit_blob "${_downloaded_tarball}" + fi + done + + bosh create-release \ + --timestamp-version \ + --force \ + --tarball=./cf-cli-dev-release.tgz + + echo "::group::Blobs in pending Bosh release" + bosh blobs + echo "::endgroup::" + + echo "blobs_updated=${_blobs_updated}" >> "${GITHUB_OUTPUT}" +} + +# Was the script sourced or executed? +if [[ "$(realpath "${0}")" == "$(realpath "${BASH_SOURCE[0]}")" ]]; then + create_bosh_release_candidate "$@" +fi diff --git a/ci/scripts/download-cf-cli-binary.sh b/ci/scripts/download-cf-cli-binary.sh new file mode 100755 index 0000000..ad882de --- /dev/null +++ b/ci/scripts/download-cf-cli-binary.sh @@ -0,0 +1,53 @@ +#!/bin/bash +set -o errexit -o nounset -o pipefail +[[ "${TRACE:-0}" == "1" ]] && set -o xtrace + +# TODO: How do we configure shellcheck to find the right file? +# See https://www.shellcheck.net/wiki/SC1091 +# shellcheck source=/dev/null +source "$(dirname "${BASH_SOURCE[0]}")"/stdlib.sh + +download_cf_cli_binary() { + # Parse and validate arguments + while [ ${#} -gt 0 ] ; do + case "${1}" in + --major-version) + case "${2:-}" in + "") fail_with "Must provide value for --major-version";; + *) _major_version="${2}"; shift 2;; + esac;; + --output-dir) + case "${2:-}" in + "") fail_with "Must provide value for --output-dir, or leave blank to default to cwd";; + *) _output_dir="${2}"; shift 2;; + esac;; + (-*) + fail_with "Unrecognized option ${1}";; + (*) + fail_with "Unexpected argument ${1}";; + esac + done + + [[ -z "${_major_version:-}" ]] && fail_with "Must provide --major-version" + [[ "${_major_version}" =~ [^0-9]+ ]] && fail_with "--major-version must be specified as number, e.g. \"8\" instead of \"v8\"." + + # Create named subdir in base output directory for version. Base output directory defaults to cwd if not specified. + _resolved_output_dir="$(realpath -m "${_output_dir:-${PWD}}")/${_major_version}" + + + # Download specified binary + echo "Downloading CF CLI ${_major_version} to directory ${_resolved_output_dir}" + + wget --trust-server-names \ + --directory-prefix "${_resolved_output_dir}" \ + --no-verbose \ + "https://packages.cloudfoundry.org/stable?release=linux64-binary&version=v${_major_version}&source=bosh-package-cf-cli-release-workflow" + + echo "Download complete." +} + + +# Was the script sourced or executed? +if [[ "$(realpath "${0}")" == "$(realpath "${BASH_SOURCE[0]}")" ]]; then + download_cf_cli_binary "$@" +fi diff --git a/ci/scripts/hijack-act.sh b/ci/scripts/hijack-act.sh new file mode 100755 index 0000000..16a0eb8 --- /dev/null +++ b/ci/scripts/hijack-act.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +image_id=$(docker ps --format=json | jq --slurp '.[] | select(.Image == "cfcli/cli-release-base") | .ID' --raw-output) +docker exec --interactive --tty "${image_id}" /bin/bash diff --git a/ci/scripts/stdlib.sh b/ci/scripts/stdlib.sh new file mode 100644 index 0000000..1552766 --- /dev/null +++ b/ci/scripts/stdlib.sh @@ -0,0 +1,5 @@ +# shellcheck disable=SC2148 +fail_with() { + echo -e "ERROR: ${1}" + exit 1 +} diff --git a/ci/tasks/bump-cli-release.yml b/ci/tasks/bump-cli-release.yml deleted file mode 100644 index 0c43334..0000000 --- a/ci/tasks/bump-cli-release.yml +++ /dev/null @@ -1,127 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: - repository: cfcli/cli-release-base - -params: - ACCESS_KEY_ID: - SECRET_KEY: - -inputs: -- name: cf-cli-release-input -- name: v6-cli-binary -- name: v7-cli-binary -- name: v8-cli-binary - -outputs: - - name: candidate-release-output - - name: cf-cli-release-output - -run: - path: bash - args: - - -c - - | - set -ex - - LATEST_V6_CLI_VERSION=$(cat v6-cli-binary/version) - LATEST_V7_CLI_VERSION=$(cat v7-cli-binary/version) - LATEST_V8_CLI_VERSION=$(cat v8-cli-binary/version) - - cp -r cf-cli-release-input/. cf-cli-release-output - - cd cf-cli-release-output - - echo "Checking if V6 CLI has been upgraded..." - ( - OLD_V6_BLOB_PATH=$(bosh blobs --column=path | grep "cf-") - OLD_V6_CLI_VERSION=$(echo ${OLD_BLOB_PATH} | cut -d_ -f2) - - if [[ "${OLD_V6_CLI_VERSION}" != "${LATEST_V6_CLI_VERSION}" ]]; then - git config --global user.email cf-cli-eng@pivotal.io - git config --global user.name "CI Bot" - - cat << EOF > config/private.yml - blobstore: - options: - access_key_id: "$ACCESS_KEY_ID" - secret_access_key: "$SECRET_KEY" - assume_role_arn: "$AWS_ROLE_ARN" - EOF - - bosh remove-blob $OLD_V6_BLOB_PATH - - bosh add-blob ../v6-cli-binary/cf-cli_${LATEST_V6_CLI_VERSION}_linux_x86-64.tgz cf-cli_${LATEST_V6_CLI_VERSION}_linux_x86-64.tgz - bosh upload-blobs - - git add config/blobs.yml - git status - git commit -m "bump v6 cli to ${LATEST_V6_CLI_VERSION}" - else - echo "Release has latest v6 CLI version, skipping bump." - fi - ) - - echo "Checking if V7 CLI has been upgraded..." - ( - OLD_V7_BLOB_PATH=$(bosh blobs --column=path | grep "cf7-") - OLD_V7_CLI_VERSION=$(echo ${OLD_BLOB_PATH} | cut -d_ -f2) - - if [[ "${OLD_V7_CLI_VERSION}" != "${LATEST_V7_CLI_VERSION}" ]]; then - git config --global user.email cf-cli-eng@pivotal.io - git config --global user.name "CI Bot" - - cat << EOF > config/private.yml - blobstore: - options: - access_key_id: "$ACCESS_KEY_ID" - secret_access_key: "$SECRET_KEY" - assume_role_arn: "$AWS_ROLE_ARN" - EOF - - bosh remove-blob $OLD_V7_BLOB_PATH - - bosh add-blob ../v7-cli-binary/cf7-cli_${LATEST_V7_CLI_VERSION}_linux_x86-64.tgz cf7-cli_${LATEST_V7_CLI_VERSION}_linux_x86-64.tgz - bosh upload-blobs - - git add config/blobs.yml - git status - git commit -m "bump v7 cli to ${LATEST_V7_CLI_VERSION}" - else - echo "Release has latest v7 CLI version, skipping bump." - fi - ) - - echo "Checking if V8 CLI has been upgraded..." - ( - OLD_V8_BLOB_PATH=$(bosh blobs --column=path | grep "cf8-") - OLD_V8_CLI_VERSION=$(echo ${OLD_BLOB_PATH} | cut -d_ -f2) - - if [[ "${OLD_V8_CLI_VERSION}" != "${LATEST_V8_CLI_VERSION}" ]]; then - git config --global user.email cf-cli-eng@pivotal.io - git config --global user.name "CI Bot" - - cat << EOF > config/private.yml - blobstore: - options: - access_key_id: "$ACCESS_KEY_ID" - secret_access_key: "$SECRET_KEY" - assume_role_arn: "$AWS_ROLE_ARN" - EOF - - bosh remove-blob $OLD_V8_BLOB_PATH - - bosh add-blob ../v8-cli-binary/cf8-cli_${LATEST_V8_CLI_VERSION}_linux_x86-64.tgz cf8-cli_${LATEST_V8_CLI_VERSION}_linux_x86-64.tgz - bosh upload-blobs - - git add config/blobs.yml - git status - git commit -m "bump v8 cli to ${LATEST_V8_CLI_VERSION}" - else - echo "Release has latest v8 CLI version, skipping bump." - fi - ) - bosh create-release --timestamp-version --tarball=../candidate-release-output/cf-cli-dev-release.tgz diff --git a/ci/tasks/finalize-release.yml b/ci/tasks/finalize-release.yml deleted file mode 100644 index a48070a..0000000 --- a/ci/tasks/finalize-release.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: - repository: cfcli/cli-release-base - -params: - ACCESS_KEY_ID: - SECRET_KEY: - -inputs: -- name: cf-cli-release-input -- name: release-version - -outputs: -- name: cf-cli-release-output - -run: - path: bash - args: - - -c - - | - set -ex - - RELEASE_VERSION=$(cat release-version/version) - - cp -r cf-cli-release-input/. cf-cli-release-output - - cd cf-cli-release-output - - git config --global user.email cf-cli-eng@pivotal.io - git config --global user.name "CI Bot" - - cat << EOF > config/private.yml - blobstore: - options: - access_key_id: "$ACCESS_KEY_ID" - secret_access_key: "$SECRET_KEY" - assume_role_arn: "$AWS_ROLE_ARN" - EOF - - bosh create-release --final --version="${RELEASE_VERSION}" --tarball="./cf-cli-v${RELEASE_VERSION}.tgz" - - git add --all - git status - git commit -m "create final release ${RELEASE_VERSION}" diff --git a/ci/tasks/run-tests-toolsmiths.yml b/ci/tasks/run-tests-toolsmiths.yml deleted file mode 100644 index b1fe7ef..0000000 --- a/ci/tasks/run-tests-toolsmiths.yml +++ /dev/null @@ -1,27 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: - repository: cfcli/cli-release-base - -inputs: -- name: cf-cli-release -- name: gcp-env - -run: - path: bash - args: - - -c - - | - set -e - - ENV=$(cat gcp-env/name) - - cat gcp-env/metadata > $ENV.json - eval "$(bbl print-env --metadata-file $ENV.json)" - - set -x - cd cf-cli-release - ./tests/run.sh diff --git a/ci/tasks/run-tests.yml b/ci/tasks/run-tests.yml deleted file mode 100644 index cc40a26..0000000 --- a/ci/tasks/run-tests.yml +++ /dev/null @@ -1,28 +0,0 @@ ---- -platform: linux - -image_resource: - type: docker-image - source: - repository: cfcli/cli-release-base - -inputs: -- name: cf-cli-release -- name: bbl-store -- name: bosh-lite-lock - -run: - path: bash - args: - - -c - - | - set -e - - ENV=$(cat bosh-lite-lock/name | cut -d "." -f 1) - pushd bbl-store/ci/infrastructure/$ENV - eval "$(bbl print-env)" - popd - - set -x - cd cf-cli-release - ./tests/run.sh diff --git a/config/blobs.yml b/config/blobs.yml index a388293..b8d7e25 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -10,3 +10,4 @@ cf8-cli_8.99.99_linux_x86-64.tgz: size: 8966421 object_id: fb7941f6-596b-4ca2-4c9f-4b987a088e72 sha: sha256:bfda5c48cd281985f3005f584c7034227ebe814c51fc35211ff3819f05ac2c58 + \ No newline at end of file diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..2b051a7 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,32 @@ +# Architecture + +## Workflow + +- Manually create a release by clicking a button to execute the workflow action +- Download CLI binaries from CLAW and compare against the current bosh release manifest +- Configure Bosh S3 blobs backend +- For each difference between bosh manifest and CLAW: + - Add the blob + - Upload the blob + - Create git commit for the new blob + - Push to github +- Create dev bosh release, which is exported to a .tgz +- Create a bosh deployment in shepherd that includes the new dev release +- Run the tests in shepherd using the new dev release +- Calculate new version number for this pending release + - Point of question — how to calculate this semver id? Maybe changesets? How could we configure changesets to find the correct "current" version of the bosh release? +- Create a final bosh release with that version + - This does an S3 push + - It also makes a git commit +- Create git tag with that version +- Push to github +- Create github release with this tag + +## Improvements + +- Make this more transactional. + - Can we do the github and S3 pushes very closely together, at the very end? + - Can we make them succeed or fail together? + - Maybe we push to a dev bucket before tests, then — only on green tests — push everything to a prod bucket? +- Autogenerate release notes from release notes on CLI. + - Can changesets help here too? diff --git a/jobs/cf-cli-6-linux-test/monit b/jobs/cf-cli-6-linux-test/monit deleted file mode 100644 index e69de29..0000000 diff --git a/jobs/cf-cli-6-linux-test/spec b/jobs/cf-cli-6-linux-test/spec deleted file mode 100644 index 46c139b..0000000 --- a/jobs/cf-cli-6-linux-test/spec +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: cf-cli-6-linux-test - -templates: - run: bin/run - -packages: [] - -properties: {} - diff --git a/jobs/cf-cli-6-linux-test/templates/run b/jobs/cf-cli-6-linux-test/templates/run deleted file mode 100755 index a8927de..0000000 --- a/jobs/cf-cli-6-linux-test/templates/run +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -e - -export PATH=/var/vcap/packages/cf-cli-6-linux/bin:$PATH - -cf -v diff --git a/jobs/cf-cli-6-linux/monit b/jobs/cf-cli-6-linux/monit deleted file mode 100644 index e69de29..0000000 diff --git a/jobs/cf-cli-6-linux/spec b/jobs/cf-cli-6-linux/spec deleted file mode 100644 index d5df2a1..0000000 --- a/jobs/cf-cli-6-linux/spec +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: cf-cli-6-linux - -templates: {} - -packages: -- cf-cli-6-linux - -properties: {} - diff --git a/manifests/test.yml b/manifests/test.yml index 7c058ea..1ff9ffb 100644 --- a/manifests/test.yml +++ b/manifests/test.yml @@ -18,22 +18,6 @@ update: update_watch_time: 5000-60000 instance_groups: -- name: cf-cli-6-linux - lifecycle: errand - azs: [z1] - instances: 1 - jobs: - - name: cf-cli-6-linux - release: cf-cli - properties: {} - - name: cf-cli-6-linux-test - release: cf-cli - properties: {} - vm_type: default - stemcell: default - networks: - - name: default - - name: cf-cli-7-linux lifecycle: errand azs: [z1] @@ -65,5 +49,3 @@ instance_groups: stemcell: default networks: - name: default - - diff --git a/packages/cf-cli-6-linux/packaging b/packages/cf-cli-6-linux/packaging deleted file mode 100644 index 6640fe0..0000000 --- a/packages/cf-cli-6-linux/packaging +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -set -exu - -tar xzf cf-cli_6.*_linux_x86-64.tgz - -mkdir ${BOSH_INSTALL_TARGET}/bin -cp ./cf ${BOSH_INSTALL_TARGET}/bin/cf diff --git a/packages/cf-cli-6-linux/spec b/packages/cf-cli-6-linux/spec deleted file mode 100644 index 9a2e100..0000000 --- a/packages/cf-cli-6-linux/spec +++ /dev/null @@ -1,5 +0,0 @@ ---- -name: cf-cli-6-linux - -files: -- cf-cli_6.*_linux_x86-64.tgz diff --git a/tests/run.sh b/tests/run.sh index 1927206..3c10fb5 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -5,25 +5,22 @@ set -e export BOSH_DEPLOYMENT=cf-cli-test export BOSH_NON_INTERACTIVE=true -echo "-----> `date`: Delete previous deployment" +echo "-----> $(date): Delete previous deployment" bosh delete-deployment --force -echo "-----> `date`: Deploy" +echo "-----> $(date): Deploy" bosh deploy ./manifests/test.yml -echo "-----> `date`: Run test errand for cf6" -bosh run-errand cf-cli-6-linux-test - -echo "-----> `date`: Run test errand for cf7" +echo "-----> $(date): Run test errand for cf7" bosh run-errand cf-cli-7-linux-test -echo "-----> `date`: Run test errand for cf8" +echo "-----> $(date): Run test errand for cf8" bosh run-errand cf-cli-8-linux-test -echo "-----> `date`: Delete deployments" +echo "-----> $(date): Delete deployments" bosh delete-deployment -echo "-----> `date`: Clean up" +echo "-----> $(date): Clean up" bosh clean-up --all -echo "-----> `date`: Done" +echo "-----> $(date): Done"