01 - Powerpipe: Prerelease #72
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "01 - Powerpipe: Prerelease" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| pre_release_identifier: | |
| type: choice | |
| description: "Select Pre-Release Identifier" | |
| options: | |
| - alpha | |
| - beta | |
| - rc | |
| required: true | |
| env: | |
| PROJECT_ID: steampipe | |
| ORG: turbot | |
| CORE_REPO: us-docker.pkg.dev/steampipe/steampipe | |
| ASSET_IMAGE_NAME: assets | |
| POWERPIPE_UPDATE_CHECK: false | |
| GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} | |
| SPIPETOOLS_TOKEN: ${{ secrets.SPIPETOOLS_TOKEN }} | |
| jobs: | |
| goreleaser: | |
| name: Build | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: powerpipe | |
| ref: ${{ github.event.ref }} | |
| - name: Checkout Pipe Fittings Components repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: turbot/pipe-fittings | |
| path: pipe-fittings | |
| ref: integrated-2024 | |
| # this is required, check golangci-lint-action docs | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.23' | |
| cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704 | |
| - name: Build | |
| run: |- | |
| cd powerpipe | |
| make release-dry-run | |
| - name: Move build artifacts | |
| run: | | |
| ls -al $GITHUB_WORKSPACE/powerpipe | |
| sudo chown -R runner:docker $GITHUB_WORKSPACE/powerpipe/dist | |
| mkdir ~/artifacts | |
| mv $GITHUB_WORKSPACE/powerpipe/dist/powerpipe.linux.amd64.tar.gz ~/artifacts/linux.tar.gz | |
| mv $GITHUB_WORKSPACE/powerpipe/dist/powerpipe.linux.arm64.tar.gz ~/artifacts/linux-arm.tar.gz | |
| mv $GITHUB_WORKSPACE/powerpipe/dist/powerpipe.darwin.amd64.tar.gz ~/artifacts/darwin.tar.gz | |
| mv $GITHUB_WORKSPACE/powerpipe/dist/powerpipe.darwin.arm64.tar.gz ~/artifacts/darwin-arm.tar.gz | |
| - name: List Build Artifacts | |
| run: ls -l ~/artifacts | |
| - name: Save Linux Build Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-artifact-linux | |
| path: ~/artifacts/linux.tar.gz | |
| if-no-files-found: error | |
| - name: Save Linux ARM Build Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-artifact-linux-arm | |
| path: ~/artifacts/linux-arm.tar.gz | |
| if-no-files-found: error | |
| - name: Save MacOS Build Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-artifact-darwin | |
| path: ~/artifacts/darwin.tar.gz | |
| if-no-files-found: error | |
| - name: Save MacOS ARM Build Artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: build-artifact-darwin-arm | |
| path: ~/artifacts/darwin-arm.tar.gz | |
| if-no-files-found: error | |
| acceptance_test: | |
| name: Test | |
| needs: goreleaser | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| platform: [ubuntu-latest, macos-latest] | |
| test_block: | |
| - "check" | |
| - "resource_show_outputs" | |
| - "dashboard" | |
| - "backend" | |
| - "mod" | |
| - "mod_install" | |
| - "sp_files" | |
| - "var_resolution" | |
| - "params_and_args" | |
| - "snapshot" | |
| - "dashboard_parsing_validation" | |
| - "database_precedence" | |
| runs-on: ${{ matrix.platform }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| path: powerpipe | |
| ref: ${{ github.event.ref }} | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: 1.22 | |
| cache: false | |
| - name: Prepare for downloads | |
| id: prepare-for-downloads | |
| run: | | |
| mkdir ~/artifacts | |
| - name: Download Linux Build Artifacts | |
| uses: actions/download-artifact@v3 | |
| if: ${{ matrix.platform == 'ubuntu-latest' }} | |
| with: | |
| name: build-artifact-linux | |
| path: ~/artifacts | |
| - name: Download Darwin Build Artifacts | |
| uses: actions/download-artifact@v3 | |
| if: ${{ matrix.platform == 'macos-latest' }} | |
| with: | |
| name: build-artifact-darwin-arm | |
| path: ~/artifacts | |
| - name: Extract Darwin Artifacts and Install Binary | |
| if: ${{ matrix.platform == 'macos-latest' }} | |
| run: | | |
| mkdir ~/build | |
| tar -xf ~/artifacts/darwin-arm.tar.gz -C ~/build | |
| - name: Extract Ubuntu Artifacts and Install Binary | |
| if: ${{ matrix.platform == 'ubuntu-latest' }} | |
| run: | | |
| mkdir ~/build | |
| tar -xf ~/artifacts/linux.tar.gz -C ~/build | |
| - name: Install Steampipe(Darwin) | |
| if: ${{ matrix.platform == 'macos-latest' }} | |
| run: | | |
| brew install turbot/tap/steampipe | |
| steampipe -v | |
| - name: Install Steampipe(Linux) | |
| if: ${{ matrix.platform == 'ubuntu-latest' }} | |
| run: | | |
| sudo /bin/sh -c "$(curl -fsSL https://steampipe.io/install/steampipe.sh)" | |
| steampipe -v | |
| - name: Set PATH | |
| run: | | |
| echo "PATH=$PATH:$HOME/build:$GTIHUB_WORKSPACE/powerpipe/tests/acceptance/lib/bats-core/libexec" >> $GITHUB_ENV | |
| - name: Go install jd | |
| run: | | |
| go install github.com/josephburnett/jd@latest | |
| - name: Start steamipipe service | |
| run: | | |
| steampipe service start | |
| - name: Run Test Suite | |
| id: run-test-suite | |
| timeout-minutes: 15 | |
| continue-on-error: true | |
| run: | | |
| chmod +x $GITHUB_WORKSPACE/powerpipe/tests/acceptance/run.sh | |
| $GITHUB_WORKSPACE/powerpipe/tests/acceptance/run.sh ${{ matrix.test_block }}.bats | |
| echo "exit_code=$(echo $?)" >> $GITHUB_OUTPUT | |
| echo ">> here" | |
| # This job checks whether the test suite has passed or not. | |
| # Since the exit_code is set only when the bats test suite pass, | |
| # we have added the if-conditional block | |
| - name: Check Test Passed/Failed | |
| if: ${{ success() }} | |
| continue-on-error: false | |
| run: | | |
| if [ ${{ steps.run-test-suite.outputs.exit_code }} -eq 0 ]; then | |
| exit 0 | |
| else | |
| exit 1 | |
| fi | |
| - name: Stop steampipe service | |
| run: | | |
| steampipe service stop | |
| smoke_test_ubuntu_24: | |
| name: Smoke test (Ubuntu 24, x86_64) | |
| runs-on: ubuntu-latest | |
| needs: [goreleaser] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download Linux Build Artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: build-artifact-linux | |
| path: ./artifacts | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Pull Ubuntu latest Image | |
| run: docker pull ubuntu:latest | |
| - name: Create and Start Ubuntu latest Container | |
| run: | | |
| docker run -d --name ubuntu-24-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts ubuntu:latest tail -f /dev/null | |
| - name: Get runner/container info | |
| run: | | |
| docker exec ubuntu-24-test /scripts/linux_container_info.sh | |
| - name: Install dependencies, create user, and assign necessary permissions | |
| run: | | |
| docker exec ubuntu-24-test /scripts/prepare_ubuntu_container.sh | |
| - name: Run smoke tests | |
| run: | | |
| docker exec -u steampipe ubuntu-24-test /scripts/smoke_test.sh | |
| - name: Stop and Remove Container | |
| run: | | |
| docker stop ubuntu-24-test | |
| docker rm ubuntu-24-test | |
| smoke_test_centos_9: | |
| name: Smoke test (Centos stream 9, x86_64) | |
| runs-on: ubuntu-latest | |
| needs: [goreleaser] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download Linux Build Artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: build-artifact-linux | |
| path: ./artifacts | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Pull CentOS Stream 9 image | |
| run: docker pull quay.io/centos/centos:stream9 | |
| - name: Create and Start CentOS stream9 Container | |
| run: | | |
| docker run -d --name centos-stream9-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts quay.io/centos/centos:stream9 tail -f /dev/null | |
| - name: Get runner/container info | |
| run: | | |
| docker exec centos-stream9-test /scripts/linux_container_info.sh | |
| - name: Install dependencies, create user, and assign necessary permissions | |
| run: | | |
| docker exec centos-stream9-test /scripts/prepare_centos_container.sh | |
| - name: Run smoke tests | |
| run: | | |
| docker exec -u steampipe centos-stream9-test /scripts/smoke_test.sh | |
| - name: Stop and Remove Container | |
| run: | | |
| docker stop centos-stream9-test | |
| docker rm centos-stream9-test | |
| smoke_test_amazonlinux: | |
| name: Smoke test (Amazonlinux 2023, x86_64) | |
| runs-on: ubuntu-latest | |
| needs: [goreleaser] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download Linux Build Artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: build-artifact-linux | |
| path: ./artifacts | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Pull Amazon Linux 2023 Image | |
| run: docker pull amazonlinux:2023 | |
| - name: Create and Start Amazon Linux 2023 Container | |
| run: | | |
| docker run -d --name amazonlinux-2023-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts amazonlinux:2023 tail -f /dev/null | |
| - name: Get runner/container info | |
| run: | | |
| docker exec amazonlinux-2023-test /scripts/linux_container_info.sh | |
| - name: Install dependencies, create user, and assign necessary permissions | |
| run: | | |
| docker exec amazonlinux-2023-test /scripts/prepare_amazonlinux_container.sh | |
| - name: Run smoke tests | |
| run: | | |
| docker exec -u steampipe amazonlinux-2023-test /scripts/smoke_test.sh | |
| - name: Stop and Remove Container | |
| run: | | |
| docker stop amazonlinux-2023-test | |
| docker rm amazonlinux-2023-test | |
| # smoke_test_ubuntu_24_arm64: | |
| # name: Smoke test (Ubuntu 24, ARM64) | |
| # runs-on: ubuntu-latest | |
| # needs: [goreleaser] | |
| # steps: | |
| # - name: Checkout | |
| # uses: actions/checkout@v4 | |
| # - name: Download Linux Build Artifact | |
| # uses: actions/download-artifact@v3 | |
| # with: | |
| # name: build-artifact-linux-arm | |
| # path: ./artifacts | |
| # - name: Set up Docker Buildx | |
| # uses: docker/setup-buildx-action@v3 | |
| # - name: Set up QEMU | |
| # uses: docker/setup-qemu-action@v3 | |
| # with: | |
| # platforms: arm64 | |
| # - name: Create Buildx Builder | |
| # run: | | |
| # docker buildx create --use --name mybuilder | |
| # docker buildx inspect --bootstrap | |
| # - name: Pull Ubuntu 24 ARM64 Image | |
| # run: docker pull arm64v8/ubuntu:latest | |
| # - name: Create and Start Ubuntu 24 ARM64 Container | |
| # run: | | |
| # docker run -d --name ubuntu-24-arm64-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts arm64v8/ubuntu:latest tail -f /dev/null | |
| # - name: Get runner/container info | |
| # run: | | |
| # docker exec ubuntu-24-arm64-test /scripts/linux_container_info.sh | |
| # - name: Install dependencies, create user, and assign necessary permissions | |
| # run: | | |
| # docker exec ubuntu-24-arm64-test /scripts/prepare_ubuntu_container.sh | |
| # - name: Run smoke tests | |
| # run: | | |
| # docker exec -u steampipe ubuntu-24-arm64-test /scripts/smoke_test.sh | |
| # - name: Stop and Remove Container | |
| # run: | | |
| # docker stop ubuntu-24-arm64-test | |
| # docker rm ubuntu-24-arm64-test | |
| smoke_test_darwin_amd: | |
| name: Smoke test (MacOS 13, x86_64) | |
| runs-on: macos-13 | |
| needs: [goreleaser] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download Darwin Build Artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: build-artifact-darwin | |
| path: ~/artifacts | |
| - name: Extract Darwin Artifacts and Install Binary | |
| run: | | |
| tar -xf ~/artifacts/darwin.tar.gz -C /usr/local/bin/ | |
| - name: Install Steampipe(Darwin) | |
| run: | | |
| sudo /bin/sh -c "$(curl -fsSL https://steampipe.io/install/steampipe.sh)" | |
| steampipe -v | |
| - name: Install jq | |
| run: | | |
| brew install jq | |
| - name: Get runner/container info | |
| run: | | |
| uname -a | |
| sw_vers | |
| - name: Run Smoke tests | |
| run: | | |
| ls -al $GITHUB_WORKSPACE/scripts | |
| chmod +x $GITHUB_WORKSPACE/scripts/smoke_test.sh | |
| $GITHUB_WORKSPACE/scripts/smoke_test.sh | |
| build_assets: | |
| name: Build and Release Dashboard Assets | |
| needs: [acceptance_test, smoke_test_ubuntu_24, smoke_test_centos_9, smoke_test_amazonlinux, smoke_test_darwin_amd] | |
| runs-on: ubuntu-latest | |
| outputs: | |
| version: ${{ steps.set_version.outputs.version }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: powerpipe | |
| ref: ${{ github.event.ref }} | |
| - name: Calculate version | |
| id: calculate_version | |
| run: | | |
| VERSION_JSON=$(cat powerpipe/internal/version/version.json) | |
| MAJOR=$(echo $VERSION_JSON | jq '.major') | |
| MINOR=$(echo $VERSION_JSON | jq '.minor') | |
| PATCH=$(echo $VERSION_JSON | jq '.patch') | |
| VERSION="v$MAJOR.$MINOR.$PATCH" | |
| if [ "${{ github.event.inputs.pre_release_identifier }}" = "alpha" ]; then | |
| echo "VERSION=${VERSION}-alpha.$(date +'%Y%m%d%H%M')" >> $GITHUB_ENV | |
| elif [ "${{ github.event.inputs.pre_release_identifier }}" = "beta" ]; then | |
| echo "VERSION=${VERSION}-beta.$(date +'%Y%m%d%H%M')" >> $GITHUB_ENV | |
| elif [ "${{ github.event.inputs.pre_release_identifier }}" = "rc" ]; then | |
| echo "VERSION=${VERSION}-rc.$(date +'%Y%m%d%H%M')" >> $GITHUB_ENV | |
| fi | |
| - name: Set VERSION as output | |
| run: echo "::set-output name=version::$VERSION" | |
| id: set_version | |
| - name: Trim asset version prefix and Validate | |
| run: |- | |
| echo $VERSION | |
| trim=${VERSION#"v"} | |
| echo $trim | |
| if [[ $trim =~ ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then | |
| echo "Version OK: $trim" | |
| else | |
| echo "Invalid version: $trim" | |
| exit 1 | |
| fi | |
| echo "VERSION=${trim}" >> $GITHUB_ENV | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: ">=18.12.0" | |
| - name: Setup Yarn | |
| working-directory: ./powerpipe/ui/dashboard | |
| run: |- | |
| rm -rf ./build | |
| # Cache node modules to speed up builds | |
| - name: Cache node modules | |
| id: cache-npm | |
| uses: actions/cache@v4 | |
| env: | |
| cache-name: cache-node-modules | |
| with: | |
| # npm cache files are stored in `~/.npm` on Linux/macOS | |
| path: ~/.npm | |
| key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} | |
| - name: YARN Install | |
| working-directory: ./powerpipe/ui/dashboard | |
| run: |- | |
| unset CI | |
| corepack enable | |
| yarn install | |
| - name: Generate dashboard UI icons | |
| working-directory: ./powerpipe/ui/dashboard | |
| run: |- | |
| make setup-icons | |
| - name: Run Unit Tests | |
| working-directory: ./powerpipe/ui/dashboard | |
| run: | | |
| CI=true yarn test | |
| - name: YARN Build | |
| working-directory: ./powerpipe/ui/dashboard | |
| run: |- | |
| unset CI | |
| yarn build | |
| env: | |
| REACT_APP_HEAP_ID: ${{ secrets.HEAP_ANALYTICS_PRODUCTION_ID }} | |
| REACT_APP_VERSION: ${{ env.VERSION }} | |
| - name: Move Build Assets | |
| run: |- | |
| mkdir -p dashboard_ui_build | |
| mv ./powerpipe/ui/dashboard/build/* dashboard_ui_build/ | |
| ls -la dashboard_ui_build | |
| - name: Create a version file | |
| run: |- | |
| JSON_STRING=$( jq -n \ | |
| --arg version "$VERSION" \ | |
| '{ | |
| "version":$version, | |
| }' ) | |
| echo $JSON_STRING > ./dashboard_ui_build/version.json | |
| - name: Archive built dashboard assets | |
| run: | | |
| cd dashboard_ui_build | |
| tar -czvf ../dashboard_ui_assets.tar.gz * | |
| - name: Upload dashboard assets artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: dashboard-ui-assets-archive | |
| path: dashboard_ui_assets.tar.gz | |
| build_and_release: | |
| name: Build and Release Powerpipe | |
| needs: [build_assets] | |
| runs-on: ubuntu-latest | |
| env: | |
| VERSION: ${{ needs.build_assets.outputs.version }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| path: powerpipe | |
| ref: ${{ github.event.ref }} | |
| - name: Checkout Pipe Fittings Components repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: turbot/pipe-fittings | |
| path: pipe-fittings | |
| ref: integrated-2024 | |
| - name: Tag Release | |
| run: | | |
| cd powerpipe | |
| git config user.name "Powerpipe GitHub Actions Bot" | |
| git config user.email [email protected] | |
| git tag $VERSION | |
| git push origin $VERSION | |
| # this is required, check golangci-lint-action docs | |
| - uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.22" | |
| cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704 | |
| - name: Download Dashboard UI Assets Artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: dashboard-ui-assets-archive | |
| path: powerpipe/internal/dashboardassets | |
| - name: Check assets location | |
| run: | | |
| ls -la powerpipe/internal/dashboardassets | |
| mv powerpipe/internal/dashboardassets/dashboard_ui_assets.tar.gz powerpipe/internal/dashboardassets/assets.tar.gz | |
| ls -la powerpipe/internal/dashboardassets | |
| - name: Setup release environment | |
| run: |- | |
| cd powerpipe | |
| echo 'GITHUB_TOKEN=${{secrets.GH_ACCESS_TOKEN}}' > .release-env | |
| - name: Release publish | |
| run: |- | |
| cd powerpipe | |
| make release |