01-powerpipe-pre-release.yaml

name: "01 - Powerpipe: Prerelease"

on:
  workflow_dispatch:
    inputs:
      pre_release_identifier:
        type: choice
        description: "Select Pre-Release Identifier"
        options:
          - alpha
          - beta
          - rc
        required: true

env:
  PROJECT_ID: steampipe
  ORG: turbot
  CORE_REPO: us-docker.pkg.dev/steampipe/steampipe
  ASSET_IMAGE_NAME: assets
  POWERPIPE_UPDATE_CHECK: false
  GH_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }}
  SPIPETOOLS_TOKEN: ${{ secrets.SPIPETOOLS_TOKEN }}

jobs:
  goreleaser:
    name: Build
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: powerpipe
          ref: ${{ github.event.ref }}

      - name: Checkout Pipe Fittings Components repository
        uses: actions/checkout@v4
        with:
          repository: turbot/pipe-fittings
          path: pipe-fittings
          ref: v1.6.x

       # this is required, check golangci-lint-action docs
      - uses: actions/setup-go@v5
        with:
          go-version: '1.23'
          cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704

      - name: Build
        run: |-
          cd powerpipe
          make release-dry-run

      - name: Move build artifacts
        run: |
          ls -al $GITHUB_WORKSPACE/powerpipe
          sudo chown -R runner:docker $GITHUB_WORKSPACE/powerpipe/dist
          mkdir ~/artifacts
          mv $GITHUB_WORKSPACE/powerpipe/dist/powerpipe.linux.amd64.tar.gz ~/artifacts/linux.tar.gz
          mv $GITHUB_WORKSPACE/powerpipe/dist/powerpipe.linux.arm64.tar.gz ~/artifacts/linux-arm.tar.gz
          mv $GITHUB_WORKSPACE/powerpipe/dist/powerpipe.darwin.amd64.tar.gz ~/artifacts/darwin.tar.gz
          mv $GITHUB_WORKSPACE/powerpipe/dist/powerpipe.darwin.arm64.tar.gz ~/artifacts/darwin-arm.tar.gz

      - name: List Build Artifacts
        run: ls -l ~/artifacts

      - name: Save Linux Build Artifact
        uses: actions/upload-artifact@v3
        with:
          name: build-artifact-linux
          path: ~/artifacts/linux.tar.gz
          if-no-files-found: error

      - name: Save Linux ARM Build Artifact
        uses: actions/upload-artifact@v3
        with:
          name: build-artifact-linux-arm
          path: ~/artifacts/linux-arm.tar.gz
          if-no-files-found: error

      - name: Save MacOS Build Artifact
        uses: actions/upload-artifact@v3
        with:
          name: build-artifact-darwin
          path: ~/artifacts/darwin.tar.gz
          if-no-files-found: error

      - name: Save MacOS ARM Build Artifact
        uses: actions/upload-artifact@v3
        with:
          name: build-artifact-darwin-arm
          path: ~/artifacts/darwin-arm.tar.gz
          if-no-files-found: error

  acceptance_test:
    name: Test
    needs: goreleaser
    strategy:
      fail-fast: false
      matrix:
        platform: [ubuntu-latest, macos-latest]
        test_block:
          - "check"
          - "resource_show_outputs"
          - "dashboard"
          - "backend"
          - "mod"
          - "mod_install"
          - "sp_files"
          - "var_resolution"
          - "params_and_args"
          - "snapshot"
          - "dashboard_parsing_validation"
          - "database_precedence"
    runs-on: ${{ matrix.platform }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: true
          path: powerpipe
          ref: ${{ github.event.ref }}

      - uses: actions/setup-go@v5
        with:
          go-version: 1.22
          cache: false

      - name: Prepare for downloads
        id: prepare-for-downloads
        run: |
          mkdir ~/artifacts
  
      - name: Download Linux Build Artifacts
        uses: actions/download-artifact@v3
        if: ${{ matrix.platform == 'ubuntu-latest' }}
        with:
          name: build-artifact-linux
          path: ~/artifacts

      - name: Download Darwin Build Artifacts
        uses: actions/download-artifact@v3
        if: ${{ matrix.platform == 'macos-latest' }}
        with:
          name: build-artifact-darwin-arm
          path: ~/artifacts

      - name: Extract Darwin Artifacts and Install Binary
        if: ${{ matrix.platform == 'macos-latest' }}
        run: |
          mkdir ~/build
          tar -xf ~/artifacts/darwin-arm.tar.gz -C ~/build

      - name: Extract Ubuntu Artifacts and Install Binary
        if: ${{ matrix.platform == 'ubuntu-latest' }}
        run: |
          mkdir ~/build
          tar -xf ~/artifacts/linux.tar.gz -C ~/build
      
      - name: Install Steampipe(Darwin)
        if: ${{ matrix.platform == 'macos-latest' }}
        run: |
          brew install turbot/tap/steampipe
          steampipe -v

      - name: Install Steampipe(Linux)
        if: ${{ matrix.platform == 'ubuntu-latest' }}
        run: |
          sudo /bin/sh -c "$(curl -fsSL https://steampipe.io/install/steampipe.sh)"
          steampipe -v
      
      - name: Set PATH
        run: |
          echo "PATH=$PATH:$HOME/build:$GTIHUB_WORKSPACE/powerpipe/tests/acceptance/lib/bats-core/libexec" >> $GITHUB_ENV

      - name: Go install jd
        run: |
          go install github.com/josephburnett/jd@latest

      - name: Start steamipipe service
        run: |
          steampipe service start

      - name: Run Test Suite
        id: run-test-suite
        timeout-minutes: 15
        continue-on-error: true
        run: |
          chmod +x $GITHUB_WORKSPACE/powerpipe/tests/acceptance/run.sh
          $GITHUB_WORKSPACE/powerpipe/tests/acceptance/run.sh ${{ matrix.test_block }}.bats
          echo "exit_code=$(echo $?)" >> $GITHUB_OUTPUT
          echo ">> here"

      # This job checks whether the test suite has passed or not.
      # Since the exit_code is set only when the bats test suite pass,
      # we have added the if-conditional block
      - name: Check Test Passed/Failed
        if: ${{ success() }}
        continue-on-error: false
        run: |
          if [ ${{ steps.run-test-suite.outputs.exit_code }} -eq 0 ]; then
            exit 0
          else
            exit 1
          fi

      - name: Stop steampipe service
        run: |
          steampipe service stop

  smoke_test_ubuntu_24:
    name: Smoke test (Ubuntu 24, x86_64)
    runs-on: ubuntu-latest
    needs: [goreleaser]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download Linux Build Artifact
        uses: actions/download-artifact@v3
        with:
          name: build-artifact-linux
          path: ./artifacts
  
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
  
      - name: Pull Ubuntu latest Image
        run: docker pull ubuntu:latest
  
      - name: Create and Start Ubuntu latest Container
        run: |
          docker run -d --name ubuntu-24-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts ubuntu:latest tail -f /dev/null

      - name: Get runner/container info
        run: |
          docker exec ubuntu-24-test /scripts/linux_container_info.sh
  
      - name: Install dependencies, create user, and assign necessary permissions
        run: |
          docker exec ubuntu-24-test /scripts/prepare_ubuntu_container.sh

      - name: Run smoke tests
        run: |
          docker exec -u steampipe ubuntu-24-test /scripts/smoke_test.sh
  
      - name: Stop and Remove Container
        run: |
          docker stop ubuntu-24-test
          docker rm ubuntu-24-test

  smoke_test_centos_9:
    name: Smoke test (Centos stream 9, x86_64)
    runs-on: ubuntu-latest
    needs: [goreleaser]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
  
      - name: Download Linux Build Artifact
        uses: actions/download-artifact@v3
        with:
          name: build-artifact-linux
          path: ./artifacts
  
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
  
      - name: Pull CentOS Stream 9 image
        run: docker pull quay.io/centos/centos:stream9
  
      - name: Create and Start CentOS stream9 Container
        run: |
          docker run -d --name centos-stream9-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts quay.io/centos/centos:stream9 tail -f /dev/null

      - name: Get runner/container info
        run: |
          docker exec centos-stream9-test /scripts/linux_container_info.sh
  
      - name: Install dependencies, create user, and assign necessary permissions
        run: |
          docker exec centos-stream9-test /scripts/prepare_centos_container.sh
        
      - name: Run smoke tests
        run: |
          docker exec -u steampipe centos-stream9-test /scripts/smoke_test.sh
  
      - name: Stop and Remove Container
        run: |
          docker stop centos-stream9-test
          docker rm centos-stream9-test

  smoke_test_amazonlinux:
    name: Smoke test (Amazonlinux 2023, x86_64)
    runs-on: ubuntu-latest
    needs: [goreleaser]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download Linux Build Artifact
        uses: actions/download-artifact@v3
        with:
          name: build-artifact-linux
          path: ./artifacts
  
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
  
      - name: Pull Amazon Linux 2023 Image
        run: docker pull amazonlinux:2023
  
      - name: Create and Start Amazon Linux 2023 Container
        run: |
          docker run -d --name amazonlinux-2023-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts amazonlinux:2023 tail -f /dev/null

      - name: Get runner/container info
        run: |
          docker exec amazonlinux-2023-test /scripts/linux_container_info.sh
  
      - name: Install dependencies, create user, and assign necessary permissions
        run: |
          docker exec amazonlinux-2023-test /scripts/prepare_amazonlinux_container.sh

      - name: Run smoke tests
        run: |
          docker exec -u steampipe amazonlinux-2023-test /scripts/smoke_test.sh
  
      - name: Stop and Remove Container
        run: |
          docker stop amazonlinux-2023-test
          docker rm amazonlinux-2023-test

  smoke_test_ubuntu_24_arm64:
    name: Smoke test (Ubuntu 24, ARM64)
    runs-on: ubuntu-latest
    needs: [goreleaser]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download Linux Build Artifact
        uses: actions/download-artifact@v3
        with:
          name: build-artifact-linux-arm
          path: ./artifacts
  
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
  
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: arm64
  
      - name: Create Buildx Builder
        run: |
          docker buildx create --use --name mybuilder
          docker buildx inspect --bootstrap
  
      - name: Pull Ubuntu 24 ARM64 Image
        run: docker pull arm64v8/ubuntu:latest
  
      - name: Create and Start Ubuntu 24 ARM64 Container
        run: |
          docker run -d --name ubuntu-24-arm64-test -v ${{ github.workspace }}/artifacts:/artifacts -v ${{ github.workspace }}/scripts:/scripts arm64v8/ubuntu:latest tail -f /dev/null

      - name: Get runner/container info
        run: |
          docker exec ubuntu-24-arm64-test /scripts/linux_container_info.sh

      - name: Install dependencies, create user, and assign necessary permissions
        run: |
          docker exec ubuntu-24-arm64-test /scripts/prepare_ubuntu_container.sh

      - name: Run smoke tests
        run: |
          docker exec -u steampipe ubuntu-24-arm64-test /scripts/smoke_test.sh
  
      - name: Stop and Remove Container
        run: |
          docker stop ubuntu-24-arm64-test
          docker rm ubuntu-24-arm64-test

  smoke_test_darwin_amd:
    name: Smoke test (MacOS 13, x86_64)
    runs-on: macos-13
    needs: [goreleaser]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download Darwin Build Artifact
        uses: actions/download-artifact@v3
        with:
          name: build-artifact-darwin
          path: ~/artifacts

      - name: Extract Darwin Artifacts and Install Binary
        run: |
          tar -xf ~/artifacts/darwin.tar.gz -C /usr/local/bin/

      - name: Install Steampipe(Darwin)
        run: |
          sudo /bin/sh -c "$(curl -fsSL https://steampipe.io/install/steampipe.sh)"
          steampipe -v

      - name: Install jq
        run: |
          brew install jq

      - name: Get runner/container info
        run: |
          uname -a
          sw_vers

      - name: Run Smoke tests
        run: |
          ls -al $GITHUB_WORKSPACE/scripts
          chmod +x $GITHUB_WORKSPACE/scripts/smoke_test.sh
          $GITHUB_WORKSPACE/scripts/smoke_test.sh

  build_assets:
    name: Build and Release Dashboard Assets
    needs: [acceptance_test, smoke_test_ubuntu_24, smoke_test_centos_9, smoke_test_amazonlinux, smoke_test_ubuntu_24_arm64, smoke_test_darwin_amd]
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.set_version.outputs.version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: powerpipe
          ref: ${{ github.event.ref }}

      - name: Calculate version
        id: calculate_version
        run: |
          VERSION_JSON=$(cat powerpipe/internal/version/version.json)
          MAJOR=$(echo $VERSION_JSON | jq '.major')
          MINOR=$(echo $VERSION_JSON | jq '.minor')
          PATCH=$(echo $VERSION_JSON | jq '.patch')
          VERSION="v$MAJOR.$MINOR.$PATCH"

          if [ "${{ github.event.inputs.pre_release_identifier }}" = "alpha" ]; then
            echo "VERSION=${VERSION}-alpha.$(date +'%Y%m%d%H%M')" >> $GITHUB_ENV
          elif [ "${{ github.event.inputs.pre_release_identifier }}" = "beta" ]; then
            echo "VERSION=${VERSION}-beta.$(date +'%Y%m%d%H%M')" >> $GITHUB_ENV
          elif [ "${{ github.event.inputs.pre_release_identifier }}" = "rc" ]; then
            echo "VERSION=${VERSION}-rc.$(date +'%Y%m%d%H%M')" >> $GITHUB_ENV
          fi

      - name: Set VERSION as output
        run: echo "::set-output name=version::$VERSION"
        id: set_version

      - name: Trim asset version prefix and Validate
        run: |-
          echo $VERSION
          trim=${VERSION#"v"}
          echo $trim
          if [[ $trim =~  ^[0-9]+\.[0-9]+\.[0-9]+(-.+)?$ ]]; then
            echo "Version OK: $trim"
          else
            echo "Invalid version: $trim"
            exit 1
          fi
          echo "VERSION=${trim}" >> $GITHUB_ENV

      - name: Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: ">=18.12.0"

      - name: Setup Yarn
        working-directory: ./powerpipe/ui/dashboard
        run: |-
          rm -rf ./build

      # Cache node modules to speed up builds
      - name: Cache node modules
        id: cache-npm
        uses: actions/cache@v4
        env:
          cache-name: cache-node-modules
        with:
          # npm cache files are stored in `~/.npm` on Linux/macOS
          path: ~/.npm
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}

      - name: YARN Install
        working-directory: ./powerpipe/ui/dashboard
        run: |-
          unset CI
          corepack enable
          yarn install

      - name: Generate dashboard UI icons
        working-directory: ./powerpipe/ui/dashboard
        run: |-
          make setup-icons

      - name: Run Unit Tests
        working-directory: ./powerpipe/ui/dashboard
        run: |
          CI=true yarn test

      - name: YARN Build
        working-directory: ./powerpipe/ui/dashboard
        run: |-
          unset CI
          yarn build
        env:
          REACT_APP_HEAP_ID: ${{ secrets.HEAP_ANALYTICS_PRODUCTION_ID }}
          REACT_APP_VERSION: ${{ env.VERSION }}

      - name: Move Build Assets
        run: |-
          mkdir -p dashboard_ui_build
          mv ./powerpipe/ui/dashboard/build/* dashboard_ui_build/
          ls -la dashboard_ui_build

      - name: Create a version file
        run: |-
          JSON_STRING=$( jq -n \
              --arg version "$VERSION" \
            '{
                "version":$version,
            }' )

            echo $JSON_STRING > ./dashboard_ui_build/version.json

      - name: Archive built dashboard assets
        run: |
          cd dashboard_ui_build
          tar -czvf ../dashboard_ui_assets.tar.gz *

      - name: Upload dashboard assets artifact
        uses: actions/upload-artifact@v4
        with:
          name: dashboard-ui-assets-archive
          path: dashboard_ui_assets.tar.gz

  build_and_release:
    name: Build and Release Powerpipe
    needs: [build_assets]
    runs-on: ubuntu-latest
    env:
      VERSION: ${{ needs.build_assets.outputs.version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          path: powerpipe
          ref: ${{ github.event.ref }}

      - name: Checkout Pipe Fittings Components repository
        uses: actions/checkout@v4
        with:
          repository: turbot/pipe-fittings
          path: pipe-fittings
          ref: v1.6.x

      - name: Tag Release
        run: |
          cd powerpipe
          git config user.name "Powerpipe GitHub Actions Bot"
          git config user.email noreply@github.com
          git tag $VERSION
          git push origin $VERSION

      # this is required, check golangci-lint-action docs
      - uses: actions/setup-go@v5
        with:
          go-version: "1.22"
          cache: false # setup-go v4 caches by default, do not change this parameter, check golangci-lint-action doc: https://github.com/golangci/golangci-lint-action/pull/704

      - name: Download Dashboard UI Assets Artifact
        uses: actions/download-artifact@v4
        with:
          name: dashboard-ui-assets-archive
          path: powerpipe/internal/dashboardassets

      - name: Check assets location
        run: |
          ls -la powerpipe/internal/dashboardassets
          mv powerpipe/internal/dashboardassets/dashboard_ui_assets.tar.gz powerpipe/internal/dashboardassets/assets.tar.gz
          ls -la powerpipe/internal/dashboardassets

      - name: Setup release environment
        run: |-
          cd powerpipe
          echo 'GITHUB_TOKEN=${{secrets.GH_ACCESS_TOKEN}}' > .release-env

      - name: Release publish
        run: |-
          cd powerpipe
          make release