release.yml

name: Release

on:
  push:
    tags:
      - "[0-9]+.[0-9]+.[0-9]+a[0-9]+"
      - "[0-9]+.[0-9]+.[0-9]+rc[0-9]+"
      - "[0-9]+.[0-9]+.[0-9]+"
      - "[0-9]+.[0-9]+.[0-9]+.post[0-9]+"

env:
  BUILD_INCREMENT: 150
  PIP_DISABLE_PIP_VERSION_CHECK: on
  PIP_DEFAULT_TIMEOUT: 10
  PIP_PROGRESS_BAR: off

jobs:
  python-lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - run: python -m pip install pre-commit
      - run: pre-commit run -a --show-diff-on-failure

  frontend-build:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: frontend
    steps:
      - uses: actions/checkout@v4
      - name: Set up Node
        uses: actions/setup-node@v4
        with:
          node-version: 18
      - name: Install packages
        run: npm install
      - name: Build dist
        run: npm run build
      - name: Upload dist
        uses: actions/upload-artifact@v4
        with:
          name: frontend-build
          path: frontend/build

  python-test:
    needs: [ python-lint, frontend-build ]
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ macos-latest, ubuntu-latest, windows-latest ]
        python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: pip install -U '.[all]' -r requirements_dev.txt
      - name: Download frontend build
        uses: actions/download-artifact@v4
        with:
          name: frontend-build
          path: src/dstack/_internal/server/statics
      - name: Run pytest on POSIX
        if: matrix.os != 'windows-latest'
        # Skip Postgres tests on macos since macos runner doesn't have Docker.
        # Skip Postgres tests for Python 3.8 since testcontainers<4 doesn't support asyncpg correctly.
        run: |
          RUNPOSTGRES=""
          if [ "${{ matrix.os }}" != "macos-latest" ] && [ "${{ matrix.python-version }}" != "3.8" ]; then
            RUNPOSTGRES="--runpostgres"
          fi
          pytest src/tests --runui $RUNPOSTGRES
      - name: Run pytest on Windows
        if: matrix.os == 'windows-latest'
        run: |
          pytest src/tests --runui --runpostgres

  runner-test:
    defaults:
      run:
        working-directory: runner
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version-file: runner/go.mod
          cache-dependency-path: runner/go.sum
      - name: Run golangci-lint
        uses: golangci/golangci-lint-action@v6
        with:
          version: v1.58
          args: --timeout=20m
          working-directory: runner
      - name: Test
        run: |
          go version
          go fmt $(go list ./... | grep -v /vendor/)
          go vet $(go list ./... | grep -v /vendor/)
          go test -race $(go list ./... | grep -v /vendor/)

  runner-compile:
    needs: [runner-test]
    defaults:
      run:
        working-directory: runner
    env:
      REPO_NAME: github.com/dstackai/dstack
    strategy:
      matrix:
        include:
          - { goos: "linux", goarch: "amd64", runson: "ubuntu-latest" }
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version-file: runner/go.mod
          cache-dependency-path: runner/go.sum
      - name: build
        env:
          GOOS: ${{ matrix.goos }}
          GOARCH: ${{ matrix.goarch }}
          ACTIONSOS: ${{ matrix.runson }}
          CGO_ENABLED: 0
        run: |
          VERSION=${GITHUB_REF#refs/tags/}
          go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-runner-$GOOS-$GOARCH $REPO_NAME/runner/cmd/runner
          go build -ldflags "-X 'main.Version=$VERSION' -extldflags '-static'" -o dstack-shim-$GOOS-$GOARCH $REPO_NAME/runner/cmd/shim
      - uses: actions/upload-artifact@v4
        with:
          name: dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
          path: |
            runner/dstack-runner-${{ matrix.goos }}-${{ matrix.goarch }}
            runner/dstack-shim-${{ matrix.goos }}-${{ matrix.goarch }}

  gateway-build:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: gateway
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install AWS
        run: pip install awscli
      - name: Install dependencies
        run: pip install wheel build
      - name: Store version
        run: echo VERSION=${GITHUB_REF#refs/tags/} > $GITHUB_ENV
      - name: Build package
        run: |
          echo "__version__ = \"${{ env.VERSION }}\"" > src/dstack/gateway/version.py
          python -m build .
      - name: Upload to S3
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          WHEEL=dstack_gateway-${{ env.VERSION }}-py3-none-any.whl
          aws s3 cp dist/$WHEEL "s3://dstack-gateway-downloads/release/$WHEEL"
          echo "${{ env.VERSION }}" | aws s3 cp - "s3://dstack-gateway-downloads/release/latest-version"

  runner-upload:
    needs: [runner-compile, gateway-build, python-test]
    runs-on: ubuntu-latest
    steps:
      - name: Install AWS
        run: pip install awscli
      - name: Download Runner
        uses: actions/download-artifact@v4
        with:
          pattern: dstack-runner-*
          merge-multiple: true
          path: runner
      - name: Upload to S3
        working-directory: runner
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          VERSION=${GITHUB_REF#refs/tags/}
          aws s3 cp . "s3://dstack-runner-downloads/$VERSION/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read
          aws s3 cp . "s3://dstack-runner-downloads/latest/binaries/" --recursive --exclude "*" --include "dstack-*" --acl public-read

  pypi-upload:
    needs: [ python-test, runner-upload ]
    runs-on: ubuntu-latest
    outputs:
      LATEST: ${{ steps.set_latest.outputs.LATEST }}
      name: Set latest variable
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Download frontend build
        uses: actions/download-artifact@v4
        with:
          name: frontend-build
          path: src/dstack/_internal/server/statics
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install wheel twine packaging
      - name: Set output
        id: set_latest
        run: |
          VERSION=${GITHUB_REF#refs/tags/}
          LATEST=$(python -c "from packaging import version as pkg_version; print('' if pkg_version.parse('$VERSION').is_prerelease else '1', end='')")
          echo "LATEST=$LATEST" >> "$GITHUB_OUTPUT"
      - name: Upload pip package
        run: |
          VERSION=${GITHUB_REF#refs/tags/}
          BASE_IMAGE=$(cat src/dstack/version.py | grep base_image)
          echo "__version__ = \"$VERSION\"" > src/dstack/version.py
          echo "__is_release__ = True" >> src/dstack/version.py
          echo $BASE_IMAGE >> src/dstack/version.py
          cp README.md src
          python setup.py sdist bdist_wheel -v
          python -m twine upload --repository pypi --username ${{ secrets.PYPI_USERNAME }} --password ${{ secrets.PYPI_PASSWORD }} dist/*        

  update-get-dstack-tag:
    needs: [ pypi-upload ]
    runs-on: ubuntu-latest
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
    steps:
      - name: Install AWS
        run: pip install awscli
      - run: |
          VERSION=${GITHUB_REF#refs/tags/}
          echo $VERSION | aws s3 cp - s3://get-dstack/cli/latest-version --acl public-read

  server-docker-upload:
    needs: [ pypi-upload ]
    defaults:
      run:
        working-directory: docker/server
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Build and upload to DockerHub
        run: |
          VERSION=${GITHUB_REF#refs/tags/}
          docker buildx build --platform linux/arm64/v8 --build-arg VERSION=$VERSION --push --provenance=false --tag dstackai/dstack:$VERSION-arm64 -f release/Dockerfile .
          docker buildx build --platform linux/amd64 --build-arg VERSION=$VERSION --push --provenance=false --tag dstackai/dstack:$VERSION-amd64 -f release/Dockerfile .
          docker manifest create dstackai/dstack:$VERSION --amend dstackai/dstack:$VERSION-arm64 --amend dstackai/dstack:$VERSION-amd64
          docker manifest push dstackai/dstack:$VERSION
          if [ -n "${{ needs.pypi-upload.outputs.LATEST }}" ]; then
            docker manifest create dstackai/dstack:latest --amend dstackai/dstack:$VERSION-arm64 --amend dstackai/dstack:$VERSION-amd64
            docker manifest push dstackai/dstack:latest
          fi
      - name: Docker Hub Description
        uses: peter-evans/dockerhub-description@v4
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
          repository: dstackai/dstack
          readme-filepath: ./docker/server/README.md

  generate-json-schema:
    needs: [ pypi-upload ]
    env:
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install AWS
        run: pip install awscli
      - name: Install dstack
        run: pip install .
      - name: Generate json schema
        run: |
          python -c "from dstack._internal.core.models.configurations import DstackConfiguration; print(DstackConfiguration.schema_json(indent=2))" > configuration.json
          python -c "from dstack._internal.core.models.profiles import ProfilesConfig; print(ProfilesConfig.schema_json(indent=2))" > profiles.json
      - name: Upload json schema to S3
        run: |
          VERSION=${GITHUB_REF#refs/tags/}
          aws s3 cp configuration.json "s3://dstack-runner-downloads/$VERSION/schemas/configuration.json" --acl public-read
          aws s3 cp profiles.json "s3://dstack-runner-downloads/$VERSION/schemas/profiles.json" --acl public-read      
          if [ -n "${{ needs.pypi-upload.outputs.LATEST }}" ]; then
            aws s3 cp configuration.json "s3://dstack-runner-downloads/latest/schemas/configuration.json" --acl public-read
            aws s3 cp profiles.json "s3://dstack-runner-downloads/latest/schemas/profiles.json" --acl public-read
          fi