e2e.yml

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

on:
  pull_request:
  push:
    branches:
      - dev
      - '[0-9]+.[0-9]+.[0-9]+-prepare'
      - '[0-9]+.[0-9]+.[0-9]+-release'

name: E2E

concurrency:
  group: e2e-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

permissions:
  pull-requests: write

jobs:
  paths-filter:
    name: E2E-Path-Filter
    runs-on: ubuntu-latest
    outputs:
      not-ignore: ${{ steps.filter.outputs.not-ignore }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
        id: filter
        with:
          filters: |
            not-ignore:
              - '!(docs/**)'
  build:
    name: E2E-Build
    needs: paths-filter
    if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Sanity Check
        uses: ./.github/actions/sanity-check
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
      - name: Cache local Maven repository
        uses: actions/cache@v3
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
          restore-keys: ${{ runner.os }}-maven-
      - name: Build Image
        run: |
          ./mvnw -B clean install \
          -Dmaven.test.skip \
          -Dmaven.javadoc.skip \
          -Dspotless.skip=true \
          -Pdocker,release -Ddocker.tag=ci \
          -pl dolphinscheduler-standalone-server -am
      - name: Export Docker Images
        run: |
          docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
          && du -sh /tmp/standalone-image.tar
      - uses: actions/upload-artifact@v4
        name: Upload Docker Images
        with:
          name: standalone-image
          path: /tmp/standalone-image.tar
          retention-days: 1
  e2e:
    name: ${{ matrix.case.name }}
    needs: build
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      matrix:
        case:
          - name: Tenant
            class: org.apache.dolphinscheduler.e2e.cases.TenantE2ETest
          - name: User
            class: org.apache.dolphinscheduler.e2e.cases.UserE2ETest
          - name: WorkerGroup
            class: org.apache.dolphinscheduler.e2e.cases.WorkerGroupE2ETest
          - name: Project
            class: org.apache.dolphinscheduler.e2e.cases.ProjectE2ETest
          - name: Queue
            class: org.apache.dolphinscheduler.e2e.cases.QueueE2ETest
          - name: Environment
            class: org.apache.dolphinscheduler.e2e.cases.EnvironmentE2ETest
          - name: Cluster
            class: org.apache.dolphinscheduler.e2e.cases.ClusterE2ETest
          - name: Token
            class: org.apache.dolphinscheduler.e2e.cases.TokenE2ETest
          - name: Workflow
            class: org.apache.dolphinscheduler.e2e.cases.WorkflowE2ETest
          - name: WorkflowHttp
            class: org.apache.dolphinscheduler.e2e.cases.WorkflowHttpTaskE2ETest
          - name: WorkflowJava
            class: org.apache.dolphinscheduler.e2e.cases.WorkflowJavaTaskE2ETest
#          - name: WorkflowForSwitch
#            class: org.apache.dolphinscheduler.e2e.cases.WorkflowSwitchE2ETest
          - name: FileManage
            class: org.apache.dolphinscheduler.e2e.cases.FileManageE2ETest
          - name: MysqlDataSource
            class: org.apache.dolphinscheduler.e2e.cases.MysqlDataSourceE2ETest
          - name: ClickhouseDataSource
            class: org.apache.dolphinscheduler.e2e.cases.ClickhouseDataSourceE2ETest
          - name: PostgresDataSource
            class: org.apache.dolphinscheduler.e2e.cases.PostgresDataSourceE2ETest
          - name: ShellTaskE2ETest
            class: org.apache.dolphinscheduler.e2e.cases.tasks.ShellTaskE2ETest
          - name: PythonTaskE2ETest
            class: org.apache.dolphinscheduler.e2e.cases.tasks.PythonTaskE2ETest
    env:
      RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Set up JDK 11
        uses: actions/setup-java@v4
        with:
          java-version: 11
          distribution: 'adopt'
      - name: Collect Workflow Telemetry
        uses: ./.github/actions/workflow-telemetry-action
        with:
          comment_on_pr: false
      - name: Cache local Maven repository
        uses: actions/cache@v3
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
          restore-keys: ${{ runner.os }}-maven-
      - uses: actions/download-artifact@v4
        name: Download Docker Images
        with:
          name: standalone-image
          path: /tmp
      - name: Load Docker Images
        run: |
          docker load -i /tmp/standalone-image.tar
      - name: Run Test
        run: |
          ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \
            -DfailIfNoTests=false \
            -Dtest=${{ matrix.case.class }} test
      - uses: actions/upload-artifact@v4
        if: always()
        name: Upload Recording
        with:
          name: recording-${{ matrix.case.name }}
          path: ${{ env.RECORDING_PATH }}
          retention-days: 1
  e2e-optional:
    name: ${{ matrix.case.name }}
    needs: build
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      matrix:
        case:
          - name: SqlServerDataSource
            class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest
          - name: HiveDataSource
            class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest
    env:
      RECORDING_PATH: /tmp/recording-${{ matrix.case.name }}
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Cache local Maven repository
        uses: actions/cache@v3
        with:
          path: ~/.m2/repository
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
          restore-keys: ${{ runner.os }}-maven-
      - uses: actions/download-artifact@v4
        name: Download Docker Images
        with:
          name: standalone-image
          path: /tmp
      - name: Load Docker Images
        run: |
          docker load -i /tmp/standalone-image.tar
      - name: Run Test
        run: |
          ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \
            -DfailIfNoTests=false \
            -Dtest=${{ matrix.case.class }} test
      - uses: actions/upload-artifact@v4
        if: always()
        name: Upload Recording
        with:
          name: recording-${{ matrix.case.name }}
          path: ${{ env.RECORDING_PATH }}
          retention-days: 1
  result:
    name: E2E
    runs-on: ubuntu-latest
    timeout-minutes: 30
    needs: [ e2e, paths-filter ]
    if: always()
    steps:
      - name: Status
        run: |
          if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
            echo "Skip E2E!"
            exit 0
          fi
          if [[ ${{ needs.e2e.result }} != 'success' ]]; then
            echo "E2E Failed!"
            exit -1
          fi