Skip to content

[Improvement-16887][Dependent Task] Dependet task improvement #5978

[Improvement-16887][Dependent Task] Dependet task improvement

[Improvement-16887][Dependent Task] Dependet task improvement #5978

Workflow file for this run

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
on:
pull_request:
push:
branches:
- dev
- '[0-9]+.[0-9]+.[0-9]+-prepare'
- '[0-9]+.[0-9]+.[0-9]+-release'
name: E2E-K8S
concurrency:
group: E2E-K8S-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
paths-filter:
name: E2E-K8S-Path-Filter
runs-on: ubuntu-latest
outputs:
not-ignore: ${{ steps.filter.outputs.not-ignore }}
steps:
- uses: actions/checkout@v4
- uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721
id: filter
with:
filters: |
not-ignore:
- '!(docs/**)'
e2e-k8s:
name: E2E-K8S-Execute
needs: paths-filter
if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }}
runs-on: ubuntu-latest
timeout-minutes: 40
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: Maximize runner space
uses: ./.github/actions/maximize-build-space
with:
root-reserve-mb: 30720
temp-reserve-mb: 10240
remove-dotnet: 'true'
remove-android: 'true'
remove-haskell: 'true'
remove-codeql: 'true'
remove-docker-images: 'true'
- uses: actions/checkout@v4
with:
submodules: true
- name: Build Image
run: |
./mvnw -B clean package \
-Dmaven.test.skip=true \
-Dspotless.skip=true \
-Pdocker,staging -Ddocker.tag=ci
- name: Create k8s Kind Cluster
run: |
# install kubectl
curl -LO "https://dl.k8s.io/release/v1.28.3/bin/linux/amd64/kubectl"
sudo chmod +x kubectl
sudo mv kubectl /usr/local/bin/kubectl
# install kind
curl -LO https://github.com/kubernetes-sigs/kind/releases/download/v0.20.0/kind-linux-amd64
sudo chmod +x kind-linux-amd64
sudo mv kind-linux-amd64 /usr/local/bin/kind
kind version
# create kind cluster
kind_node_image="kindest/node:v1.23.17"
echo "Kubernetes version: ${kind_node_image}"
kind create cluster --name dolphinscheduler --image ${kind_node_image}
kubectl version
kubectl get all --all-namespaces
- name: Load images
run: |
components=("master" "worker" "api" "tools" "alert-server")
for component in "${components[@]}"; do
kind load docker-image apache/dolphinscheduler-${component}:ci --name dolphinscheduler
done
- name: Helm install dolphinscheduler
working-directory: ${{ github.workspace }}/deploy/kubernetes/dolphinscheduler
run: |
# install helm
curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
# install ds chart via helm
helm repo add bitnami https://charts.bitnami.com/bitnami
helm dependency update .
helm upgrade --install --create-namespace -n dolphinscheduler dolphinscheduler . -f - <<EOF
image:
registry: apache
tag: ci
master:
replicas: 1
livenessProbe:
initialDelaySeconds: 120
readinessProbe:
initialDelaySeconds: 120
worker:
replicas: 1
livenessProbe:
initialDelaySeconds: 120
readinessProbe:
initialDelaySeconds: 120
alert:
livenessProbe:
initialDelaySeconds: 120
readinessProbe:
initialDelaySeconds: 120
api:
livenessProbe:
initialDelaySeconds: 120
readinessProbe:
initialDelaySeconds: 120
EOF
- name: Wait for pods
run: |
JSONPATH='{range .items[*]}{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}';
components=("master" "worker" "api" "alert")
for component in "${components[@]}"; do
until kubectl -n dolphinscheduler get pods -l app.kubernetes.io/component=${component} -o jsonpath="$JSONPATH" 2>&1 | grep -q "Ready=True"; do
echo "waiting for dolphinscheduler ${component} to be available"
sleep 10
kubectl get pods --all-namespaces
kubectl get events --all-namespaces
for pod in $(kubectl get pods -n dolphinscheduler -o jsonpath='{.items[*].metadata.name}'); do
echo "logs for pod $pod:"
kubectl logs --tail=1000 -n dolphinscheduler $pod
done
done
done
result:
name: E2E-K8S-Result
runs-on: ubuntu-latest
timeout-minutes: 30
needs: [ e2e-k8s, paths-filter ]
if: always()
steps:
- name: Status
run: |
if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then
echo "Skip E2E-K8S!"
exit 0
fi
if [[ ${{ needs.e2e-k8s.result }} != 'success' ]]; then
echo "E2E-K8S Failed!"
exit -1
fi