Skip to content

Commit

Permalink
Update ci (#40)
Browse files Browse the repository at this point in the history
  • Loading branch information
4eUeP authored Apr 12, 2024
1 parent 0f71a0b commit 48cdf61
Show file tree
Hide file tree
Showing 4 changed files with 168 additions and 62 deletions.
48 changes: 48 additions & 0 deletions .github/actions/collect-tests/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
name: "collect-tests"
description: "collect-tests"

inputs:
tests-repo:
description: "The repository of kafka-tests"
required: true
default: ${{ github.repository }}
path:
description: "Relative path under $GITHUB_WORKSPACE to place the repository"
default: "."
required: true
hstream-image:
description: "hstream-image"
required: true
default: "hstreamdb/hstream:latest"

outputs:
legacy_tests:
description: "legacy_tests"
value: ${{ steps.collect_tests.outputs.legacy_tests }}
tests:
description: "tests"
value: ${{ steps.collect_tests.outputs.tests }}

runs:
using: "composite"
steps:
- uses: actions/setup-java@v2
with:
distribution: "adopt"
java-version: 11
cache: "gradle"

- name: collect all tests
id: collect_tests
shell: bash
run: |
cd ${{ inputs.path }}
tmp_dir=$(mktemp -d)
result_dir=$tmp_dir hstream_image=${{ inputs.hstream-image }} ./script/collect_tests.sh
legacy_tests=$(cat $tmp_dir/legacy_tests.json)
echo "legacy_tests=$legacy_tests" >> $GITHUB_OUTPUT
tests=$(cat $tmp_dir/tests.json)
echo "tests=$tests" >> $GITHUB_OUTPUT
94 changes: 94 additions & 0 deletions .github/actions/run-tests/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
name: "run-tests"
description: "run-tests"

inputs:
tests-repo:
description: "The repository of kafka-tests"
required: true
default: ${{ github.repository }}
path:
description: "Relative path under $GITHUB_WORKSPACE to place the repository"
required: true
default: "."
hstream-image:
description: "hstream-image"
required: true
default: "hstreamdb/hstream:latest"
distribution:
description: "The distribution of java"
required: true
default: "adopt"
java-version:
description: "The version of java"
required: true
default: "11"
tests_arg:
description: "The argument of tests"
required: true
kafka_client_version:
description: "The version of kafka client"
required: true
is_lagacy_tests:
description: "Is lagacy tests"
required: true

runs:
using: "composite"
steps:
- uses: actions/setup-java@v2
with:
distribution: ${{ inputs.distribution }}
java-version: ${{ inputs.java-version }}
cache: "gradle"

- name: Start required services
if: ${{ inputs.is_lagacy_tests != 'true' }}
shell: bash
run: |
cd ${{ inputs.path }}
./script/dev-tools start-services --services store zookeeper
- name: Run lagacy tests
if: ${{ inputs.is_lagacy_tests == 'true' }}
shell: bash
run: |
cd ${{ inputs.path }}
echo "=> Run tests ${{ inputs.tests_arg }} with kafka client ${{ inputs.kafka_client_version }}"
KAFKA_CLIENT_VERSION=${{ inputs.kafka_client_version }} ./gradlew test \
--fail-fast -i ${{ inputs.tests_arg }}
- name: Run tests
if: ${{ inputs.is_lagacy_tests != 'true' }}
shell: bash
run: |
cd ${{ inputs.path }}
echo "=> Run tests ${{ matrix.tests.tests_arg }} with kafka client ${{ matrix.tests.kafka_client_version }}"
KAFKA_CLIENT_VERSION=${{ matrix.tests.kafka_client_version }} ./script/run_test.sh \
--fail-fast -i ${{ matrix.tests.tests_arg }}
- name: generate artifacts name (lagacy_tests)
if: ${{ inputs.is_lagacy_tests == 'true' && failure() }}
shell: bash
run: |
name="reports-lagacy-kafka-tests-$(uuidgen)"
echo "Reports name: $name"
echo "name=$name" >> $GITHUB_OUTPUT
- name: generate artifacts name
id: artifacts
if: ${{ inputs.is_lagacy_tests != 'true' && failure() }}
shell: bash
run: |
name="reports-kafka-tests-$(uuidgen)"
echo "Reports name: $name"
# NOTE: canot use GITHUB_ENV
# https://github.com/orgs/community/discussions/51280
echo "name=$name" >> $GITHUB_OUTPUT
- uses: actions/upload-artifact@v4
if: ${{ failure() }}
with:
name: ${{ steps.artifacts.outputs.name }}
path: |
${{ inputs.path }}/.logs
${{ inputs.path }}/app/build/reports
84 changes: 24 additions & 60 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,16 +6,13 @@ on:
branches: [main]

jobs:
pre-tests:
check:
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest]
distribution: ["adopt"]
java-version: [11]
outputs:
legacy_tests: ${{ steps.group_tests.outputs.legacy_tests }}
tests: ${{ steps.group_tests.outputs.tests }}
steps:
- uses: actions/checkout@v3
with:
Expand All @@ -34,17 +31,18 @@ jobs:
./gradlew spotlessJavaCheck
./gradlew spotlessGroovyGradleCheck
- name: Group all tests
id: group_tests
run: |
tmp_dir=$(mktemp -d)
result_dir=$tmp_dir ./script/collect_tests.sh
legacy_tests=$(cat $tmp_dir/legacy_tests.json)
echo "legacy_tests=$legacy_tests" >> $GITHUB_OUTPUT
pre-tests:
runs-on: ubuntu-latest
outputs:
legacy_tests: ${{ steps.collect_tests.outputs.legacy_tests }}
tests: ${{ steps.collect_tests.outputs.tests }}
steps:
- uses: actions/checkout@v4
with:
submodules: "recursive"

tests=$(cat $tmp_dir/tests.json)
echo "tests=$tests" >> $GITHUB_OUTPUT
- id: collect_tests
uses: ./.github/actions/collect-tests

legacy-tests:
needs: pre-tests
Expand All @@ -59,34 +57,18 @@ jobs:
tests: ${{ fromJson(needs.pre-tests.outputs.legacy_tests) }}

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
submodules: "recursive"

- uses: actions/setup-java@v2
- id: run_tests
uses: ./.github/actions/run-tests
with:
distribution: ${{ matrix.distribution }}
java-version: ${{ matrix.java-version }}
cache: "gradle"

- run: |
UUID=$(uuidgen)
echo "Reports id: $UUID"
echo "UUID=$UUID" >> $GITHUB_ENV
- name: Run tests
run: |
echo "=> Run tests ${{ matrix.tests.tests_arg }} with kafka client ${{ matrix.tests.kafka_client_version }}"
KAFKA_CLIENT_VERSION=${{ matrix.tests.kafka_client_version }} ./gradlew test \
--fail-fast -i ${{ matrix.tests.tests_arg }}
- uses: actions/upload-artifact@v3
if: ${{ failure() }}
with:
name: reports-lagacy-tests-${{ env.UUID }}
path: |
.logs
app/build/reports
tests_arg: ${{ matrix.tests.tests_arg }}
kafka_client_version: ${{ matrix.tests.kafka_client_version }}
is_lagacy_tests: 'true'

tests:
needs: pre-tests
Expand All @@ -101,33 +83,15 @@ jobs:
tests: ${{ fromJson(needs.pre-tests.outputs.tests) }}

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
submodules: "recursive"

- uses: actions/setup-java@v2
- id: run_tests
uses: ./.github/actions/run-tests
with:
distribution: ${{ matrix.distribution }}
java-version: ${{ matrix.java-version }}
cache: "gradle"

- run: |
UUID=$(uuidgen)
echo "Reports id: $UUID"
echo "UUID=$UUID" >> $GITHUB_ENV
- name: Start required services
run: ./script/dev-tools start-services --services store zookeeper

- name: Run tests
run: |
echo "=> Run tests ${{ matrix.tests.tests_arg }} with kafka client ${{ matrix.tests.kafka_client_version }}"
KAFKA_CLIENT_VERSION=${{ matrix.tests.kafka_client_version }} ./script/run_test.sh \
--fail-fast -i ${{ matrix.tests.tests_arg }}
- uses: actions/upload-artifact@v3
if: ${{ failure() }}
with:
name: reports-tests-${{ env.UUID }}
path: |
app/build/reports
tests_arg: ${{ matrix.tests.tests_arg }}
kafka_client_version: ${{ matrix.tests.kafka_client_version }}
is_lagacy_tests: 'false'
4 changes: 2 additions & 2 deletions script/run_test.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
set -e

hstream_image=${hstream_image:-hstreamdb/hstream:latest}
CONFIG_FILE=$PWD/local-data/config.yaml

find_freeport() {
Expand All @@ -17,9 +18,8 @@ generate_config() {
store_admin_port=$(cat $env_file | grep STORE_ADMIN_LOCAL_PORT | cut -d'=' -f2)
zookeeper_port=$(cat $env_file | grep ZOOKEEPER_LOCAL_PORT | cut -d'=' -f2)
base_port=$(find_freeport)
image="hstreamdb/hstream:latest"
sed -e "s/\${base_port}/$base_port/g" \
-e "s#\${image}#$image#g" \
-e "s#\${image}#$hstream_image#g" \
-e "s/\${metastore_port}/$zookeeper_port/g" \
-e "s#\${store_dir}#$PWD/local-data/logdevice#g" \
-e "s/\${store_admin_port}/$store_admin_port/g" \
Expand Down

0 comments on commit 48cdf61

Please sign in to comment.