Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat/e2e test #4486

Merged
merged 5 commits into from
Jan 17, 2024
Merged

Feat/e2e test #4486

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b82301e
init e2e multi node test
bxy4543 Dec 15, 2023
cddda8b
fix get cert kube && get ready node
bxy4543 Dec 18, 2023
10ac12a
generated terraform && add workflow
bxy4543 Dec 20, 2023
96cc839
optimize test
bxy4543 Jan 16, 2024
a590744
ignore test ci-lint
bxy4543 Jan 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions .github/workflows/e2e_test_apply.yml → .github/workflows/e2e_k3s_multi_node.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
name: E2E Apply Test
name: E2E k3s Multi Node Test

on:
workflow_dispatch:
pull_request_target:
types: [ labeled ]

env:
TERRAFORM_VERSION: "1.5.7"

jobs:
call_ci_workflow:
uses: ./.github/workflows/import-patch-image.yml
Expand All @@ -27,6 +30,10 @@ jobs:
test_arm64_result: ${{ steps.apply_test.outputs.test_arm64_result }}
test_amd64_result: ${{ steps.apply_test.outputs.test_amd64_result }}
steps:
- name: Download terraform
uses: hashicorp/setup-terraform@v3
with:
terraform_version: ${{ env.TERRAFORM_VERSION }}
- name: Download sealos binary
uses: actions/download-artifact@v3
with:
Expand Down Expand Up @@ -55,7 +62,8 @@ jobs:
id: apply_test
shell: bash
env:
SEALOS_E2E_TEST_IMAGE_NAME: registry.cn-shanghai.aliyuncs.com/labring/kubernetes:v1.25.6
SEALOS_TEST_ARCH: ${{ matrix.arch }}
SEALOS_E2E_TEST_IMAGE_NAME: registry.cn-shanghai.aliyuncs.com/labring/k3s:v1.28.4
SEALOS_E2E_TEST_PATCH_IMAGE_TAR: /tmp/sealos/images/patch-${{ matrix.arch }}.tar.gz
SEALOS_E2E_TEST_PATCH_IMAGE_NAME: ghcr.io/labring/sealos-patch:${{ env.GIT_COMMIT_SHORT_SHA }}-${{ matrix.arch }}
SEALOS_E2E_TEST_SEALOS_BIN_PATH: /tmp/sealos/bin/sealos
Expand All @@ -68,7 +76,7 @@ jobs:
sudo su
chmod a+x /tmp/e2e.test
gzip /tmp/sealos/images/patch-${{ matrix.arch }}.tar
/tmp/e2e.test --ginkgo.v --ginkgo.focus="E2E_sealos_apply_infra_test"
/tmp/e2e.test --ginkgo.v --ginkgo.focus="E2E_sealos_multi_node_test"
echo 'test_${{ matrix.arch }}_result=success' >> "$GITHUB_OUTPUT"
issue_commit:
needs: [ e2e_apply_test ]
Expand All @@ -87,5 +95,5 @@ jobs:
comment-author: 'github-actions[bot]'
repository: ${{ github.repository }}
body: |
🤖 says: E2E ARM64 Apply Test Result: ${{ env.ARM_TEST_RESULT }} , E2E AMD64 Apply Test Result: ${{ env.AMD_TEST_RESULT }}<br/>See: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
🤖 says: E2E k3s multi node `ARM64` Test Result: ${{ env.ARM_TEST_RESULT }} , E2E k3s multi node `AMD64` Apply Test Result: ${{ env.AMD_TEST_RESULT }}<br/>See: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
token: "${{ secrets.GH_PAT }}"
99 changes: 99 additions & 0 deletions .github/workflows/e2e_k8s_multi_node.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
name: E2E k8s Multi Node Test

on:
workflow_dispatch:
pull_request_target:
types: [ labeled ]

env:
TERRAFORM_VERSION: "1.5.7"

jobs:
call_ci_workflow:
uses: ./.github/workflows/import-patch-image.yml
if: ${{ github.event.action == 'labeled' && github.event.label.name == 'need-e2e-apply-test' }}
with:
arch: amd64,arm64
e2e: true
image: true
target_pull: true
e2e_apply_test:
needs: [ call_ci_workflow ]
runs-on: ubuntu-latest
permissions:
issues: write
strategy:
fail-fast: false
matrix:
arch: [ arm64, amd64 ]
outputs:
test_arm64_result: ${{ steps.apply_test.outputs.test_arm64_result }}
test_amd64_result: ${{ steps.apply_test.outputs.test_amd64_result }}
steps:
- name: Download terraform
uses: hashicorp/setup-terraform@v3
with:
terraform_version: ${{ env.TERRAFORM_VERSION }}
- name: Download sealos binary
uses: actions/download-artifact@v3
with:
name: sealos-${{ matrix.arch }}
path: /tmp/sealos/bin/
- name: Download sealos
uses: actions/download-artifact@v3
with:
name: sealos-amd64
path: /tmp/verify/
- name: Verify sealos
run: |
sudo chmod a+x /tmp/verify/sealos
sudo /tmp/verify/sealos version
- name: Download patch image tar
uses: actions/download-artifact@v3
with:
name: patch-image-${{ matrix.arch }}.tar
path: /tmp/sealos/images/
- name: Download e2e test
uses: actions/download-artifact@v3
with:
name: e2e.test
path: /tmp/
- name: Install ginkgo && Run e2e apply test
id: apply_test
shell: bash
env:
SEALOS_TEST_ARCH: ${{ matrix.arch }}
SEALOS_E2E_TEST_IMAGE_NAME: registry.cn-shanghai.aliyuncs.com/labring/kubernetes:v1.28.4
SEALOS_E2E_TEST_PATCH_IMAGE_TAR: /tmp/sealos/images/patch-${{ matrix.arch }}.tar.gz
SEALOS_E2E_TEST_PATCH_IMAGE_NAME: ghcr.io/labring/sealos-patch:${{ env.GIT_COMMIT_SHORT_SHA }}-${{ matrix.arch }}
SEALOS_E2E_TEST_SEALOS_BIN_PATH: /tmp/sealos/bin/sealos
ALIYUN_ACCESS_KEY_ID: ${{ secrets.E2E_ALIYUN_ACCESS_KEY_ID }}
ALIYUN_ACCESS_KEY_SECRET: ${{ secrets.E2E_ALIYUN_ACCESS_KEY_SECRET }}
ALIYUN_RESOURCE_GROUP_ID: ${{ secrets.E2E_ALIYUN_RESOURCE_GROUP_ID }}
ALIYUN_REGION_ID: ${{ secrets.E2E_ALIYUN_REGION_ID }}
run: |
set -ex
sudo su
chmod a+x /tmp/e2e.test
gzip /tmp/sealos/images/patch-${{ matrix.arch }}.tar
/tmp/e2e.test --ginkgo.v --ginkgo.focus="E2E_sealos_multi_node_test"
echo 'test_${{ matrix.arch }}_result=success' >> "$GITHUB_OUTPUT"
issue_commit:
needs: [ e2e_apply_test ]
runs-on: ubuntu-latest
permissions:
issues: write
if: ${{ always() && github.event.label.name == 'need-e2e-apply-test' }} #success() || failure()
steps:
- name: Add comment to PR with test status
uses: peter-evans/create-or-update-comment@v1
env:
ARM_TEST_RESULT: ${{ needs.e2e_apply_test.outputs.test_arm64_result == 'success' && 'Success 🎉🎉!' || '‼️ Failure' }}
AMD_TEST_RESULT: ${{ needs.e2e_apply_test.outputs.test_amd64_result == 'success' && 'Success 🎉🎉!' || '‼️ Failure' }}
with:
issue-number: ${{ github.event.number }}
comment-author: 'github-actions[bot]'
repository: ${{ github.repository }}
body: |
🤖 says: E2E k8s multi node `ARM64` Test Result: ${{ env.ARM_TEST_RESULT }} , E2E k8s multi node `AMD64` Test Result: ${{ env.AMD_TEST_RESULT }}<br/>See: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
token: "${{ secrets.GH_PAT }}"
21 changes: 19 additions & 2 deletions pkg/client-go/kubernetes/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ package kubernetes
import (
"path/filepath"

"github.com/labring/sealos/pkg/utils/logger"

fileutil "github.com/labring/sealos/pkg/utils/file"

"k8s.io/client-go/discovery"
Expand Down Expand Up @@ -44,8 +46,7 @@ type kubernetesClient struct {
config *rest.Config
}

// NewKubernetesClient creates a KubernetesClient
func NewKubernetesClient(kubeconfig, apiserver string) (Client, error) {
func newKubernetesClient(kubeconfig, apiserver string, insecure bool) (Client, error) {
if kubeconfig == "" || !fileutil.IsExist(kubeconfig) {
defaultKubeconfig := filepath.Join(homedir.HomeDir(), ".kube", "config")
if !fileutil.IsExist(defaultKubeconfig) {
Expand All @@ -60,6 +61,13 @@ func NewKubernetesClient(kubeconfig, apiserver string) (Client, error) {
}
config.QPS = 1e6
config.Burst = 1e6
if insecure {
config.TLSClientConfig.CAFile = ""
config.TLSClientConfig.CAData = nil
config.TLSClientConfig.Insecure = true
//logger.Info("insecure mode, skip verify apiserver certificate", config.TLSClientConfig)
}
logger.Info("kubernetes client config", config.TLSClientConfig)
var k kubernetesClient

k8s, err := kubernetes.NewForConfig(config)
Expand All @@ -84,6 +92,15 @@ func NewKubernetesClient(kubeconfig, apiserver string) (Client, error) {
return &k, nil
}

func NewKubernetesClientSkipTLSVerify(kubeconfig, apiserver string) (Client, error) {
return newKubernetesClient(kubeconfig, apiserver, true)
}

// NewKubernetesClient creates a KubernetesClient
func NewKubernetesClient(kubeconfig, apiserver string) (Client, error) {
return newKubernetesClient(kubeconfig, apiserver, false)
}

func NewKubernetesClientByConfig(config *rest.Config) (Client, error) {
config.QPS = 1e6
config.Burst = 1e6
Expand Down
2 changes: 1 addition & 1 deletion scripts/resolve-modules.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ echo "Resolving modules in $(pwd)"

PATHS=$(find $FIND_VAR -type f -name go.mod -printf '{"workdir":"%h"},')
if [[ $FIND_VAR == "." ]]; then
PATHS=$(find $FIND_VAR ! -path './controllers/*' ! -path './deprecated/*' ! -path './webhooks/*' ! -path './service/*' -type f -name go.mod -printf '{"workdir":"%h"},')
PATHS=$(find $FIND_VAR ! -path './controllers/*' ! -path './deprecated/*' ! -path './webhooks/*' ! -path './service/*' ! -path './test/*' -type f -name go.mod -printf '{"workdir":"%h"},')
fi
set -ex
echo matrix="{\"include\":[${PATHS%?}]}" >> $GITHUB_OUTPUT
Loading
Loading