♻️ Apply new internal/sync for continous benchmark (#2175)

Signed-off-by: vankichi <[email protected]>
vdaas · Sep 15, 2023 · ba17192 · ba17192
1 parent 6104f14
commit ba17192
Show file tree

Hide file tree

Showing 19 changed files with 84 additions and 237 deletions.
diff --git a/.github/workflows/dockers-benchmark-job-image.yml b/.github/workflows/dockers-benchmark-job-image.yml
@@ -17,117 +17,66 @@ name: "Build docker image: benchmark-job"
 on:
   push:
     branches:
-      - master
+      - main
     tags:
       - "*.*.*"
       - "v*.*.*"
       - "*.*.*-*"
       - "v*.*.*-*"
     paths:
       - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
       - ".github/workflows/dockers-benchmak-job-image.yml"
       - "go.mod"
       - "go.sum"
       - "internal/**"
       - "!internal/**/*_test.go"
       - "!internal/db/**"
-      - "!internal/k8s/**"
       - "apis/grpc/**"
+      - "pkg/benchmark/operator/**"
+      - "cmd/benchmark/operator/**"
       - "pkg/benchmark/job/**"
       - "cmd/benchmark/job/**"
-      - "dockers/benchmark/job/Dockerfile"
+      - "dockers/tools/benchmark/job/Dockerfile"
       - "versions/GO_VERSION"
-      - "versions/NGT_VERSION"
   pull_request:
     paths:
       - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
       - ".github/workflows/dockers-benchmak-job-image.yml"
       - "go.mod"
       - "go.sum"
       - "internal/**"
       - "!internal/**/*_test.go"
       - "!internal/db/**"
-      - "!internal/k8s/**"
       - "apis/grpc/**"
+      - "pkg/benchmark/operator/**"
+      - "cmd/benchmark/operator/**"
       - "pkg/benchmark/job/**"
       - "cmd/benchmark/job/**"
-      - "dockers/benchmark/job/Dockerfile"
+      - "dockers/tools/benchmark/job/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request_target:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-benchmak-job-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "apis/grpc/**"
+      - "pkg/benchmark/operator/**"
+      - "cmd/benchmark/operator/**"
+      - "pkg/benchmark/job/**"
+      - "cmd/benchmark/job/**"
+      - "dockers/tools/benchmark/job/Dockerfile"
       - "versions/GO_VERSION"
-      - "versions/NGT_VERSION"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }}
-  cancel-in-progress: true
 
 jobs:
   build:
-    strategy:
-      max-parallel: 4
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: all
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_PASS }}
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.PACKAGE_USER }}
-          password: ${{ secrets.PACKAGE_TOKEN }}
-      - name: Build and Publish
-        id: build_and_publish
-        uses: ./.github/actions/docker-build
-        with:
-          target: benchmark-job
-          builder: ${{ steps.buildx.outputs.name }}
-      - name: Initialize CodeQL
-        if: startsWith( github.ref, 'refs/tags/')
-        uses: github/codeql-action/init@v2
-      - name: Run vulnerability scanner (table)
-        if: startsWith( github.ref, 'refs/tags/')
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
-          format: "table"
-      - name: Run vulnerability scanner (sarif)
-        if: startsWith( github.ref, 'refs/tags/')
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
-          format: "template"
-          template: "@/contrib/sarif.tpl"
-          output: "trivy-results.sarif"
-      - name: Upload Trivy scan results to Security tab
-        if: startsWith( github.ref, 'refs/tags/')
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-  slack:
-    name: Slack notification
-    needs: build
-    runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/')
-    steps:
-      - uses: technote-space/workflow-conclusion-action@v2
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - uses: 8398a7/action-slack@v3
-        with:
-          author_name: benchmark-job image build
-          status: ${{ env.WORKFLOW_CONCLUSION }}
-          only_mention_fail: channel
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }}
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: benchmark-job
+    secrets: inherit
diff --git a/.github/workflows/dockers-benchmark-operator-image.yaml b/.github/workflows/dockers-benchmark-operator-image.yaml
@@ -17,30 +17,32 @@ name: "Build docker image: benchmark-operator"
 on:
   push:
     branches:
-      - master
+      - main
     tags:
       - "*.*.*"
       - "v*.*.*"
       - "*.*.*-*"
       - "v*.*.*-*"
     paths:
       - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
       - ".github/workflows/dockers-benchmak-operator-image.yml"
       - "go.mod"
       - "go.sum"
       - "internal/**"
       - "!internal/**/*_test.go"
       - "!internal/db/**"
-      - "!internal/k8s/**"
       - "apis/grpc/**"
       - "pkg/benchmark/operator/**"
       - "cmd/benchmark/operator/**"
-      - "dockers/benchmark/operator/Dockerfile"
+      - "pkg/benchmark/job/**"
+      - "cmd/benchmark/job/**"
+      - "dockers/tools/benchmark/operator/Dockerfile"
       - "versions/GO_VERSION"
-      - "versions/NGT_VERSION"
   pull_request:
     paths:
       - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
       - ".github/workflows/dockers-benchmak-operator-image.yml"
       - "go.mod"
       - "go.sum"
@@ -50,83 +52,31 @@ on:
       - "apis/grpc/**"
       - "pkg/benchmark/operator/**"
       - "cmd/benchmark/operator/**"
-      - "dockers/benchmark/operator/Dockerfile"
+      - "pkg/benchmark/job/**"
+      - "cmd/benchmark/job/**"
+      - "dockers/tools/benchmark/operator/Dockerfile"
+      - "versions/GO_VERSION"
+  pull_request_target:
+    paths:
+      - ".github/actions/docker-build/actions.yaml"
+      - ".github/workflows/_docker-image.yaml"
+      - ".github/workflows/dockers-benchmak-operator-image.yml"
+      - "go.mod"
+      - "go.sum"
+      - "internal/**"
+      - "!internal/**/*_test.go"
+      - "!internal/db/**"
+      - "apis/grpc/**"
+      - "pkg/benchmark/operator/**"
+      - "cmd/benchmark/operator/**"
+      - "pkg/benchmark/job/**"
+      - "cmd/benchmark/job/**"
+      - "dockers/tools/benchmark/operator/Dockerfile"
       - "versions/GO_VERSION"
-      - "versions/NGT_VERSION"
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref != 'refs/heads/main' && github.ref || github.sha }}-${{ github.event_name }}
-  cancel-in-progress: true
 
 jobs:
   build:
-    strategy:
-      max-parallel: 4
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: all
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      - name: Login to DockerHub
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKERHUB_USER }}
-          password: ${{ secrets.DOCKERHUB_PASS }}
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.PACKAGE_USER }}
-          password: ${{ secrets.PACKAGE_TOKEN }}
-      - name: Build and Publish
-        id: build_and_publish
-        uses: ./.github/actions/docker-build
-        with:
-          target: benchmark-operator
-          builder: ${{ steps.buildx.outputs.name }}
-      - name: Initialize CodeQL
-        if: startsWith( github.ref, 'refs/tags/')
-        uses: github/codeql-action/init@v2
-      - name: Run vulnerability scanner (table)
-        if: startsWith( github.ref, 'refs/tags/')
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
-          format: "table"
-      - name: Run vulnerability scanner (sarif)
-        if: startsWith( github.ref, 'refs/tags/')
-        uses: aquasecurity/trivy-action@master
-        with:
-          image-ref: "${{ steps.build_and_publish.outputs.IMAGE_NAME }}:${{ steps.build_and_publish.outputs.PRIMARY_TAG }}"
-          format: "template"
-          template: "@/contrib/sarif.tpl"
-          output: "trivy-results.sarif"
-      - name: Upload Trivy scan results to Security tab
-        if: startsWith( github.ref, 'refs/tags/')
-        uses: github/codeql-action/upload-sarif@v2
-        with:
-          sarif_file: "trivy-results.sarif"
-  slack:
-    name: Slack notification
-    needs: build
-    runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/main' || startsWith( github.ref, 'refs/tags/')
-    steps:
-      - uses: technote-space/workflow-conclusion-action@v2
-        with:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - uses: 8398a7/action-slack@v3
-        with:
-          author_name: benchmark-operator image build
-          status: ${{ env.WORKFLOW_CONCLUSION }}
-          only_mention_fail: channel
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_NOTIFY_WEBHOOK_URL }}
+    uses: ./.github/workflows/_docker-image.yaml
+    with:
+      target: benchmark-operator
+    secrets: inherit
diff --git a/internal/k8s/vald/benchmark/api/v1/job_types.go b/internal/k8s/vald/benchmark/api/v1/job_types.go
@@ -23,7 +23,7 @@ import (
 )
 
 type BenchmarkJobSpec struct {
-	*config.GlobalConfig    `json:",omitempty"              yaml:""`
+	*config.GlobalConfig    `json:",omitempty" yaml:""`
 	ServerConfig            *config.Servers            `json:"server_config,omitempty"              yaml:"server_config"`
 	Target                  *BenchmarkTarget           `json:"target,omitempty"                     yaml:"target"`
 	Dataset                 *BenchmarkDataset          `json:"dataset,omitempty"                    yaml:"dataset"`

diff --git a/pkg/tools/benchmark/job/router/option.go b/pkg/tools/benchmark/job/router/option.go
@@ -18,7 +18,7 @@
 package router
 
 import (
-	"github.com/vdaas/vald/internal/errgroup"
+	"github.com/vdaas/vald/internal/sync/errgroup"
 	"github.com/vdaas/vald/pkg/tools/benchmark/job/handler/rest"
 )
 

diff --git a/pkg/tools/benchmark/job/router/router.go b/pkg/tools/benchmark/job/router/router.go
@@ -20,9 +20,9 @@ package router
 import (
 	"net/http"
 
-	"github.com/vdaas/vald/internal/errgroup"
 	"github.com/vdaas/vald/internal/net/http/middleware"
 	"github.com/vdaas/vald/internal/net/http/routing"
+	"github.com/vdaas/vald/internal/sync/errgroup"
 	"github.com/vdaas/vald/pkg/tools/benchmark/job/handler/rest"
 )
 

diff --git a/pkg/tools/benchmark/job/service/insert.go b/pkg/tools/benchmark/job/service/insert.go
@@ -25,7 +25,7 @@ import (
 	"github.com/vdaas/vald/apis/grpc/v1/payload"
 	"github.com/vdaas/vald/internal/errors"
 	"github.com/vdaas/vald/internal/log"
-	"golang.org/x/sync/errgroup"
+	"github.com/vdaas/vald/internal/sync/errgroup"
 )
 
 func (j *job) insert(ctx context.Context, ech chan error) error {
@@ -38,10 +38,8 @@ func (j *job) insert(ctx context.Context, ech chan error) error {
 	if j.timestamp > int64(0) {
 		cfg.Timestamp = j.timestamp
 	}
-	eg, egctx := errgroup.WithContext(ctx)
+	eg, egctx := errgroup.New(ctx)
 	eg.SetLimit(j.concurrencyLimit)
-	// eg, egctx := errgroup.New(ctx)
-	// eg.Limitation(j.concurrencyLimit)
 	for i := j.dataset.Range.Start; i <= j.dataset.Range.End; i++ {
 		iter := i
 		eg.Go(func() error {
@@ -50,7 +48,6 @@ func (j *job) insert(ctx context.Context, ech chan error) error {
 			if err != nil {
 				log.Errorf("[benchmark job] limiter error is detected: %s", err.Error())
 				if errors.Is(err, context.Canceled) {
-					// return errors.Join(err, context.Canceled)
 					return nil
 				}
 				select {
@@ -74,9 +71,6 @@ func (j *job) insert(ctx context.Context, ech chan error) error {
 					log.Errorf("[benchmark job] context error is detected: %s\t%s", err.Error(), egctx.Err())
 					return errors.Join(err, egctx.Err())
 				default:
-					// if st, ok := status.FromError(err); ok {
-					// 	log.Warnf("[benchmark job] insert error is detected: code = %d, msg = %s", st.Code(), err.Error())
-					// }
 				}
 			}
 			// TODO: send metrics to the Prometeus

diff --git a/pkg/tools/benchmark/job/service/job.go b/pkg/tools/benchmark/job/service/job.go
@@ -28,13 +28,13 @@ import (
 	"github.com/vdaas/vald/apis/grpc/v1/payload"
 	"github.com/vdaas/vald/internal/client/v1/client/vald"
 	"github.com/vdaas/vald/internal/config"
-	"github.com/vdaas/vald/internal/errgroup"
 	"github.com/vdaas/vald/internal/errors"
 	"github.com/vdaas/vald/internal/k8s/client"
 	v1 "github.com/vdaas/vald/internal/k8s/vald/benchmark/api/v1"
 	"github.com/vdaas/vald/internal/log"
 	"github.com/vdaas/vald/internal/rand"
 	"github.com/vdaas/vald/internal/safety"
+	"github.com/vdaas/vald/internal/sync/errgroup"
 	"github.com/vdaas/vald/internal/test/data/hdf5"
 	"github.com/vdaas/vald/internal/timeutil/rate"
 )