diff --git a/.all-contributorsrc b/.all-contributorsrc index a750612cb3..f10efcf65c 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -194,6 +194,16 @@ "contributions": [ "doc" ] + }, + { + "login": "smorihira", + "name": "Shunya Morihira (森平 隼矢)", + "avatar_url": "https://avatars.githubusercontent.com/u/105629359?v=4", + "profile": "https://github.com/smorihira", + "contributions": [ + "tool", + "code" + ] } ], "contributorsPerLine": 7, diff --git a/.github/workflows/dockers-agent-faiss-image.yaml b/.github/workflows/dockers-agent-faiss-image.yaml new file mode 100755 index 0000000000..bda1be82ae --- /dev/null +++ b/.github/workflows/dockers-agent-faiss-image.yaml @@ -0,0 +1,82 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: agent-faiss' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/agent/core/faiss/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-agent-faiss-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/agent/core/faiss/** + - pkg/agent/core/faiss/** + - pkg/agent/internal/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - versions/FAISS_VERSION + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/agent/core/faiss/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-agent-faiss-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/agent/core/faiss/** + - pkg/agent/core/faiss/** + - pkg/agent/internal/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - versions/FAISS_VERSION + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: agent-faiss + secrets: inherit diff --git a/.github/workflows/dockers-agent-faiss-image.yml b/.github/workflows/dockers-agent-faiss-image.yml deleted file mode 100644 index d623609a56..0000000000 --- a/.github/workflows/dockers-agent-faiss-image.yml +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: agent-faiss" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-agent-faiss-image.yml" - - "Makefile" - - "apis/grpc/**" - - "cmd/agent/core/faiss/**" - - "dockers/agent/core/faiss/Dockerfile" - - "go.mod" - - "go.sum" - - "internal/**" - - "pkg/agent/core/faiss/**" - - "versions/FAISS_VERSION" - - "versions/GO_VERSION" - pull_request_target: - paths: - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-agent-faiss-image.yml" - - "Makefile" - - "apis/grpc/**" - - "cmd/agent/core/faiss/**" - - "dockers/agent/core/faiss/Dockerfile" - - "go.mod" - - "go.sum" - - "internal/**" - - "pkg/agent/core/faiss/**" - - "versions/FAISS_VERSION" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: agent-faiss - secrets: inherit diff --git a/.github/workflows/dockers-agent-image.yaml b/.github/workflows/dockers-agent-image.yaml new file mode 100755 index 0000000000..d6665fe250 --- /dev/null +++ b/.github/workflows/dockers-agent-image.yaml @@ -0,0 +1,84 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: agent' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/agent/core/agent/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-agent-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/agent/core/agent/** + - pkg/agent/core/agent/** + - apis/grpc/** + - apis/proto/** + - rust/Cargo.lock + - rust/Cargo.toml + - rust/bin/agent + - rust/libs/ngt-rs/** + - rust/libs/ngt/** + - rust/libs/proto/** + - versions/RUST_VERSION + - versions/FAISS_VERSION + - versions/NGT_VERSION + - versions/FAISS_VERSION + - versions/NGT_VERSION + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/agent/core/agent/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-agent-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/agent/core/agent/** + - pkg/agent/core/agent/** + - apis/grpc/** + - apis/proto/** + - rust/Cargo.lock + - rust/Cargo.toml + - rust/bin/agent + - rust/libs/ngt-rs/** + - rust/libs/ngt/** + - rust/libs/proto/** + - versions/RUST_VERSION + - versions/FAISS_VERSION + - versions/NGT_VERSION + - versions/FAISS_VERSION + - versions/NGT_VERSION + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: agent + secrets: inherit diff --git a/.github/workflows/dockers-agent-image.yml b/.github/workflows/dockers-agent-image.yml deleted file mode 100644 index 0aa4f2e98c..0000000000 --- a/.github/workflows/dockers-agent-image.yml +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: agent" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-agent-image.yml" - - "Makefile" - - "dockers/agent/core/agent/Dockerfile" - - "rust/Cargo.lock" - - "rust/Cargo.toml" - - "rust/bin/agent/**" - - "rust/libs/ngt-rs/**" - - "rust/libs/ngt/**" - - "rust/libs/proto/**" - - "versions/FAISS_VERSION" - - "versions/NGT_VERSION" - - "versions/RUST_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-agent-image.yml" - - "Makefile" - - "dockers/agent/core/agent/Dockerfile" - - "rust/Cargo.lock" - - "rust/Cargo.toml" - - "rust/bin/agent/**" - - "rust/libs/ngt-rs/**" - - "rust/libs/ngt/**" - - "rust/libs/proto/**" - - "versions/FAISS_VERSION" - - "versions/NGT_VERSION" - - "versions/RUST_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: agent - secrets: inherit diff --git a/.github/workflows/dockers-agent-ngt-image.yaml b/.github/workflows/dockers-agent-ngt-image.yaml new file mode 100755 index 0000000000..e2587cc568 --- /dev/null +++ b/.github/workflows/dockers-agent-ngt-image.yaml @@ -0,0 +1,80 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: agent-ngt' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/agent/core/ngt/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-agent-ngt-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/agent/core/ngt/** + - pkg/agent/core/ngt/** + - pkg/agent/internal/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - versions/NGT_VERSION + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/agent/core/ngt/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-agent-ngt-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/agent/core/ngt/** + - pkg/agent/core/ngt/** + - pkg/agent/internal/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - versions/NGT_VERSION + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: agent-ngt + secrets: inherit diff --git a/.github/workflows/dockers-agent-ngt-image.yml b/.github/workflows/dockers-agent-ngt-image.yml deleted file mode 100644 index 3f8ff939bc..0000000000 --- a/.github/workflows/dockers-agent-ngt-image.yml +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: agent-ngt" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-agent-ngt-image.yml" - - "Makefile" - - "apis/grpc/**" - - "cmd/agent/core/ngt/**" - - "dockers/agent/core/ngt/Dockerfile" - - "go.mod" - - "go.sum" - - "internal/**" - - "pkg/agent/core/ngt/**" - - "versions/NGT_VERSION" - - "versions/GO_VERSION" - pull_request_target: - paths: - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-agent-ngt-image.yml" - - "Makefile" - - "apis/grpc/**" - - "cmd/agent/core/ngt/**" - - "dockers/agent/core/ngt/Dockerfile" - - "go.mod" - - "go.sum" - - "internal/**" - - "pkg/agent/core/ngt/**" - - "versions/NGT_VERSION" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: agent-ngt - secrets: inherit diff --git a/.github/workflows/dockers-agent-sidecar-image.yaml b/.github/workflows/dockers-agent-sidecar-image.yaml new file mode 100755 index 0000000000..78456cab05 --- /dev/null +++ b/.github/workflows/dockers-agent-sidecar-image.yaml @@ -0,0 +1,80 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: agent-sidecar' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/agent/sidecar/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-agent-sidecar-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/agent/sidecar/** + - pkg/agent/sidecar/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - internal/db/storage/blob/** + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/agent/sidecar/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-agent-sidecar-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/agent/sidecar/** + - pkg/agent/sidecar/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - internal/db/storage/blob/** + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: agent-sidecar + secrets: inherit diff --git a/.github/workflows/dockers-agent-sidecar-image.yml b/.github/workflows/dockers-agent-sidecar-image.yml deleted file mode 100644 index 38ae9eeb3a..0000000000 --- a/.github/workflows/dockers-agent-sidecar-image.yml +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: agent-sidecar" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-agent-sidecar-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "internal/db/storage/blob/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/agent/sidecar/**" - - "cmd/agent/sidecar/**" - - "dockers/agent/sidecar/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-agent-sidecar-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "internal/db/storage/blob/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/agent/sidecar/**" - - "cmd/agent/sidecar/**" - - "dockers/agent/sidecar/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: agent-sidecar - secrets: inherit diff --git a/.github/workflows/dockers-benchmark-job-image.yaml b/.github/workflows/dockers-benchmark-job-image.yaml new file mode 100755 index 0000000000..6ff59e9816 --- /dev/null +++ b/.github/workflows/dockers-benchmark-job-image.yaml @@ -0,0 +1,80 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: benchmark-job' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/tools/benchmark/job/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-benchmark-job-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/tools/benchmark/job/** + - pkg/tools/benchmark/job/** + - cmd/tools/benchmark/operators/** + - pkg/tools/benchmark/operators/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/tools/benchmark/job/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-benchmark-job-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/tools/benchmark/job/** + - pkg/tools/benchmark/job/** + - cmd/tools/benchmark/operators/** + - pkg/tools/benchmark/operators/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: benchmark-job + secrets: inherit diff --git a/.github/workflows/dockers-benchmark-job-image.yml b/.github/workflows/dockers-benchmark-job-image.yml deleted file mode 100644 index 2d59b4356c..0000000000 --- a/.github/workflows/dockers-benchmark-job-image.yml +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: benchmark-job" -on: - push: - branches: - - main - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-benchmak-job-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "apis/grpc/**" - - "pkg/tools/benchmark/operator/**" - - "cmd/tools/benchmark/operator/**" - - "pkg/tools/benchmark/job/**" - - "cmd/tools/benchmark/job/**" - - "dockers/tools/benchmark/job/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-benchmak-job-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "apis/grpc/**" - - "pkg/tools/benchmark/operator/**" - - "cmd/tools/benchmark/operator/**" - - "pkg/tools/benchmark/job/**" - - "cmd/tools/benchmark/job/**" - - "dockers/tools/benchmark/job/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: benchmark-job - secrets: inherit diff --git a/.github/workflows/dockers-benchmark-operator-image.yaml b/.github/workflows/dockers-benchmark-operator-image.yaml old mode 100644 new mode 100755 index 2cc23d85b3..31c0f0f341 --- a/.github/workflows/dockers-benchmark-operator-image.yaml +++ b/.github/workflows/dockers-benchmark-operator-image.yaml @@ -13,50 +13,65 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Build docker image: benchmark-operator" +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: benchmark-operator' on: push: branches: - - main + - main + - release/v*.* + - '!release/v*.*.*' tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* pull_request: paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-benchmak-operator-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "apis/grpc/**" - - "pkg/tools/benchmark/operator/**" - - "cmd/tools/benchmark/operator/**" - - "pkg/tools/benchmark/job/**" - - "cmd/tools/benchmark/job/**" - - "dockers/tools/benchmark/operator/Dockerfile" - - "versions/GO_VERSION" + - hack/docker/gen/main.go + - dockers/tools/benchmark/operator/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-benchmark-operator-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/tools/benchmark/operator/** + - pkg/tools/benchmark/operator/** + - cmd/tools/benchmark/jobs/** + - pkg/tools/benchmark/jobs/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** pull_request_target: paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-benchmak-operator-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "apis/grpc/**" - - "pkg/tools/benchmark/operator/**" - - "cmd/tools/benchmark/operator/**" - - "pkg/tools/benchmark/job/**" - - "cmd/tools/benchmark/job/**" - - "dockers/tools/benchmark/operator/Dockerfile" - - "versions/GO_VERSION" + - hack/docker/gen/main.go + - dockers/tools/benchmark/operator/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-benchmark-operator-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/tools/benchmark/operator/** + - pkg/tools/benchmark/operator/** + - cmd/tools/benchmark/jobs/** + - pkg/tools/benchmark/jobs/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** jobs: build: uses: ./.github/workflows/_docker-image.yaml diff --git a/.github/workflows/dockers-binfmt-image.yaml b/.github/workflows/dockers-binfmt-image.yaml old mode 100644 new mode 100755 index 9a5d961c9d..4689f56333 --- a/.github/workflows/dockers-binfmt-image.yaml +++ b/.github/workflows/dockers-binfmt-image.yaml @@ -13,32 +13,61 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Build docker image: binfmt" +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: binfmt' on: schedule: - - cron: "0 * * * *" + - cron: 0 * * * * push: branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" + - main + - release/v*.* + - '!release/v*.*.*' tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* pull_request: paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-binfmt-image.yml" - - "dockers/binfmt/Dockerfile" + - hack/docker/gen/main.go + - dockers/binfmt/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-binfmt-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/binfmt/** + - pkg/binfmt/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' pull_request_target: paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-binfmt-image.yml" - - "dockers/binfmt/Dockerfile" + - hack/docker/gen/main.go + - dockers/binfmt/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-binfmt-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/binfmt/** + - pkg/binfmt/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' jobs: build: uses: ./.github/workflows/_docker-image.yaml diff --git a/.github/workflows/dockers-buildbase-image.yaml b/.github/workflows/dockers-buildbase-image.yaml new file mode 100755 index 0000000000..5117c243fc --- /dev/null +++ b/.github/workflows/dockers-buildbase-image.yaml @@ -0,0 +1,77 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: buildbase' +on: + schedule: + - cron: 0 * * * * + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/buildbase/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-buildbase-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/buildbase/** + - pkg/buildbase/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/buildbase/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-buildbase-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/buildbase/** + - pkg/buildbase/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: buildbase + platforms: linux/amd64,linux/arm64 + secrets: inherit diff --git a/.github/workflows/dockers-buildbase-image.yml b/.github/workflows/dockers-buildbase-image.yml deleted file mode 100644 index 6819dd7618..0000000000 --- a/.github/workflows/dockers-buildbase-image.yml +++ /dev/null @@ -1,48 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: buildbase" -on: - schedule: - - cron: "0 * * * *" - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-buildbase-image.yml" - - "dockers/buildbase/Dockerfile" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-buildbase-image.yml" - - "dockers/buildbase/Dockerfile" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: buildbase - platforms: linux/amd64,linux/arm64 - secrets: inherit diff --git a/.github/workflows/dockers-buildkit-image.yaml b/.github/workflows/dockers-buildkit-image.yaml old mode 100644 new mode 100755 index 8b202e1355..df7d955c0f --- a/.github/workflows/dockers-buildkit-image.yaml +++ b/.github/workflows/dockers-buildkit-image.yaml @@ -13,32 +13,61 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Build docker image: buildkit" +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: buildkit' on: schedule: - - cron: "0 * * * *" + - cron: 0 * * * * push: branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" + - main + - release/v*.* + - '!release/v*.*.*' tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* pull_request: paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-buildkit-image.yml" - - "dockers/buildkit/Dockerfile" + - hack/docker/gen/main.go + - dockers/buildkit/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-buildkit-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/buildkit/** + - pkg/buildkit/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' pull_request_target: paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-buildkit-image.yml" - - "dockers/buildkit/Dockerfile" + - hack/docker/gen/main.go + - dockers/buildkit/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-buildkit-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/buildkit/** + - pkg/buildkit/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' jobs: build: uses: ./.github/workflows/_docker-image.yaml diff --git a/.github/workflows/dockers-buildkit-syft-scanner-image.yaml b/.github/workflows/dockers-buildkit-syft-scanner-image.yaml old mode 100644 new mode 100755 index 0476cfdfb9..b74ee0acfb --- a/.github/workflows/dockers-buildkit-syft-scanner-image.yaml +++ b/.github/workflows/dockers-buildkit-syft-scanner-image.yaml @@ -13,32 +13,61 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Build docker image: buildkit-syft-scanner" +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: buildkit-syft-scanner' on: schedule: - - cron: "0 * * * *" + - cron: 0 * * * * push: branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" + - main + - release/v*.* + - '!release/v*.*.*' tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* pull_request: paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-buildkit-syft-scanner-image.yml" - - "dockers/buildkit/syft/scanner/Dockerfile" + - hack/docker/gen/main.go + - dockers/buildkit/syft/scanner/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-buildkit-syft-scanner-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/buildkit/syft/scanner/** + - pkg/buildkit/syft/scanner/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' pull_request_target: paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-buildkit-syft-scanner-image.yml" - - "dockers/buildkit/syft/scanner/Dockerfile" + - hack/docker/gen/main.go + - dockers/buildkit/syft/scanner/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-buildkit-syft-scanner-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/buildkit/syft/scanner/** + - pkg/buildkit/syft/scanner/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' jobs: build: uses: ./.github/workflows/_docker-image.yaml diff --git a/.github/workflows/dockers-ci-container-image.yaml b/.github/workflows/dockers-ci-container-image.yaml new file mode 100755 index 0000000000..fa6ffb3943 --- /dev/null +++ b/.github/workflows/dockers-ci-container-image.yaml @@ -0,0 +1,65 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: ci-container' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/ci/base/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-ci-container-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/ci/base/** + - pkg/ci/base/** + - apis/grpc/** + - apis/proto/** + - hack/** + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/ci/base/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-ci-container-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/ci/base/** + - pkg/ci/base/** + - apis/grpc/** + - apis/proto/** + - hack/** + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: ci-container + platforms: linux/amd64 + secrets: inherit diff --git a/.github/workflows/dockers-ci-container-image.yml b/.github/workflows/dockers-ci-container-image.yml deleted file mode 100644 index 2e0b034768..0000000000 --- a/.github/workflows/dockers-ci-container-image.yml +++ /dev/null @@ -1,54 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: ci-container" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-ci-container-image.yml" - - "dockers/ci/**" - - "Makefile" - - "Makefile.d/**" - - "versions/GO_VERSION" - - "versions/NGT_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-ci-container-image.yml" - - "dockers/ci/**" - - "Makefile" - - "Makefile.d/**" - - "versions/GO_VERSION" - - "versions/NGT_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: ci-container - platforms: linux/amd64 - secrets: inherit diff --git a/.github/workflows/dockers-dev-container-image.yaml b/.github/workflows/dockers-dev-container-image.yaml new file mode 100755 index 0000000000..ee83a872c4 --- /dev/null +++ b/.github/workflows/dockers-dev-container-image.yaml @@ -0,0 +1,64 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: dev-container' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/dev/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-dev-container-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/dev/** + - pkg/dev/** + - apis/grpc/** + - apis/proto/** + - hack/** + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/dev/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-dev-container-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/dev/** + - pkg/dev/** + - apis/grpc/** + - apis/proto/** + - hack/** + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: dev-container + secrets: inherit diff --git a/.github/workflows/dockers-dev-container-image.yml b/.github/workflows/dockers-dev-container-image.yml deleted file mode 100644 index f9eae6c6ae..0000000000 --- a/.github/workflows/dockers-dev-container-image.yml +++ /dev/null @@ -1,49 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: dev-container" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-dev-container-image.yml" - - "dockers/dev/**" - - "Makefile" - - "Makefile.d/**" - - "versions/GO_VERSION" - - "versions/NGT_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-dev-container-image.yml" - - "dockers/dev/**" - - "Makefile" - - "Makefile.d/**" - - "versions/GO_VERSION" - - "versions/NGT_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: dev-container - platforms: linux/amd64 - secrets: inherit diff --git a/.github/workflows/dockers-discoverer-k8s-image.yaml b/.github/workflows/dockers-discoverer-k8s-image.yaml new file mode 100755 index 0000000000..e0fea36ae9 --- /dev/null +++ b/.github/workflows/dockers-discoverer-k8s-image.yaml @@ -0,0 +1,76 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: discoverer-k8s' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/discoverer/k8s/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-discoverer-k8s-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/discoverer/k8s/** + - pkg/discoverer/k8s/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/discoverer/k8s/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-discoverer-k8s-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/discoverer/k8s/** + - pkg/discoverer/k8s/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: discoverer-k8s + secrets: inherit diff --git a/.github/workflows/dockers-discoverer-k8s-image.yml b/.github/workflows/dockers-discoverer-k8s-image.yml deleted file mode 100644 index 6612d9cf2d..0000000000 --- a/.github/workflows/dockers-discoverer-k8s-image.yml +++ /dev/null @@ -1,63 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: discoverer-k8s" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-discoverer-k8s-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "apis/grpc/**" - - "pkg/discoverer/k8s/**" - - "cmd/discoverer/k8s/**" - - "dockers/discoverer/k8s/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-discoverer-k8s-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "apis/grpc/**" - - "pkg/discoverer/k8s/**" - - "cmd/discoverer/k8s/**" - - "dockers/discoverer/k8s/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: discoverer-k8s - secrets: inherit diff --git a/.github/workflows/dockers-gateway-filter-image.yaml b/.github/workflows/dockers-gateway-filter-image.yaml new file mode 100755 index 0000000000..185821dfd2 --- /dev/null +++ b/.github/workflows/dockers-gateway-filter-image.yaml @@ -0,0 +1,80 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: gateway-filter' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/gateway/filter/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-gateway-filter-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/gateway/filter/** + - pkg/gateway/filter/** + - pkg/gateway/internal/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/gateway/filter/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-gateway-filter-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/gateway/filter/** + - pkg/gateway/filter/** + - pkg/gateway/internal/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: gateway-filter + secrets: inherit diff --git a/.github/workflows/dockers-gateway-filter-image.yml b/.github/workflows/dockers-gateway-filter-image.yml deleted file mode 100644 index 94d9754912..0000000000 --- a/.github/workflows/dockers-gateway-filter-image.yml +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: gateway-filter" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-gateway-filter-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/**/*_mock.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/gateway/filter/**" - - "cmd/gateway/filter/**" - - "pkg/gateway/internal/**" - - "dockers/gateway/filter/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-gateway-filter-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/**/*_mock.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/gateway/filter/**" - - "cmd/gateway/filter/**" - - "pkg/gateway/internal/**" - - "dockers/gateway/filter/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: gateway-filter - secrets: inherit diff --git a/.github/workflows/dockers-gateway-lb-image.yaml b/.github/workflows/dockers-gateway-lb-image.yaml new file mode 100755 index 0000000000..ffbe94fd42 --- /dev/null +++ b/.github/workflows/dockers-gateway-lb-image.yaml @@ -0,0 +1,80 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: gateway-lb' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/gateway/lb/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-gateway-lb-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/gateway/lb/** + - pkg/gateway/lb/** + - pkg/gateway/internal/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/gateway/lb/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-gateway-lb-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/gateway/lb/** + - pkg/gateway/lb/** + - pkg/gateway/internal/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: gateway-lb + secrets: inherit diff --git a/.github/workflows/dockers-gateway-lb-image.yml b/.github/workflows/dockers-gateway-lb-image.yml deleted file mode 100644 index a438eb29bf..0000000000 --- a/.github/workflows/dockers-gateway-lb-image.yml +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: gateway-lb" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-gateway-lb-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/**/*_mock.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/gateway/lb/**" - - "cmd/gateway/lb/**" - - "pkg/gateway/internal/**" - - "dockers/gateway/lb/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-gateway-lb-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/**/*_mock.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/gateway/lb/**" - - "cmd/gateway/lb/**" - - "pkg/gateway/internal/**" - - "dockers/gateway/lb/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: gateway-lb - secrets: inherit diff --git a/.github/workflows/dockers-gateway-mirror-image.yaml b/.github/workflows/dockers-gateway-mirror-image.yaml old mode 100644 new mode 100755 index aeddaa45f0..29e2f55f35 --- a/.github/workflows/dockers-gateway-mirror-image.yaml +++ b/.github/workflows/dockers-gateway-mirror-image.yaml @@ -13,50 +13,63 @@ # See the License for the specific language governing permissions and # limitations under the License. # -name: "Build docker image: gateway-mirror" +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: gateway-mirror' on: push: branches: - - main + - main + - release/v*.* + - '!release/v*.*.*' tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* pull_request: paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/dockers-gateway-mirror-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/**/*_mock.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/gateway/mirror/**" - - "cmd/gateway/mirror/**" - - "pkg/gateway/internal/**" - - "dockers/gateway/mirror/Dockerfile" - - "versions/GO_VERSION" + - hack/docker/gen/main.go + - dockers/gateway/mirror/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-gateway-mirror-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/gateway/mirror/** + - pkg/gateway/mirror/** + - pkg/gateway/internal/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** pull_request_target: paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/dockers-gateway-mirror-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/**/*_mock.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/gateway/mirror/**" - - "cmd/gateway/nirror/**" - - "pkg/gateway/internal/**" - - "dockers/gateway/mirror/Dockerfile" - - "versions/GO_VERSION" + - hack/docker/gen/main.go + - dockers/gateway/mirror/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-gateway-mirror-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/gateway/mirror/** + - pkg/gateway/mirror/** + - pkg/gateway/internal/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** jobs: build: uses: ./.github/workflows/_docker-image.yaml diff --git a/.github/workflows/dockers-helm-operator-image.yaml b/.github/workflows/dockers-helm-operator-image.yaml new file mode 100755 index 0000000000..ec1e04537a --- /dev/null +++ b/.github/workflows/dockers-helm-operator-image.yaml @@ -0,0 +1,72 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: helm-operator' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/operator/helm/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-helm-operator-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/operator/helm/** + - pkg/operator/helm/** + - charts/vald/Chart.yaml + - charts/vald/values.yaml + - charts/vald/templates/** + - charts/vald-helm-operator/Chart.yaml + - charts/vald-helm-operator/values.yaml + - charts/vald-helm-operator/templates/** + - versions/OPERATOR_SDK_VERSION + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/operator/helm/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-helm-operator-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/operator/helm/** + - pkg/operator/helm/** + - charts/vald/Chart.yaml + - charts/vald/values.yaml + - charts/vald/templates/** + - charts/vald-helm-operator/Chart.yaml + - charts/vald-helm-operator/values.yaml + - charts/vald-helm-operator/templates/** + - versions/OPERATOR_SDK_VERSION + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: helm-operator + secrets: inherit diff --git a/.github/workflows/dockers-helm-operator-image.yml b/.github/workflows/dockers-helm-operator-image.yml deleted file mode 100644 index 3b36c307f7..0000000000 --- a/.github/workflows/dockers-helm-operator-image.yml +++ /dev/null @@ -1,59 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: helm-operator" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-helm-operator-image.yml" - - "dockers/operator/helm/Dockerfile" - - "charts/vald/Chart.yaml" - - "charts/vald/values.yaml" - - "charts/vald/templates/**" - - "charts/vald-helm-operator/Chart.yaml" - - "charts/vald-helm-operator/values.yaml" - - "charts/vald-helm-operator/templates/**" - - "versions/OPERATOR_SDK_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-helm-operator-image.yml" - - "dockers/operator/helm/Dockerfile" - - "charts/vald/Chart.yaml" - - "charts/vald/values.yaml" - - "charts/vald/templates/**" - - "charts/vald-helm-operator/Chart.yaml" - - "charts/vald-helm-operator/values.yaml" - - "charts/vald-helm-operator/templates/**" - - "versions/OPERATOR_SDK_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: operator/helm - secrets: inherit diff --git a/.github/workflows/dockers-index-correction-image.yaml b/.github/workflows/dockers-index-correction-image.yaml new file mode 100755 index 0000000000..ed4060a8c6 --- /dev/null +++ b/.github/workflows/dockers-index-correction-image.yaml @@ -0,0 +1,78 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: index-correction' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/index/job/correction/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-correction-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/correction/** + - pkg/index/job/correction/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/index/job/correction/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-correction-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/correction/** + - pkg/index/job/correction/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: index-correction + secrets: inherit diff --git a/.github/workflows/dockers-index-correction.yml b/.github/workflows/dockers-index-correction.yml deleted file mode 100644 index c1bb1b8a29..0000000000 --- a/.github/workflows/dockers-index-correction.yml +++ /dev/null @@ -1,63 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: index-correction" -on: - push: - branches: - - main - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-index-correction.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/index/job/correction/**" - - "cmd/index/job/correction/**" - - "dockers/index/job/correction/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-index-correction.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/index/job/correction/**" - - "cmd/index/job/correction/**" - - "dockers/index/job/correction/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: index-correction - secrets: inherit diff --git a/.github/workflows/dockers-index-creation-image.yaml b/.github/workflows/dockers-index-creation-image.yaml new file mode 100755 index 0000000000..1d17b7d10f --- /dev/null +++ b/.github/workflows/dockers-index-creation-image.yaml @@ -0,0 +1,78 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: index-creation' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/index/job/creation/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-creation-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/creation/** + - pkg/index/job/creation/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/index/job/creation/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-creation-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/creation/** + - pkg/index/job/creation/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: index-creation + secrets: inherit diff --git a/.github/workflows/dockers-index-creation.yml b/.github/workflows/dockers-index-creation.yml deleted file mode 100644 index 531e16aedf..0000000000 --- a/.github/workflows/dockers-index-creation.yml +++ /dev/null @@ -1,63 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: index-creation" -on: - push: - branches: - - main - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-index-creation.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/index/job/creation/**" - - "cmd/index/job/creation/**" - - "dockers/index/job/creation/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-index-creation.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/index/job/creation/**" - - "cmd/index/job/creation/**" - - "dockers/index/job/creation/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: index-creation - secrets: inherit diff --git a/.github/workflows/dockers-index-operator-image.yaml b/.github/workflows/dockers-index-operator-image.yaml new file mode 100755 index 0000000000..f83b0f6c86 --- /dev/null +++ b/.github/workflows/dockers-index-operator-image.yaml @@ -0,0 +1,76 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: index-operator' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/index/operator/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-operator-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/operator/** + - pkg/index/operator/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/index/operator/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-operator-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/operator/** + - pkg/index/operator/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: index-operator + secrets: inherit diff --git a/.github/workflows/dockers-index-operator.yml b/.github/workflows/dockers-index-operator.yml deleted file mode 100644 index fcc748d63e..0000000000 --- a/.github/workflows/dockers-index-operator.yml +++ /dev/null @@ -1,63 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: index-operator" -on: - push: - branches: - - main - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-index-operator.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "internal/k8s/**" - - "apis/grpc/**" - - "pkg/index/operator/**" - - "cmd/index/operator/**" - - "dockers/index/operator/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-index-operator.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "internal/k8s/**" - - "apis/grpc/**" - - "pkg/index/operator/**" - - "cmd/index/operator/**" - - "dockers/index/operator/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: index-operator - secrets: inherit diff --git a/.github/workflows/dockers-index-save-image.yaml b/.github/workflows/dockers-index-save-image.yaml new file mode 100755 index 0000000000..568fbcb0b6 --- /dev/null +++ b/.github/workflows/dockers-index-save-image.yaml @@ -0,0 +1,78 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: index-save' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/index/job/save/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-save-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/save/** + - pkg/index/job/save/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/index/job/save/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-index-save-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/save/** + - pkg/index/job/save/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: index-save + secrets: inherit diff --git a/.github/workflows/dockers-index-save.yml b/.github/workflows/dockers-index-save.yml deleted file mode 100644 index 4669b534bf..0000000000 --- a/.github/workflows/dockers-index-save.yml +++ /dev/null @@ -1,63 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: index-save" -on: - push: - branches: - - main - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-index-save.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/index/job/save/**" - - "cmd/index/job/save/**" - - "dockers/index/job/save/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-index-save.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/index/job/save/**" - - "cmd/index/job/save/**" - - "dockers/index/job/save/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: index-save - secrets: inherit diff --git a/.github/workflows/dockers-loadtest-image.yaml b/.github/workflows/dockers-loadtest-image.yaml new file mode 100755 index 0000000000..6e561548bd --- /dev/null +++ b/.github/workflows/dockers-loadtest-image.yaml @@ -0,0 +1,79 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: loadtest' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/tools/cli/loadtest/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-loadtest-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/tools/cli/loadtest/** + - pkg/tools/cli/loadtest/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/tools/cli/loadtest/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-loadtest-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/tools/cli/loadtest/** + - pkg/tools/cli/loadtest/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: loadtest + platforms: linux/amd64 + secrets: inherit diff --git a/.github/workflows/dockers-loadtest-image.yml b/.github/workflows/dockers-loadtest-image.yml deleted file mode 100644 index 75e84f4cd2..0000000000 --- a/.github/workflows/dockers-loadtest-image.yml +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: loadtest" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-loadtest-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/tools/cli/loadtest/**" - - "cmd/tools/cli/loadtest/**" - - "dockers/tools/cli/loadtest/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-loadtest-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/tools/cli/loadtest/**" - - "cmd/tools/cli/loadtest/**" - - "dockers/tools/cli/loadtest/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: loadtest - platforms: linux/amd64 - secrets: inherit diff --git a/.github/workflows/dockers-manager-index-image.yaml b/.github/workflows/dockers-manager-index-image.yaml new file mode 100755 index 0000000000..d7a22972b7 --- /dev/null +++ b/.github/workflows/dockers-manager-index-image.yaml @@ -0,0 +1,78 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: manager-index' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/manager/index/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-manager-index-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/manager/index/** + - pkg/manager/index/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/manager/index/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-manager-index-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/manager/index/** + - pkg/manager/index/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - '!internal/k8s/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: manager-index + secrets: inherit diff --git a/.github/workflows/dockers-manager-index-image.yml b/.github/workflows/dockers-manager-index-image.yml deleted file mode 100644 index 774979664c..0000000000 --- a/.github/workflows/dockers-manager-index-image.yml +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: manager-index" -on: - push: - branches: - - "main" - - "release/v*.*" - - "!release/v*.*.*" - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-manager-index-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/manager/index/**" - - "cmd/manager/index/**" - - "dockers/manager/index/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-manager-index-image.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "!internal/k8s/**" - - "apis/grpc/**" - - "pkg/manager/index/**" - - "cmd/manager/index/**" - - "dockers/manager/index/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: manager-index - secrets: inherit diff --git a/.github/workflows/dockers-readreplica-rotate-image.yaml b/.github/workflows/dockers-readreplica-rotate-image.yaml new file mode 100755 index 0000000000..fd18858755 --- /dev/null +++ b/.github/workflows/dockers-readreplica-rotate-image.yaml @@ -0,0 +1,76 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +name: 'Build docker image: readreplica-rotate' +on: + push: + branches: + - main + - release/v*.* + - '!release/v*.*.*' + tags: + - '*.*.*' + - v*.*.* + - '*.*.*-*' + - v*.*.*-* + pull_request: + paths: + - hack/docker/gen/main.go + - dockers/index/job/readreplica/rotate/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-readreplica-rotate-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/readreplica/rotate/** + - pkg/index/job/readreplica/rotate/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** + pull_request_target: + paths: + - hack/docker/gen/main.go + - dockers/index/job/readreplica/rotate/Dockerfile + - hack/actions/gen/main.go + - .github/workflows/dockers-readreplica-rotate-image.yaml + - .github/actions/docker-build/action.yaml + - .github/workflows/_docker-image.yaml + - cmd/index/job/readreplica/rotate/** + - pkg/index/job/readreplica/rotate/** + - apis/grpc/** + - apis/proto/** + - go.mod + - go.sum + - versions/GO_VERSION + - internal/** + - '!internal/**/*_test.go' + - '!internal/**/*_mock.go' + - '!internal/db/**' + - Makefile + - Makefile.d/** +jobs: + build: + uses: ./.github/workflows/_docker-image.yaml + with: + target: readreplica-rotate + secrets: inherit diff --git a/.github/workflows/dockers-readreplica-rotate.yml b/.github/workflows/dockers-readreplica-rotate.yml deleted file mode 100644 index 724e88a3f1..0000000000 --- a/.github/workflows/dockers-readreplica-rotate.yml +++ /dev/null @@ -1,63 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Build docker image: readreplica-rotate" -on: - push: - branches: - - main - tags: - - "*.*.*" - - "v*.*.*" - - "*.*.*-*" - - "v*.*.*-*" - pull_request: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-readreplica-rotate.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "internal/k8s/**" - - "apis/grpc/**" - - "pkg/index/job/readreplica/rotate/**" - - "cmd/index/job/readreplica/rotate/**" - - "dockers/index/job/readreplica/rotate/Dockerfile" - - "versions/GO_VERSION" - pull_request_target: - paths: - - ".github/actions/docker-build/action.yaml" - - ".github/workflows/_docker-image.yaml" - - ".github/workflows/dockers-readreplica-rotate.yml" - - "go.mod" - - "go.sum" - - "internal/**" - - "!internal/**/*_test.go" - - "!internal/db/**" - - "internal/k8s/**" - - "apis/grpc/**" - - "pkg/index/job/readreplica/rotate/**" - - "cmd/index/job/readreplica/rotate/**" - - "dockers/index/job/readreplica/rotate/Dockerfile" - - "versions/GO_VERSION" -jobs: - build: - uses: ./.github/workflows/_docker-image.yaml - with: - target: readreplica-rotate - secrets: inherit diff --git a/Makefile b/Makefile index 4a5e001f71..42b9302bfc 100644 --- a/Makefile +++ b/Makefile @@ -442,6 +442,11 @@ license: dockerfile: $(call gen-dockerfile,$(ROOTDIR),$(MAINTAINER)) +.PHONY: workflow +## generate workflows +workflow: + $(call gen-workflow,$(ROOTDIR),$(MAINTAINER)) + .PHONY: init ## initialize development environment init: \ diff --git a/Makefile.d/docker.mk b/Makefile.d/docker.mk index 858bd69e72..9a7ddd8a8a 100644 --- a/Makefile.d/docker.mk +++ b/Makefile.d/docker.mk @@ -38,7 +38,7 @@ docker/build: \ docker/build/index-save \ docker/build/loadtest \ docker/build/manager-index \ - docker/build/operator/helm \ + docker/build/helm-operator \ docker/build/readreplica-rotate .PHONY: docker/name/org @@ -255,13 +255,13 @@ docker/build/dev-container: IMAGE=$(DEV_CONTAINER_IMAGE) \ docker/build/image -.PHONY: docker/name/operator/helm -docker/name/operator/helm: +.PHONY: docker/name/helm-operator +docker/name/helm-operator: @echo "$(ORG)/$(HELM_OPERATOR_IMAGE)" -.PHONY: docker/build/operator/helm +.PHONY: docker/build/helm-operator ## build helm-operator image -docker/build/operator/helm: +docker/build/helm-operator: @make DOCKERFILE="$(ROOTDIR)/dockers/operator/helm/Dockerfile" \ IMAGE=$(HELM_OPERATOR_IMAGE) \ EXTRA_ARGS="--build-arg OPERATOR_SDK_VERSION=$(OPERATOR_SDK_VERSION) --build-arg UPX_OPTIONS=$(UPX_OPTIONS) $(EXTRA_ARGS)" \ diff --git a/Makefile.d/functions.mk b/Makefile.d/functions.mk index 8267ef416c..1e97c315d8 100644 --- a/Makefile.d/functions.mk +++ b/Makefile.d/functions.mk @@ -326,6 +326,23 @@ define gen-dockerfile rm -rf $$BIN_PATH endef +define gen-workflow + BIN_PATH="$(TEMP_DIR)/vald-actions-gen"; \ + rm -rf $$BIN_PATH; \ + MAINTAINER=$2 \ + GOPRIVATE=$(GOPRIVATE) \ + GOARCH=$(GOARCH) \ + GOOS=$(GOOS) \ + go build -modcacherw \ + -mod=readonly \ + -a \ + -tags "osusergo netgo static_build" \ + -trimpath \ + -o $$BIN_PATH $(ROOTDIR)/hack/actions/gen/main.go; \ + $$BIN_PATH $1; \ + rm -rf $$BIN_PATH +endef + define gen-vald-helm-schema BIN_PATH="$(TEMP_DIR)/vald-helm-schema-gen"; \ rm -rf $$BIN_PATH; \ diff --git a/Makefile.d/k8s.mk b/Makefile.d/k8s.mk index 628fac13b5..5da9210129 100644 --- a/Makefile.d/k8s.mk +++ b/Makefile.d/k8s.mk @@ -41,6 +41,7 @@ k8s/manifest/update: \ --set manager.index.saver.enabled=true \ --set manager.index.creator.enabled=true \ --set manager.index.corrector.enabled=true \ + --set gateway.mirror.enabled=true \ --output-dir $(TEMP_DIR) \ charts/vald mkdir -p k8s/gateway diff --git a/README.md b/README.md index a03458bf5b..94874a6aa8 100755 --- a/README.md +++ b/README.md @@ -265,7 +265,7 @@ make init -[![All Contributors](https://img.shields.io/badge/all_contributors-18-orange.svg?style=flat-square)](#contributors) +[![All Contributors](https://img.shields.io/badge/all_contributors-19-orange.svg?style=flat-square)](#contributors) @@ -299,6 +299,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d Yusuke Kadowaki
Yusuke Kadowaki

💻 ⚠️ aknishid
aknishid

💻 🚧 📖 Hrichik Mazumder
Hrichik Mazumder

📖 + Shunya Morihira (森平 隼矢)
Shunya Morihira (森平 隼矢)

🔧 💻 diff --git a/docs/api/insert.md b/docs/api/insert.md index ef9e2dd43f..eae2bd7dce 100644 --- a/docs/api/insert.md +++ b/docs/api/insert.md @@ -100,7 +100,7 @@ Inset RPC is the method to add a new single vector. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -226,7 +226,7 @@ It's the recommended method to insert a large number of vectors. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -349,7 +349,7 @@ Please be careful that the size of the request exceeds the limit. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting diff --git a/docs/api/object.md b/docs/api/object.md index ca4b0e89c2..a9e37ea9d0 100644 --- a/docs/api/object.md +++ b/docs/api/object.md @@ -67,7 +67,7 @@ Exists RPC is the method to check that a vector exists in the `vald-agent`. | 5 | NOT_FOUND | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -148,7 +148,7 @@ GetObject RPC is the method to get the metadata of a vector inserted into the `v | 5 | NOT_FOUND | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -242,7 +242,7 @@ Each Upsert request and response are independent. | 5 | NOT_FOUND | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting diff --git a/docs/api/remove.md b/docs/api/remove.md index 156176fc70..2846ea08ee 100644 --- a/docs/api/remove.md +++ b/docs/api/remove.md @@ -101,7 +101,7 @@ Remove RPC is the method to remove a single vector. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -222,7 +222,7 @@ RemoveByTimestamp RPC is the method to remove vectors based on timestamp. | 5 | NOT_FOUND | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -344,7 +344,7 @@ It's the recommended method to remove a large number of vectors. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -465,7 +465,7 @@ Please be careful that the size of the request exceeds the limit. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting diff --git a/docs/api/search.md b/docs/api/search.md index dab76e0cec..ccec2863d0 100644 --- a/docs/api/search.md +++ b/docs/api/search.md @@ -146,7 +146,7 @@ Search RPC is the method to search vector(s) similar to the request vector. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -267,7 +267,7 @@ The vector with the same requested ID should be indexed into the `vald-agent` be | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -403,7 +403,7 @@ Each Search request and response are independent. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -539,7 +539,7 @@ Each SearchByID request and response are independent. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -684,7 +684,7 @@ Please be careful that the size of the request exceeds the limit. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -830,7 +830,7 @@ Please be careful that the size of the request exceeds the limit. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -946,7 +946,7 @@ LinearSearch RPC is the method to linear search vector(s) similar to the request | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -1064,7 +1064,7 @@ You will get a `NOT_FOUND` error if the vector isn't stored. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -1196,7 +1196,7 @@ Each LinearSearch request and response are independent. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -1328,7 +1328,7 @@ Each LinearSearchByID request and response are independent. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -1469,7 +1469,7 @@ Please be careful that the size of the request exceeds the limit. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -1611,7 +1611,7 @@ Please be careful that the size of the request exceeds the limit. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting diff --git a/docs/api/update.md b/docs/api/update.md index f1517b2a64..2fb68b05f7 100644 --- a/docs/api/update.md +++ b/docs/api/update.md @@ -102,7 +102,7 @@ Update RPC is the method to update a single vector. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -231,7 +231,7 @@ It's the recommended method to update the large amount of vectors. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -357,7 +357,7 @@ Please be careful that the size of the request exceeds the limit. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting diff --git a/docs/api/upsert.md b/docs/api/upsert.md index bd3128419d..9ed9f6572c 100644 --- a/docs/api/upsert.md +++ b/docs/api/upsert.md @@ -105,7 +105,7 @@ Upsert RPC is the method to update the inserted vector to a new single vector or | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -232,7 +232,7 @@ It’s the recommended method to upsert a large number of vectors. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting @@ -356,7 +356,7 @@ Please be careful that the size of the request exceeds the limit. | 10 | ABORTED | | 13 | INTERNAL | -Please refer to [Response Status Code](./status.md) for more details. +Please refer to [Response Status Code](../status.md) for more details. ### Troubleshooting diff --git a/docs/contributing/coding-style.md b/docs/contributing/coding-style.md index ffd67cef7c..c5bb690079 100644 --- a/docs/contributing/coding-style.md +++ b/docs/contributing/coding-style.md @@ -1407,4 +1407,4 @@ Since each package has its purpose, we decided to apply different strategies to For the rest of the `./pkg` packages, we decided to implement the unit test for the exported function only. -Please follow the [unit test guideline](./unit-test-guideline.md) for more details on how to implement good unit test. +Please follow the [unit test guideline](../unit-test-guideline.md) for more details on how to implement good unit test. diff --git a/docs/performance/continuous-benchmark.md b/docs/performance/continuous-benchmark.md index ef683b1d5d..b63e12d316 100644 --- a/docs/performance/continuous-benchmark.md +++ b/docs/performance/continuous-benchmark.md @@ -45,24 +45,24 @@ And, Benchmark Operator also applies it to the Kubernetes cluster based on `Vald **main properties** -| Name | mandatory | Description | type | sample | -| :------------------------- | :-------- | :-------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------- | :------------------------------------------------------------------------------------------- | -| target | \* | target Vald cluster | object | ref: [target](#target-prop) | -| dataset | \* | dataset information | object | ref: [dataset](#dataset-prop) | -| job_type | \* | execute job type | string enum: [insert, update, upsert, remove, search, getobject, exists] | search | -| repetition | | the number of job repetitions
default: `1` | integer | 1 | -| replica | | the number of job concurrent job executions
default: `1` | integer | 2 | -| rps | | designed request per sec to the target cluster
default: `1000` | integer | 1000 | -| concurrency_limit | | goroutine count limit for rps adjustment
default: `200` | integer | 20 | -| ttl_seconds_after_finished | | time until deletion of Pod after job end
default: `600` | integer | 120 | -| insert_config | | request config for insert job | object | ref: [config](#insert-cfg-props) | -| update_config | | request config for update job | object | ref: [config](#update-cfg-props) | -| upsert_config | | request config for upsert job | object | ref: [config](#upsert-cfg-props) | -| search_config | | request config for search job | object | ref: [config](#search-cfg-props) | -| remove_config | | request config for remove job | object | ref: [config](#remove-cfg-props) | -| object_config | | request config for object job | object | ref: [config](#object-cfg-props) | -| client_config | | gRPC client config for running benchmark job
Tune if can not getting the expected performance with default config. | object | ref: [defaults.grpc](https://github.com/vdaas/vald/blob/main/charts/vald/README.md) | -| server_config | | server config for benchmark job pod
Tune if can not getting the expected performance with default config. | object | ref: [defaults.server_config](https://github.com/vdaas/vald/blob/main/charts/vald/README.md) | +| Name | mandatory | Description | type | sample | +| :------------------------- | :-------- | :-------------------------------------------------------------------------------------------------------------------- | :----------------------------------------------------------------------- | :--------------------------------------------------------------------------------------------- | +| target | \* | target Vald cluster | object | ref: [target](#target-prop) | +| dataset | \* | dataset information | object | ref: [dataset](#dataset-prop) | +| job_type | \* | execute job type | string enum: [insert, update, upsert, remove, search, getobject, exists] | search | +| repetition | | the number of job repetitions
default: `1` | integer | 1 | +| replica | | the number of job concurrent job executions
default: `1` | integer | 2 | +| rps | | designed request per sec to the target cluster
default: `1000` | integer | 1000 | +| concurrency_limit | | goroutine count limit for rps adjustment
default: `200` | integer | 20 | +| ttl_seconds_after_finished | | time until deletion of Pod after job end
default: `600` | integer | 120 | +| insert_config | | request config for insert job | object | ref: [config](#insert-cfg-props) | +| update_config | | request config for update job | object | ref: [config](#update-cfg-props) | +| upsert_config | | request config for upsert job | object | ref: [config](#upsert-cfg-props) | +| search_config | | request config for search job | object | ref: [config](#search-cfg-props) | +| remove_config | | request config for remove job | object | ref: [config](#remove-cfg-props) | +| object_config | | request config for object job | object | ref: [config](#object-cfg-props) | +| client_config | | gRPC client config for running benchmark job
Tune if can not getting the expected performance with default config. | object | ref: [defaults.grpc](https://github.com/vdaas/vald/blob/main/charts/vald/values.yaml) | +| server_config | | server config for benchmark job pod
Tune if can not getting the expected performance with default config. | object | ref: [defaults.server_config](https://github.com/vdaas/vald/blob/main/charts/vald/values.yaml) | diff --git a/docs/troubleshooting/client-side.md b/docs/troubleshooting/client-side.md index 6efc5f1ff3..a487c81383 100644 --- a/docs/troubleshooting/client-side.md +++ b/docs/troubleshooting/client-side.md @@ -55,4 +55,4 @@ Please check your CPU information. - [Provisioning Troubleshooting](../troubleshooting/provisioning.md) - [API Status](../api/status.md) -- [FAQ](../support/FAQ.md) +- [FAQ](/docs/support/faq) diff --git a/docs/tutorial/get-started-with-faiss-agent.md b/docs/tutorial/get-started-with-faiss-agent.md index 3b8dec1ad8..2abad816da 100644 --- a/docs/tutorial/get-started-with-faiss-agent.md +++ b/docs/tutorial/get-started-with-faiss-agent.md @@ -1,6 +1,6 @@ # Get Started -This tutorial is for those who have already completed [Get Started](https://github.com/vdaas/vald/blob/main/docs/tutorial/get-started.md). +This tutorial is for those who have already completed [Get Started](../tutorial/get-started.md). Please refer to Prepare the Kubernetes Cluster and others there. ## Deploy Vald on Kubernetes Cluster diff --git a/docs/tutorial/vald-multicluster-on-k8s.md b/docs/tutorial/vald-multicluster-on-k8s.md index 6cdd344bc6..15d1498a16 100644 --- a/docs/tutorial/vald-multicluster-on-k8s.md +++ b/docs/tutorial/vald-multicluster-on-k8s.md @@ -88,7 +88,7 @@ In this section, you will deploy three Vald clusters consisting of `vald-agent-n git clone https://github.com/vdaas/vald.git && cd vald ``` -2. Deploy on the `vald-01` Namespace using [dev-vald-01.yaml](https://github.com/vdaas/vald/blob/feature/mirror-gateway-definition/charts/vald/values/multi-vald/dev-vald-01.yaml) and [values.yaml](https://github.com/vdaas/vald/blob/main/example/helm/values.yaml) +2. Deploy on the `vald-01` Namespace using [dev-vald-01.yaml](https://github.com/vdaas/vald/blob/main/charts/vald/values/multi-vald/dev-vald-01.yaml) and [values.yaml](https://github.com/vdaas/vald/blob/main/example/helm/values.yaml) ```bash helm install vald-cluster-01 charts/vald \ @@ -97,7 +97,7 @@ In this section, you will deploy three Vald clusters consisting of `vald-agent-n -n vald-01 ``` -3. Deploy on the `vald-02` Namespace using [dev-vald-02.yaml](https://github.com/vdaas/vald/blob/feature/mirror-gateway-definition/charts/vald/values/multi-vald/dev-vald-02.yaml) and [values.yaml](https://github.com/vdaas/vald/blob/main/example/helm/values.yaml) +3. Deploy on the `vald-02` Namespace using [dev-vald-02.yaml](https://github.com/vdaas/vald/blob/main/charts/vald/values/multi-vald/dev-vald-02.yaml) and [values.yaml](https://github.com/vdaas/vald/blob/main/example/helm/values.yaml) ```bash helm install vald-cluster-02 charts/vald \ @@ -106,7 +106,7 @@ In this section, you will deploy three Vald clusters consisting of `vald-agent-n -n vald-02 ``` -4. Deploy on the `vald-03` Namespace using [dev-vald-03.yaml](https://github.com/vdaas/vald/blob/feature/mirror-gateway-definition/charts/vald/values/multi-vald/dev-vald-03.yaml) and [values.yaml](https://github.com/vdaas/vald/blob/main/example/helm/values.yaml) +4. Deploy on the `vald-03` Namespace using [dev-vald-03.yaml](https://github.com/vdaas/vald/blob/main/charts/vald/values/multi-vald/dev-vald-03.yaml) and [values.yaml](https://github.com/vdaas/vald/blob/main/example/helm/values.yaml) ```bash helm install vald-cluster-03 charts/vald \ @@ -194,7 +194,7 @@ It requires applying the `ValdMirrorTarget` Custom Resource to the one Namespace When applied successfully, the destination information is automatically created on other Namespaces when interconnected with each `vald-mirror-gateway`. -This tutorial will deploy the [ValdMirrorTarget](https://github.com/vdaas/vald/tree/main/charts/vald/values/mirror-target.yaml) Custom Resource to the `vald-03` Namespace with the following command. +This tutorial will deploy the [ValdMirrorTarget](https://github.com/vdaas/vald/blob/main/charts/vald/values/multi-vald/mirror-target.yaml) Custom Resource to the `vald-03` Namespace with the following command. ```bash kubectl apply -f ./charts/vald/values/multi-vald/mirror-target.yaml -n vald-03 @@ -255,7 +255,7 @@ If you are interested, please refer to [SDKs](https://vald.vdaas.org/docs/user- 3. Run Example - We use [example/client/mirror/main.go](https://github.com/vdaas/vald/blob/feature/mirror-gateway-example/example/client/mirror/main.go) to run the example. + We use [example/client/mirror/main.go](https://github.com/vdaas/vald/blob/main/example/client/mirror/main.go) to run the example. This example will insert and index 400 vectors into the Vald cluster from the Fashion-MNIST dataset via [gRPC](https://grpc.io/). And then, after waiting for indexing, it will request to search the nearest vector 10 times to all Vald clusters. You will get the 10 nearest neighbor vectors for each search query. diff --git a/docs/user-guides/cluster-role-binding.md b/docs/user-guides/cluster-role-binding.md index 36404ec157..af66e6f6b3 100644 --- a/docs/user-guides/cluster-role-binding.md +++ b/docs/user-guides/cluster-role-binding.md @@ -141,7 +141,7 @@ In this section, we will describe how to configure it and how to customize these ### Cluster role configuration for Vald Mirror Gateway -By looking at the [cluster role configuration](https://github.com/vdaas/vald/blob/main/k8s/gatewat/mirror/clusterrole.yaml), the access right of the following resources are granted to the cluster role `gateway-mirror`. +By looking at the [cluster role configuration](https://github.com/vdaas/vald/blob/main/k8s/gateway/mirror/clusterrole.yaml), the access right of the following resources are granted to the cluster role `gateway-mirror`. ```yaml apiVersion: rbac.authorization.k8s.io/v1 diff --git a/hack/actions/gen/main.go b/hack/actions/gen/main.go new file mode 100644 index 0000000000..d5be92145f --- /dev/null +++ b/hack/actions/gen/main.go @@ -0,0 +1,733 @@ +// +// Copyright (C) 2019-2024 vdaas.org vald team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "os" + "os/signal" + "strings" + "syscall" + "text/template" + "time" + + "github.com/vdaas/vald/internal/file" + "github.com/vdaas/vald/internal/log" + "github.com/vdaas/vald/internal/safety" + "github.com/vdaas/vald/internal/sync/errgroup" + "gopkg.in/yaml.v2" +) + +var license string = `# +# Copyright (C) 2019-{{.Year}} {{.Maintainer}} +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# DO_NOT_EDIT this workflow file is generated by https://github.com/vdaas/vald/blob/main/hack/actions/gen/main.go +` + +var licenseTmpl *template.Template = template.Must(template.New("license").Parse(license)) + +type ( + Workflow struct { + Name string `yaml:"name"` + On On `yaml:"on"` + Jobs Jobs `yaml:"jobs"` + } + + On struct { + Schedule Schedule `yaml:"schedule,omitempty"` + Push Push `yaml:"push"` + PullRequest PullRequest `yaml:"pull_request"` + PullRequestTarget PullRequest `yaml:"pull_request_target"` + } + + Schedule []struct { + Cron string `yaml:"cron,omitempty"` + } + + Push struct { + Branches []string `yaml:"branches"` + Tags []string `yaml:"tags"` + } + + PullRequest struct { + Paths Paths `yaml:"paths"` + } + + Jobs struct { + Build Build `yaml:"build"` + } + + Build struct { + Uses string `yaml:"uses"` + With With `yaml:"with"` + Secrets string `yaml:"secrets"` + } + + With struct { + Target string `yaml:"target"` + Platforms string `yaml:"platforms,omitempty"` + } + + Paths []string +) + +type Data struct { + AliasImage bool + ConfigExists bool + Year int + ContainerType ContainerType + AppName string + BinDir string + BuildUser string + BuilderImage string + BuilderTag string + BuildStageName string + Maintainer string + PackageDir string + RootDir string + RuntimeImage string + RuntimeTag string + RuntimeUser string + Name string + BuildPlatforms string + Arguments map[string]string + Environments map[string]string + Entrypoints []string + EnvironmentsSlice []string + ExtraCopies []string + ExtraImages []string + ExtraPackages []string + Preprocess []string + RunCommands []string + RunMounts []string + StageFiles []string + PullRequestPaths []string +} + +type ContainerType int + +const ( + Go ContainerType = iota + Rust + DevContainer + HelmOperator + CIContainer + Other +) + +const ( + organization = "vdaas" + repository = "vald" + defaultMaintainer = organization + ".org " + repository + " team <" + repository + "@" + organization + ".org>" + defaultBuildUser = "root:root" + maintainerKey = "MAINTAINER" + minimumArgumentLength = 2 + ubuntuVersion = "22.04" + + ngtPreprocess = "make ngt/install" + faissPreprocess = "make faiss/install" + + helmOperatorRootdir = "/opt/helm" + helmOperatorWatchFile = helmOperatorRootdir + "/watches.yaml" + helmOperatorChartsDir = helmOperatorRootdir + "/charts" +) + +var ( + clangBuildDeps = []string{ + "cmake", + "gcc", + "g++", + "unzip", + "libssl-dev", + } + ngtBuildDeps = []string{ + "liblapack-dev", + "libomp-dev", + "libopenblas-dev", + } + faissBuildDeps = []string{ + "gfortran", + } + devContainerDeps = []string{ + "gawk", + "gnupg2", + "graphviz", + "jq", + "libhdf5-dev", + "libaec-dev", + "sed", + "zip", + } + + ciContainerPreprocess = []string{ + "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} deps GO_CLEAN_DEPS=false", + "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} golangci-lint/install", + "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} gotestfmt/install", + "make cmake/install", + "make buf/install", + "make hdf5/install", + "make helm-docs/install", + "make helm/install", + "make k3d/install", + "make k9s/install", + "make kind/install", + "make kubectl/install", + "make kubelinter/install", + "make reviewdog/install", + "make tparse/install", + "make valdcli/install", + "make yq/install", + "make minikube/install", + "make stern/install", + "make telepresence/install", + } + + devContainerPreprocess = []string{ + "curl -fsSL https://deb.nodesource.com/setup_current.x | bash -", + "apt-get clean", + "apt-get update -y", + "apt-get upgrade -y", + "apt-get install -y --no-install-recommends --fix-missing nodejs", + "npm install -g npm@latest", + "apt-get clean", + "apt-get autoclean -y", + "apt-get autoremove -y", + "make delve/install", + "make gomodifytags/install", + "make gopls/install", + "make gotests/install", + "make impl/install", + "make staticcheck/install", + } +) + +const baseWorkflowTmpl string = `name: "Build docker image: %s" +on: + schedule: + - cron: "0 * * * *" + push: + branches: + - "main" + - "release/v*.*" + - "!release/v*.*.*" + tags: + - "*.*.*" + - "v*.*.*" + - "*.*.*-*" + - "v*.*.*-*" + pull_request: + paths: + - "hack/docker/gen/main.go" + - "dockers/%s/Dockerfile" + - "hack/actions/gen/main.go" + - ".github/workflows/dockers-%s-image.yaml" + - ".github/actions/docker-build/action.yaml" + - ".github/workflows/_docker-image.yaml" + - "cmd/%s/**" + - "pkg/%s/**" + pull_request_target: + paths: [] + +jobs: + build: + uses: "./.github/workflows/_docker-image.yaml" + with: + target: "%s" + platforms: "" + secrets: "inherit" +` + +const ( + cmdBenchOperatorsPath = "cmd/tools/benchmark/operators/**" + pkgBenchOperatorsPath = "pkg/tools/benchmark/operators/**" + cmdBenchJobsPath = "cmd/tools/benchmark/jobs/**" + pkgBenchJobsPath = "pkg/tools/benchmark/jobs/**" + + agentInternalPath = "pkg/agent/internal/**" + gatewayInternalPath = "pkg/gateway/internal/**" + + apisGrpcPath = "apis/grpc/**" + apisProtoPath = "apis/proto/**" + + hackPath = "hack/**" + + chartsValdPath = "charts/vald" + helmOperatorPath = "charts/vald-helm-operator" + chartPath = chartsValdPath + "/Chart.yaml" + valuesPath = chartsValdPath + "/values.yaml" + templatesPath = chartsValdPath + "/templates/**" + helmOperatorChartPath = helmOperatorPath + "/Chart.yaml" + helmOperatorValuesPath = helmOperatorPath + "/values.yaml" + helmOperatorTemplatesPath = helmOperatorPath + "/templates/**" + + goModPath = "go.mod" + goSumPath = "go.sum" + + cargoLockPath = "rust/Cargo.lock" + cargoTomlPath = "rust/Cargo.toml" + rustBinAgentDirPath = "rust/bin/agent" + rustNgtRsPath = "rust/libs/ngt-rs/**" + rustNgtPath = "rust/libs/ngt/**" + rustProtoPath = "rust/libs/proto/**" + + internalPath = "internal/**" + internalStoragePath = "internal/db/storage/blob/**" + excludeTestFilesPath = "!internal/**/*_test.go" + excludeMockFilesPath = "!internal/**/*_mock.go" + excludeDbPath = "!internal/db/**" + excludeK8sPath = "!internal/k8s/**" + + versionsPath = "versions" + operatorSDKVersionPath = versionsPath + "/OPERATOR_SDK_VERSION" + goVersionPath = versionsPath + "/GO_VERSION" + rustVersionPath = versionsPath + "/RUST_VERSION" + faissVersionPath = versionsPath + "/FAISS_VERSION" + ngtVersionPath = versionsPath + "/NGT_VERSION" + + makefilePath = "Makefile" + makefileDirPath = "Makefile.d/**" +) + +const ( + agentNgt = "agent-ngt" + agentFaiss = "agent-faiss" + agentSidecar = "agent-sidecar" + agent = "agent" + + discovererK8s = "discoverer-k8s" + + gateway = "gateway" + gatewayLb = "gateway-lb" + gatewayFilter = "gateway-filter" + gatewayMirror = "gateway-mirror" + + managerIndex = "manager-index" + + indexCorrection = "index-correction" + indexCreation = "index-creation" + indexSave = "index-save" + indexOperator = "index-operator" + + readreplicaRotate = "readreplica-rotate" + + benchJob = "benchmark-job" + benchOperator = "benchmark-operator" + + helmOperator = "helm-operator" + + loadtest = "loadtest" + + ciContainer = "ci-container" + devContainer = "dev-container" + + buildbase = "buildbase" + buildkit = "buildkit" + binfmt = "binfmt" + buildkitSyftScanner = "buildkit-syft-scanner" +) + +const ( + multiPlatforms = amd64Platform + "," + arm64Platform + amd64Platform = "linux/amd64" + arm64Platform = "linux/arm64" +) + +func (data *Data) initPullRequestPaths() { + switch data.Name { + // the benchmark components trigger each other, not just themselves + case benchJob: + data.PullRequestPaths = append(data.PullRequestPaths, + cmdBenchOperatorsPath, + pkgBenchOperatorsPath, + ) + case benchOperator: + data.PullRequestPaths = append(data.PullRequestPaths, + cmdBenchJobsPath, + pkgBenchJobsPath, + ) + case agentFaiss, agentNgt: + data.PullRequestPaths = append(data.PullRequestPaths, agentInternalPath) + default: + if strings.Contains(strings.ToLower(data.Name), gateway) { + data.PullRequestPaths = append(data.PullRequestPaths, gatewayInternalPath) + } + } + + switch data.ContainerType { + case HelmOperator: + data.PullRequestPaths = append(data.PullRequestPaths, + chartPath, + valuesPath, + templatesPath, + helmOperatorChartPath, + helmOperatorValuesPath, + helmOperatorTemplatesPath, + operatorSDKVersionPath, + ) + case DevContainer, CIContainer: + data.PullRequestPaths = append(data.PullRequestPaths, + apisGrpcPath, + apisProtoPath, + hackPath, + ) + case Go: + data.PullRequestPaths = append(data.PullRequestPaths, + apisGrpcPath, + apisProtoPath, + goModPath, + goSumPath, + goVersionPath, + internalPath, + excludeTestFilesPath, + excludeMockFilesPath, + excludeDbPath, + ) + switch data.Name { + case discovererK8s, indexOperator, gatewayMirror, readreplicaRotate, agentNgt, benchJob, benchOperator: + default: + data.PullRequestPaths = append(data.PullRequestPaths, excludeK8sPath) + } + case Rust: + data.PullRequestPaths = append(data.PullRequestPaths, + apisGrpcPath, + apisProtoPath, + cargoLockPath, + cargoTomlPath, + rustBinAgentDirPath, + rustNgtRsPath, + rustNgtPath, + rustProtoPath, + rustVersionPath, + faissVersionPath, + ngtVersionPath, + ) + } + if strings.EqualFold(data.Name, agentFaiss) || data.ContainerType == Rust { + data.PullRequestPaths = append(data.PullRequestPaths, faissVersionPath) + } + if strings.EqualFold(data.Name, agentNgt) || data.ContainerType == Rust { + data.PullRequestPaths = append(data.PullRequestPaths, ngtVersionPath) + } + + if data.Name == agentSidecar { + data.PullRequestPaths = append(data.PullRequestPaths, internalStoragePath) + } + if !data.AliasImage { + data.PullRequestPaths = append(data.PullRequestPaths, makefilePath, makefileDirPath) + } +} + +func (data *Data) initData() { + data.initPullRequestPaths() + + if data.AliasImage { + data.BuildPlatforms = multiPlatforms + } + if data.ContainerType == CIContainer || data.Name == loadtest { + data.BuildPlatforms = amd64Platform + } + + data.Year = time.Now().Year() + if maintainer := os.Getenv(maintainerKey); maintainer != "" { + data.Maintainer = maintainer + } else { + data.Maintainer = defaultMaintainer + } +} + +func (data *Data) generateWorkflowStruct() (*Workflow, error) { + workflow := &Workflow{} + baseWorkflow := fmt.Sprintf(baseWorkflowTmpl, + data.Name, + data.PackageDir, + data.Name, + data.PackageDir, + data.PackageDir, + data.Name, + ) + err := yaml.NewDecoder(strings.NewReader(baseWorkflow)).Decode(workflow) + if err != nil { + return nil, fmt.Errorf("Error decoding YAML: %v", err) + } + + if !data.AliasImage { + workflow.On.Schedule = nil + } + workflow.On.PullRequest.Paths = append(workflow.On.PullRequest.Paths, data.PullRequestPaths...) + workflow.On.PullRequestTarget.Paths = workflow.On.PullRequest.Paths + workflow.Jobs.Build.With.Platforms = data.BuildPlatforms + + return workflow, nil +} + +func main() { + log.Init() + if len(os.Args) < minimumArgumentLength { + // skipcq: RVV-A0003 + log.Fatal(errors.New("invalid argument")) + } + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, + syscall.SIGQUIT, + syscall.SIGHUP, + syscall.SIGALRM, + syscall.SIGKILL, + syscall.SIGTERM) + defer cancel() + + eg, egctx := errgroup.New(ctx) + for n, d := range map[string]Data{ + "vald-agent-ngt": { + AppName: "ngt", + PackageDir: "agent/core/ngt", + ExtraPackages: append(clangBuildDeps, ngtBuildDeps...), + Preprocess: []string{ngtPreprocess}, + }, + "vald-agent-faiss": { + AppName: "faiss", + PackageDir: "agent/core/faiss", + ExtraPackages: append(clangBuildDeps, + append(ngtBuildDeps, + faissBuildDeps...)...), + Preprocess: []string{faissPreprocess}, + }, + "vald-agent": { + AppName: "agent", + PackageDir: "agent/core/agent", + ContainerType: Rust, + RuntimeImage: "gcr.io/distroless/cc-debian12", + ExtraPackages: append(clangBuildDeps, + append(ngtBuildDeps, + faissBuildDeps...)...), + Preprocess: []string{ + ngtPreprocess, + faissPreprocess, + }, + }, + "vald-agent-sidecar": { + AppName: "sidecar", + PackageDir: "agent/sidecar", + }, + "vald-discoverer-k8s": { + AppName: "discoverer", + PackageDir: "discoverer/k8s", + }, + "vald-gateway-lb": { + AppName: "lb", + PackageDir: "gateway/lb", + }, + "vald-gateway-filter": { + AppName: "filter", + PackageDir: "gateway/filter", + }, + "vald-gateway-mirror": { + AppName: "mirror", + PackageDir: "gateway/mirror", + }, + "vald-manager-index": { + AppName: "index", + PackageDir: "manager/index", + }, + "vald-index-correction": { + AppName: "index-correction", + PackageDir: "index/job/correction", + }, + "vald-index-creation": { + AppName: "index-creation", + PackageDir: "index/job/creation", + }, + "vald-index-save": { + AppName: "index-save", + PackageDir: "index/job/save", + }, + "vald-readreplica-rotate": { + AppName: "readreplica-rotate", + PackageDir: "index/job/readreplica/rotate", + }, + "vald-index-operator": { + AppName: "index-operator", + PackageDir: "index/operator", + }, + "vald-benchmark-job": { + AppName: "job", + PackageDir: "tools/benchmark/job", + ExtraPackages: append(clangBuildDeps, "libhdf5-dev", "libaec-dev"), + Preprocess: []string{ + "make hdf5/install", + }, + }, + "vald-benchmark-operator": { + AppName: "operator", + PackageDir: "tools/benchmark/operator", + }, + "vald-helm-operator": { + AppName: "helm-operator", + PackageDir: "operator/helm", + ContainerType: HelmOperator, + Arguments: map[string]string{ + "OPERATOR_SDK_VERSION": "latest", + }, + ExtraCopies: []string{ + "--from=operator /usr/local/bin/${APP_NAME} {{$.BinDir}}/${APP_NAME}", + }, + ExtraImages: []string{ + "quay.io/operator-framework/helm-operator:${OPERATOR_SDK_VERSION} AS operator", + }, + ExtraPackages: []string{"upx"}, + Preprocess: []string{ + "mkdir -p " + helmOperatorChartsDir, + `{ \ + echo "---"; \ + echo "- version: v1"; \ + echo " group: vald.vdaas.org"; \ + echo " kind: ValdRelease"; \ + echo " chart: ` + helmOperatorChartsDir + `/vald"; \ + echo "- version: v1"; \ + echo " group: vald.vdaas.org"; \ + echo " kind: ValdHelmOperatorRelease"; \ + echo " chart: ` + helmOperatorChartsDir + `/vald-helm-operator"; \ + } > ` + helmOperatorWatchFile, + "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} helm/schema/vald", + "make GOARCH=${TARGETARCH} GOOS=${TARGETOS} helm/schema/vald-helm-operator", + "cp -r charts/* " + helmOperatorChartsDir + "/", + "upx \"{{$.BinDir}}/${APP_NAME}\"", + }, + StageFiles: []string{ + helmOperatorWatchFile, + helmOperatorChartsDir + "/vald", + helmOperatorChartsDir + "/vald-helm-operator", + }, + Entrypoints: []string{"{{$.BinDir}}/{{.AppName}}", "run", "--watches-file=" + helmOperatorWatchFile}, + }, + "vald-loadtest": { // note: this name is a little different from that of docker/gen/main.go + AppName: "loadtest", + PackageDir: "tools/cli/loadtest", + ExtraPackages: append(clangBuildDeps, "libhdf5-dev", "libaec-dev"), + Preprocess: []string{ + "make hdf5/install", + }, + }, + "vald-ci-container": { + AppName: "ci-container", + ContainerType: CIContainer, + PackageDir: "ci/base", + RuntimeUser: defaultBuildUser, + ExtraPackages: append([]string{"npm"}, append(clangBuildDeps, + append(ngtBuildDeps, + append(faissBuildDeps, + devContainerDeps...)...)...)...), + Preprocess: append(ciContainerPreprocess, ngtPreprocess, faissPreprocess), + Entrypoints: []string{"/bin/bash"}, + }, + "vald-dev-container": { + AppName: "dev-container", + BuilderImage: "mcr.microsoft.com/devcontainers/base", + BuilderTag: "ubuntu" + ubuntuVersion, + BuildUser: defaultBuildUser, + RuntimeUser: defaultBuildUser, + ContainerType: DevContainer, + PackageDir: "dev", + ExtraPackages: append(clangBuildDeps, + append(ngtBuildDeps, + append(faissBuildDeps, + devContainerDeps...)...)...), + Preprocess: append(devContainerPreprocess, + append(ciContainerPreprocess, + ngtPreprocess, + faissPreprocess)...), + }, + "vald-buildbase": { + AppName: "buildbase", + AliasImage: true, + PackageDir: "buildbase", + BuilderImage: "ubuntu", + BuilderTag: "devel", + }, + "vald-buildkit": { + AppName: "buildkit", + AliasImage: true, + PackageDir: "buildkit", + BuilderImage: "moby/buildkit", + BuilderTag: "master", + }, + "vald-binfmt": { + AppName: "binfmt", + AliasImage: true, + PackageDir: "binfmt", + BuilderImage: "tonistiigi/binfmt", + BuilderTag: "master", + }, + "vald-buildkit-syft-scanner": { + AppName: "scanner", + AliasImage: true, + PackageDir: "buildkit/syft/scanner", + BuilderImage: "docker/buildkit-syft-scanner", + BuilderTag: "edge", + BuildStageName: "scanner", + }, + } { + name := n + data := d + + eg.Go(safety.RecoverFunc(func() error { + data.Name = strings.TrimPrefix(name, "vald-") + data.initData() + + log.Infof("Generating %s's workflow", data.Name) + workflow, err := data.generateWorkflowStruct() + if err != nil { + return fmt.Errorf("Error generating workflowStruct: %w", err) + } + workflowYamlTmp, err := yaml.Marshal(workflow) + if err != nil { + return fmt.Errorf("error marshaling workflowStruct to YAML: %w", err) + } + + // remove the double quotation marks from the generated key "on": (note that the word "on" is a reserved word in gopkg.in/yaml.v2) + workflowYaml := strings.Replace(string(workflowYamlTmp), "\"on\":", "on:", 1) + + buf := bytes.NewBuffer(make([]byte, 0, len(license)+len(workflowYaml))) + err = licenseTmpl.Execute(buf, data) + if err != nil { + return fmt.Errorf("error executing template: %w", err) + } + buf.WriteString(workflowYaml) + _, err = file.OverWriteFile(egctx, file.Join(os.Args[1], ".github/workflows", "dockers-"+data.Name+"-image.yaml"), buf, fs.ModePerm) + if err != nil { + return fmt.Errorf("error writing file: %w", err) + } + return nil + })) + } + eg.Wait() +} diff --git a/internal/test/data/vector/gen_test.go b/internal/test/data/vector/gen_test.go index 24f1c11bee..4f3049b4ae 100644 --- a/internal/test/data/vector/gen_test.go +++ b/internal/test/data/vector/gen_test.go @@ -47,8 +47,13 @@ func TestFloat32VectorGenerator(t *testing.T) { } if got != nil { vectors := got(a.n, a.dim) - if len(vectors) != w.n && len(vectors[0]) != w.dim { - return errors.Errorf("got: \"%d\",\"%d\"\n\t\t\t\twant: \"%d\",\"%d\"", len(vectors), len(vectors[0]), w.n, w.dim) + if len(vectors) != w.n { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twantLen: \"%#v\"", len(vectors), w.n) + } + for _, vec := range vectors { + if len(vec) != w.dim { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twantDim: \"%#v\"", len(vec), w.dim) + } } } return nil @@ -140,8 +145,13 @@ func TestUint8VectorGenerator(t *testing.T) { } if got != nil { vectors := got(a.n, a.dim) - if len(vectors) != w.n && len(vectors[0]) != w.dim { - return errors.Errorf("got: \"%d\",\"%d\"\n\t\t\t\twant: \"%d\",\"%d\"", len(vectors), len(vectors[0]), w.n, w.dim) + if len(vectors) != w.n { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twantLen: \"%#v\"", len(vectors), w.n) + } + for _, vec := range vectors { + if len(vec) != w.dim { + return errors.Errorf("got: \"%#v\",\n\t\t\t\twantDim: \"%#v\"", len(vec), w.dim) + } } } return nil diff --git a/k8s/gateway/gateway/mirror/clusterrole.yaml b/k8s/gateway/gateway/mirror/clusterrole.yaml new file mode 100644 index 0000000000..c3e6bf4c1a --- /dev/null +++ b/k8s/gateway/gateway/mirror/clusterrole.yaml @@ -0,0 +1,55 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gateway-mirror + labels: + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.13 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.13 + app.kubernetes.io/component: gateway-mirror +rules: + - apiGroups: + - vald.vdaas.org + resources: + - valdmirrortargets + verbs: + - create + - update + - delete + - get + - list + - watch + - patch + - apiGroups: + - vald.vdaas.org + resources: + - valdmirrortargets/status + verbs: + - create + - update + - get + - list + - patch + - apiGroups: + - vald.vdaas.org + resources: + - valdmirrortargets/finalizers + verbs: + - update diff --git a/k8s/gateway/gateway/mirror/clusterrolebinding.yaml b/k8s/gateway/gateway/mirror/clusterrolebinding.yaml new file mode 100644 index 0000000000..0795540838 --- /dev/null +++ b/k8s/gateway/gateway/mirror/clusterrolebinding.yaml @@ -0,0 +1,34 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gateway-mirror + labels: + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.13 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.13 + app.kubernetes.io/component: gateway-mirror +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gateway-mirror +subjects: + - kind: ServiceAccount + name: gateway-mirror + namespace: default diff --git a/k8s/gateway/gateway/mirror/configmap.yaml b/k8s/gateway/gateway/mirror/configmap.yaml new file mode 100644 index 0000000000..a060d7020e --- /dev/null +++ b/k8s/gateway/gateway/mirror/configmap.yaml @@ -0,0 +1,28 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: ConfigMap +metadata: + name: vald-mirror-gateway-config + labels: + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.13 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.13 + app.kubernetes.io/component: gateway-mirror +data: + config.yaml: "---\nversion: v0.0.0\ntime_zone: UTC\nlogging:\n format: raw\n level: debug\n logger: glg\nserver_config:\n servers:\n - name: grpc\n host: 0.0.0.0\n port: 8081\n grpc:\n bidirectional_stream_concurrency: 20\n connection_timeout: \"\"\n enable_admin: true\n enable_reflection: true\n header_table_size: 0\n initial_conn_window_size: 2097152\n initial_window_size: 1048576\n interceptors:\n - RecoverInterceptor\n keepalive:\n max_conn_age: \"\"\n max_conn_age_grace: \"\"\n max_conn_idle: \"\"\n min_time: 10m\n permit_without_stream: false\n time: 3h\n timeout: 60s\n max_header_list_size: 0\n max_receive_message_size: 0\n max_send_message_size: 0\n read_buffer_size: 0\n write_buffer_size: 0\n mode: GRPC\n network: tcp\n probe_wait_time: 3s\n restart: true\n socket_option:\n ip_recover_destination_addr: false\n ip_transparent: false\n reuse_addr: true\n reuse_port: true\n tcp_cork: false\n tcp_defer_accept: false\n tcp_fast_open: false\n tcp_no_delay: false\n tcp_quick_ack: false\n socket_path: \"\"\n health_check_servers:\n - name: liveness\n host: 0.0.0.0\n port: 3000\n http:\n handler_timeout: \"\"\n http2:\n enabled: false\n handler_limit: 0\n max_concurrent_streams: 0\n max_decoder_header_table_size: 4096\n max_encoder_header_table_size: 4096\n max_read_frame_size: 0\n max_upload_buffer_per_connection: 0\n max_upload_buffer_per_stream: 0\n permit_prohibited_cipher_suites: true\n idle_timeout: \"\"\n read_header_timeout: \"\"\n read_timeout: \"\"\n shutdown_duration: 5s\n write_timeout: \"\"\n mode: REST\n network: tcp\n probe_wait_time: 3s\n restart: true\n socket_option:\n ip_recover_destination_addr: false\n ip_transparent: false\n reuse_addr: true\n reuse_port: true\n tcp_cork: false\n tcp_defer_accept: false\n tcp_fast_open: true\n tcp_no_delay: true\n tcp_quick_ack: true\n socket_path: \"\"\n - name: readiness\n host: 0.0.0.0\n port: 3001\n http:\n handler_timeout: \"\"\n http2:\n enabled: false\n handler_limit: 0\n max_concurrent_streams: 0\n max_decoder_header_table_size: 4096\n max_encoder_header_table_size: 4096\n max_read_frame_size: 0\n max_upload_buffer_per_connection: 0\n max_upload_buffer_per_stream: 0\n permit_prohibited_cipher_suites: true\n idle_timeout: \"\"\n read_header_timeout: \"\"\n read_timeout: \"\"\n shutdown_duration: 0s\n write_timeout: \"\"\n mode: REST\n network: tcp\n probe_wait_time: 3s\n restart: true\n socket_option:\n ip_recover_destination_addr: false\n ip_transparent: false\n reuse_addr: true\n reuse_port: true\n tcp_cork: false\n tcp_defer_accept: false\n tcp_fast_open: true\n tcp_no_delay: true\n tcp_quick_ack: true\n socket_path: \"\"\n metrics_servers:\n - name: pprof\n host: 0.0.0.0\n port: 6060\n http:\n handler_timeout: 5s\n http2:\n enabled: false\n handler_limit: 0\n max_concurrent_streams: 0\n max_decoder_header_table_size: 4096\n max_encoder_header_table_size: 4096\n max_read_frame_size: 0\n max_upload_buffer_per_connection: 0\n max_upload_buffer_per_stream: 0\n permit_prohibited_cipher_suites: true\n idle_timeout: 2s\n read_header_timeout: 1s\n read_timeout: 1s\n shutdown_duration: 5s\n write_timeout: 1m\n mode: REST\n network: tcp\n probe_wait_time: 3s\n restart: true\n socket_option:\n ip_recover_destination_addr: false\n ip_transparent: false\n reuse_addr: true\n reuse_port: true\n tcp_cork: true\n tcp_defer_accept: false\n tcp_fast_open: false\n tcp_no_delay: false\n tcp_quick_ack: false\n socket_path: \"\"\n startup_strategy:\n - liveness\n - pprof\n - grpc\n - readiness\n shutdown_strategy:\n - readiness\n - grpc\n - pprof\n - liveness\n full_shutdown_duration: 600s\n tls:\n ca: /path/to/ca\n cert: /path/to/cert\n enabled: false\n insecure_skip_verify: false\n key: /path/to/key\nobservability:\n enabled: false\n otlp:\n collector_endpoint: \"\"\n trace_batch_timeout: \"1s\"\n trace_export_timeout: \"1m\"\n trace_max_export_batch_size: 1024\n trace_max_queue_size: 256\n metrics_export_interval: \"1s\"\n metrics_export_timeout: \"1m\"\n attribute:\n namespace: \"_MY_POD_NAMESPACE_\"\n pod_name: \"_MY_POD_NAME_\"\n node_name: \"_MY_NODE_NAME_\"\n service_name: \"vald-mirror-gateway\"\n metrics:\n enable_cgo: true\n enable_goroutine: true\n enable_memory: true\n enable_version_info: true\n version_info_labels:\n - vald_version\n - server_name\n - git_commit\n - build_time\n - go_version\n - go_os\n - go_arch\n - algorithm_info\n trace:\n enabled: false\ngateway:\n pod_name: _MY_POD_NAME_\n register_duration: 1s\n namespace: _MY_POD_NAMESPACE_\n discovery_duration: 1s\n colocation: dc1\n group: \n net:\n dialer:\n dual_stack_enabled: false\n keepalive: 10m\n timeout: 30s\n dns:\n cache_enabled: true\n cache_expiration: 24h\n refresh_duration: 5m\n socket_option:\n ip_recover_destination_addr: false\n ip_transparent: false\n reuse_addr: true\n reuse_port: true\n tcp_cork: false\n tcp_defer_accept: true\n tcp_fast_open: true\n tcp_no_delay: true\n tcp_quick_ack: true\n tls:\n ca: /path/to/ca\n cert: /path/to/cert\n enabled: false\n insecure_skip_verify: false\n key: /path/to/key\n client:\n addrs:\n - vald-lb-gateway.default.svc.cluster.local:8081\n health_check_duration: \"1s\"\n connection_pool:\n enable_dns_resolver: true\n enable_rebalance: true\n old_conn_close_duration: 2m\n rebalance_duration: 30m\n size: 3\n backoff:\n backoff_factor: 1.1\n backoff_time_limit: 5s\n enable_error_log: true\n initial_duration: 5ms\n jitter_limit: 100ms\n maximum_duration: 5s\n retry_count: 100\n circuit_breaker:\n closed_error_rate: 0.7\n closed_refresh_timeout: 10s\n half_open_error_rate: 0.5\n min_samples: 1000\n open_timeout: 1s\n call_option:\n max_recv_msg_size: 0\n max_retry_rpc_buffer_size: 0\n max_send_msg_size: 0\n wait_for_ready: true\n dial_option:\n backoff_base_delay: 1s\n backoff_jitter: 0.2\n backoff_max_delay: 120s\n backoff_multiplier: 1.6\n enable_backoff: false\n initial_connection_window_size: 2097152\n initial_window_size: 1048576\n insecure: true\n interceptors: []\n keepalive:\n permit_without_stream: false\n time: \"\"\n timeout: 30s\n max_msg_size: 0\n min_connection_timeout: 20s\n net:\n dialer:\n dual_stack_enabled: true\n keepalive: \"\"\n timeout: \"\"\n dns:\n cache_enabled: true\n cache_expiration: 1h\n refresh_duration: 30m\n socket_option:\n ip_recover_destination_addr: false\n ip_transparent: false\n reuse_addr: true\n reuse_port: true\n tcp_cork: false\n tcp_defer_accept: false\n tcp_fast_open: false\n tcp_no_delay: false\n tcp_quick_ack: false\n tls:\n ca: /path/to/ca\n cert: /path/to/cert\n enabled: false\n insecure_skip_verify: false\n key: /path/to/key\n read_buffer_size: 0\n timeout: \"\"\n write_buffer_size: 0\n tls:\n ca: /path/to/ca\n cert: /path/to/cert\n enabled: false\n insecure_skip_verify: false\n key: /path/to/key\n self_mirror_addr: vald-mirror-gateway.default.svc.cluster.local:8081\n gateway_addr: vald-lb-gateway.default.svc.cluster.local:8081\n" diff --git a/k8s/gateway/gateway/mirror/deployment.yaml b/k8s/gateway/gateway/mirror/deployment.yaml new file mode 100644 index 0000000000..a1674d0acd --- /dev/null +++ b/k8s/gateway/gateway/mirror/deployment.yaml @@ -0,0 +1,184 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vald-mirror-gateway + labels: + app: vald-mirror-gateway + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.13 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.13 + app.kubernetes.io/component: gateway-mirror +spec: + progressDeadlineSeconds: 600 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: vald-mirror-gateway + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: vald-mirror-gateway + app.kubernetes.io/name: vald + app.kubernetes.io/instance: release-name + app.kubernetes.io/component: gateway-mirror + annotations: + checksum/configmap: 8546d68c1063b706eda3ba96b346cf7e76c59d8010692426f21b802aa1f35a2e + pyroscope.io/scrape: "true" + pyroscope.io/application-name: vald-mirror-gateway + pyroscope.io/profile-cpu-enabled: "true" + pyroscope.io/profile-mem-enabled: "true" + pyroscope.io/port: "6060" + spec: + initContainers: + - name: wait-for-gateway-lb + image: busybox:stable + command: + - /bin/sh + - -e + - -c + - | + until [ "$(wget --server-response --spider --quiet http://vald-lb-gateway.default.svc.cluster.local:3001/readiness 2>&1 | awk 'NR==1{print $2}')" == "200" ]; do + echo "waiting for gateway-lb to be ready..." + sleep 2; + done + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: [] + podAffinity: + preferredDuringSchedulingIgnoredDuringExecution: [] + requiredDuringSchedulingIgnoredDuringExecution: [] + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - vald-mirror-gateway + topologyKey: kubernetes.io/hostname + weight: 100 + requiredDuringSchedulingIgnoredDuringExecution: [] + containers: + - name: vald-mirror-gateway + image: "vdaas/vald-mirror-gateway:nightly" + imagePullPolicy: Always + livenessProbe: + failureThreshold: 2 + httpGet: + path: /liveness + port: liveness + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 2 + readinessProbe: + failureThreshold: 2 + httpGet: + path: /readiness + port: readiness + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 3 + successThreshold: 1 + timeoutSeconds: 2 + startupProbe: + failureThreshold: 30 + httpGet: + path: /liveness + port: liveness + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 2 + ports: + - name: liveness + protocol: TCP + containerPort: 3000 + - name: readiness + protocol: TCP + containerPort: 3001 + - name: grpc + protocol: TCP + containerPort: 8081 + - name: pprof + protocol: TCP + containerPort: 6060 + resources: + limits: + cpu: 2000m + memory: 700Mi + requests: + cpu: 200m + memory: 150Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: vald-mirror-gateway-config + mountPath: /etc/server/ + dnsPolicy: ClusterFirst + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: gateway-mirror + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + terminationGracePeriodSeconds: 30 + volumes: + - name: vald-mirror-gateway-config + configMap: + defaultMode: 420 + name: vald-mirror-gateway-config + priorityClassName: default-vald-mirror-gateway-priority +status: diff --git a/k8s/gateway/gateway/mirror/hpa.yaml b/k8s/gateway/gateway/mirror/hpa.yaml new file mode 100644 index 0000000000..d3d4d03b69 --- /dev/null +++ b/k8s/gateway/gateway/mirror/hpa.yaml @@ -0,0 +1,35 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: vald-mirror-gateway + labels: + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.13 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.13 + app.kubernetes.io/component: gateway-mirror +spec: + maxReplicas: 9 + minReplicas: 3 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: vald-mirror-gateway + targetCPUUtilizationPercentage: 80 +status: diff --git a/k8s/gateway/gateway/mirror/pdb.yaml b/k8s/gateway/gateway/mirror/pdb.yaml new file mode 100644 index 0000000000..37f81f96ee --- /dev/null +++ b/k8s/gateway/gateway/mirror/pdb.yaml @@ -0,0 +1,31 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: vald-mirror-gateway + labels: + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.13 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.13 + app.kubernetes.io/component: gateway-mirror +spec: + maxUnavailable: 50% + selector: + matchLabels: + app: vald-mirror-gateway diff --git a/k8s/gateway/gateway/mirror/priorityclass.yaml b/k8s/gateway/gateway/mirror/priorityclass.yaml new file mode 100644 index 0000000000..a3be44bc0a --- /dev/null +++ b/k8s/gateway/gateway/mirror/priorityclass.yaml @@ -0,0 +1,29 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: default-vald-mirror-gateway-priority + labels: + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.13 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.13 + app.kubernetes.io/component: gateway-mirror +value: 1e+06 +globalDefault: false +description: "A priority class for Vald mirror gateway." diff --git a/k8s/gateway/gateway/mirror/serviceaccount.yaml b/k8s/gateway/gateway/mirror/serviceaccount.yaml new file mode 100644 index 0000000000..ac49327d35 --- /dev/null +++ b/k8s/gateway/gateway/mirror/serviceaccount.yaml @@ -0,0 +1,26 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gateway-mirror + labels: + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.13 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.13 + app.kubernetes.io/component: gateway-mirror diff --git a/k8s/gateway/gateway/mirror/svc.yaml b/k8s/gateway/gateway/mirror/svc.yaml new file mode 100644 index 0000000000..9718722b17 --- /dev/null +++ b/k8s/gateway/gateway/mirror/svc.yaml @@ -0,0 +1,45 @@ +# +# Copyright (C) 2019-2024 vdaas.org vald team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# You may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: v1 +kind: Service +metadata: + name: vald-mirror-gateway + labels: + app.kubernetes.io/name: vald + helm.sh/chart: vald-v1.7.13 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: v1.7.13 + app.kubernetes.io/component: gateway-mirror +spec: + ports: + - name: grpc + port: 8081 + targetPort: 8081 + protocol: TCP + - name: readiness + port: 3001 + targetPort: 3001 + protocol: TCP + - name: pprof + port: 6060 + targetPort: 6060 + protocol: TCP + selector: + app.kubernetes.io/name: vald + app.kubernetes.io/component: gateway-mirror + clusterIP: None + type: ClusterIP diff --git a/pkg/agent/core/ngt/handler/grpc/index.go b/pkg/agent/core/ngt/handler/grpc/index.go index 62df71e180..8e7f24d9f6 100644 --- a/pkg/agent/core/ngt/handler/grpc/index.go +++ b/pkg/agent/core/ngt/handler/grpc/index.go @@ -64,6 +64,9 @@ func (s *server) CreateIndex( }, }, info.Get())...) code = codes.FailedPrecondition + case errors.Is(err, errors.ErrFlushingIsInProgress): + err = status.WrapWithAborted("CreateIndex API aborted to process create indexes request due to flushing indices is in progress", err, details...) + code = codes.Aborted case errors.Is(err, context.Canceled): err = status.WrapWithCanceled(fmt.Sprintf("CreateIndex API canceled to create indexes pool_size = %d, error: %v", c.GetPoolSize(), err), err, details...) code = codes.Canceled @@ -149,6 +152,9 @@ func (s *server) CreateAndSaveIndex( }, }, info.Get())...) code = codes.FailedPrecondition + case errors.Is(err, errors.ErrFlushingIsInProgress): + err = status.WrapWithAborted("CreateAndSaveIndex API aborted to process create indexes request due to flushing indices is in progress", err, details...) + code = codes.Aborted case errors.Is(err, context.Canceled): err = status.WrapWithCanceled(fmt.Sprintf("CreateAndSaveIndex API canceled to create indexes pool_size = %d, error: %v", c.GetPoolSize(), err), err, details...) code = codes.Canceled diff --git a/pkg/agent/core/ngt/service/ngt.go b/pkg/agent/core/ngt/service/ngt.go index 095f0b6f0a..09bf8577f5 100644 --- a/pkg/agent/core/ngt/service/ngt.go +++ b/pkg/agent/core/ngt/service/ngt.go @@ -260,6 +260,26 @@ func newNGT(cfg *config.NGT, opts ...Option) (n *ngt, err error) { return n, nil } +func (n *ngt) copyNGT(src *ngt) { + // instances + n.core = src.core + n.kvs = src.kvs + n.fmap = src.fmap + n.vq = src.vq + + // counters + n.wfci = src.wfci + n.nobic = src.nobic + n.nopvq = atomic.Uint64{} + + // paths + n.path = src.path + n.tmpPath = src.tmpPath + n.oldPath = src.oldPath + n.basePath = src.basePath + n.brokenPath = src.brokenPath +} + // migrate migrates the index directory from old to new under the input path if necessary. // Migration happens when the path is not empty and there is no `path/origin` directory, // which indicates that the user has NOT been using CoW mode and the index directory is not migrated yet. @@ -908,7 +928,7 @@ func (n *ngt) Start(ctx context.Context) <-chan error { } return ctx.Err() case <-tick.C: - if n.vq.IVQLen() >= n.alen { + if n.vq != nil && !n.IsFlushing() && n.vq.IVQLen() >= n.alen { err = n.CreateIndex(ctx, n.poolSize) } case <-limit.C: @@ -1242,14 +1262,12 @@ func (n *ngt) RegenerateIndexes(ctx context.Context) (err error) { if err != nil { log.Errorf("failed to flushing vector to ngt index in delete kvs. error: %v", err) } - n.kvs = nil - n.vq = nil // gc runtime.GC() atomic.AddUint64(&n.nogce, 1) - if n.inMem { + if !n.inMem { // delete file err = file.DeleteDir(ctx, n.path) if err != nil { @@ -1265,30 +1283,14 @@ func (n *ngt) RegenerateIndexes(ctx context.Context) (err error) { } } - nkvs := kvs.New(kvs.WithConcurrency(n.kvsdbConcurrency)) - - nvq, err := vqueue.New() - if err != nil { - log.Errorf("failed to create new vector vector queue. error: %v", err) - } - // renew instance nn, err := newNGT(n.cfg, n.opts...) if err != nil { return err } - nn.kvs = nkvs - nn.vq = nvq - - // Regenerate with flags set - nn.flushing.Store(true) - nn.indexing.Store(true) - defer nn.flushing.Store(false) - defer nn.indexing.Store(false) + n.copyNGT(nn) - n = nn - - return nil + return n.loadStatistics() } func (n *ngt) CreateIndex(ctx context.Context, poolSize uint32) (err error) { @@ -1299,8 +1301,11 @@ func (n *ngt) CreateIndex(ctx context.Context, poolSize uint32) (err error) { } }() - if n.isReadReplica { + switch { + case n.isReadReplica: return errors.ErrWriteOperationToReadReplica + case n.IsFlushing(): + return errors.ErrFlushingIsInProgress } ic := n.vq.IVQLen() + n.vq.DVQLen() @@ -1428,6 +1433,10 @@ func (n *ngt) CreateIndex(ctx context.Context, poolSize uint32) (err error) { return err } } + return n.loadStatistics() +} + +func (n *ngt) loadStatistics() error { if n.IsStatisticsEnabled() { log.Info("loading index statistics to cache") stats, err := n.core.GetGraphStatistics(core.AdditionalStatistics) @@ -1471,8 +1480,7 @@ func (n *ngt) CreateIndex(ctx context.Context, poolSize uint32) (err error) { IndegreeHistogram: stats.IndegreeHistogram, }) } - - return err + return nil } func (n *ngt) removeInvalidIndex(ctx context.Context) { @@ -1941,15 +1949,24 @@ func (n *ngt) gc() { } func (n *ngt) Len() uint64 { - return n.kvs.Len() + if n.kvs != nil && !n.IsFlushing() { + return n.kvs.Len() + } + return 0 } func (n *ngt) InsertVQueueBufferLen() uint64 { - return uint64(n.vq.IVQLen()) + if n.vq != nil && !n.IsFlushing() { + return uint64(n.vq.IVQLen()) + } + return 0 } func (n *ngt) DeleteVQueueBufferLen() uint64 { - return uint64(n.vq.DVQLen()) + if n.vq != nil && !n.IsFlushing() { + return uint64(n.vq.DVQLen()) + } + return 0 } func (n *ngt) GetDimensionSize() int {