Merge branch 'main' into feature/agent/implement-usearch

.gitfiles

@@ -46,7 +46,7 @@
 .github/chatops_permissions.yaml
 .github/codeql/codeql-config.yaml
 .github/conflint.yaml
-.github/dependabot.yml
+.github/dependabot.yaml
 .github/helm/values/vald-mirror-target.yaml
 .github/helm/values/values-agent-sidecar.yaml
 .github/helm/values/values-chaos.yaml
@@ -60,72 +60,72 @@
 .github/helm/values/values-readreplica.yaml
 .github/issue_label_bot.yaml
 .github/kubelinter.yaml
-.github/labeler.yml
+.github/labeler.yaml
 .github/valdrelease/valdrelease.yaml
-.github/workflows/_detect-ci-container.yml
+.github/workflows/_detect-ci-container.yaml
 .github/workflows/_docker-image-scan.yaml
 .github/workflows/_docker-image.yaml
-.github/workflows/_release-pr.yml
+.github/workflows/_release-pr.yaml
 .github/workflows/_update-protobuf.yaml
-.github/workflows/backport.yml
-.github/workflows/build-binaries.yml
-.github/workflows/build-protobuf.yml
-.github/workflows/chatops-help.yml
-.github/workflows/chatops.yml
-.github/workflows/check-conflict.yml
-.github/workflows/codeql-analysis.yml
-.github/workflows/coverage.yml
-.github/workflows/detect-internal-config-changes.yml
-.github/workflows/dockers-agent-faiss-image.yml
-.github/workflows/dockers-agent-image.yml
-.github/workflows/dockers-agent-ngt-image.yml
-.github/workflows/dockers-agent-sidecar-image.yml
-.github/workflows/dockers-benchmark-job-image.yml
+.github/workflows/backport.yaml
+.github/workflows/build-binaries.yaml
+.github/workflows/build-protobuf.yaml
+.github/workflows/chatops-help.yaml
+.github/workflows/chatops.yaml
+.github/workflows/check-conflict.yaml
+.github/workflows/codeql-analysis.yaml
+.github/workflows/coverage.yaml
+.github/workflows/detect-internal-config-changes.yaml
+.github/workflows/dockers-agent-faiss-image.yaml
+.github/workflows/dockers-agent-image.yaml
+.github/workflows/dockers-agent-ngt-image.yaml
+.github/workflows/dockers-agent-sidecar-image.yaml
+.github/workflows/dockers-benchmark-job-image.yaml
 .github/workflows/dockers-benchmark-operator-image.yaml
 .github/workflows/dockers-binfmt-image.yaml
-.github/workflows/dockers-buildbase-image.yml
+.github/workflows/dockers-buildbase-image.yaml
 .github/workflows/dockers-buildkit-image.yaml
 .github/workflows/dockers-buildkit-syft-scanner-image.yaml
-.github/workflows/dockers-ci-container-image.yml
-.github/workflows/dockers-dev-container-image.yml
-.github/workflows/dockers-discoverer-k8s-image.yml
-.github/workflows/dockers-gateway-filter-image.yml
-.github/workflows/dockers-gateway-lb-image.yml
+.github/workflows/dockers-ci-container-image.yaml
+.github/workflows/dockers-dev-container-image.yaml
+.github/workflows/dockers-discoverer-k8s-image.yaml
+.github/workflows/dockers-gateway-filter-image.yaml
+.github/workflows/dockers-gateway-lb-image.yaml
 .github/workflows/dockers-gateway-mirror-image.yaml
-.github/workflows/dockers-helm-operator-image.yml
-.github/workflows/dockers-image-scan.yml
-.github/workflows/dockers-index-correction.yml
-.github/workflows/dockers-index-creation.yml
-.github/workflows/dockers-index-operator.yml
-.github/workflows/dockers-index-save.yml
-.github/workflows/dockers-loadtest-image.yml
-.github/workflows/dockers-manager-index-image.yml
-.github/workflows/dockers-readreplica-rotate.yml
+.github/workflows/dockers-helm-operator-image.yaml
+.github/workflows/dockers-image-scan.yaml
+.github/workflows/dockers-index-correction-image.yaml
+.github/workflows/dockers-index-creation-image.yaml
+.github/workflows/dockers-index-operator-image.yaml
+.github/workflows/dockers-index-save-image.yaml
+.github/workflows/dockers-loadtest-image.yaml
+.github/workflows/dockers-manager-index-image.yaml
+.github/workflows/dockers-readreplica-rotate-image.yaml
 .github/workflows/dockers-release-branch-image.yaml
 .github/workflows/e2e-chaos.yaml
 .github/workflows/e2e-code-bench-agent.yaml
-.github/workflows/e2e-max-dim.yml
-.github/workflows/e2e-profiling.yml
-.github/workflows/e2e.yml
-.github/workflows/format.yml
-.github/workflows/fossa.yml
-.github/workflows/helm-lint.yml
-.github/workflows/helm.yml
+.github/workflows/e2e-max-dim.yaml
+.github/workflows/e2e-profiling.yaml
+.github/workflows/e2e.yaml
+.github/workflows/format.yaml
+.github/workflows/fossa.yaml
+.github/workflows/helm-lint.yaml
+.github/workflows/helm.yaml
 .github/workflows/issue-metrics.yaml
-.github/workflows/labeler.yml
-.github/workflows/release.yml
-.github/workflows/reviewdog-hadolint.yml
-.github/workflows/reviewdog-k8s.yml
-.github/workflows/reviewdog-markdown.yml
-.github/workflows/reviewdog.yml
+.github/workflows/labeler.yaml
+.github/workflows/release.yaml
+.github/workflows/reviewdog-hadolint.yaml
+.github/workflows/reviewdog-k8s.yaml
+.github/workflows/reviewdog-markdown.yaml
+.github/workflows/reviewdog.yaml
 .github/workflows/semver-major-minor.yaml
 .github/workflows/semver-patch.yaml
-.github/workflows/test-hack.yml
+.github/workflows/test-hack.yaml
 .github/workflows/unit-test.yaml
 .github/workflows/update-actions.yaml
 .github/workflows/update-protobuf.yaml
-.github/workflows/update-pull-request-and-issue-template.yml
-.github/workflows/update-web-docs.yml
+.github/workflows/update-pull-request-and-issue-template.yaml
+.github/workflows/update-web-docs.yaml
 .gitignore
 .golangci.yml
 .prh.yaml
@@ -140,7 +140,6 @@ Makefile
 Makefile.d/actions.mk
 Makefile.d/bench.mk
 Makefile.d/build.mk
-Makefile.d/client.mk
 Makefile.d/dependencies.mk
 Makefile.d/docker.mk
 Makefile.d/e2e.mk
@@ -511,6 +510,7 @@ cmd/tools/cli/loadtest/main_test.go
 cmd/tools/cli/loadtest/sample.yaml
 cmd/tools/cli/vdctl/main.go
 cmd/tools/cli/vdctl/main_test.go
+codecov.yaml
 design/Vald Architecture Assets.drawio
 design/Vald Architecture Dataflow.drawio
 design/Vald Architecture Overview.drawio
@@ -625,6 +625,7 @@ example/manifest/scylla/job.yaml
 go.mod
 go.sum
 hack/CHANGELOG.template.md
+hack/actions/gen/main.go
 hack/benchmark/assets/checksum/fashion-mnist-784-euclidean.md5
 hack/benchmark/assets/checksum/gist-960-euclidean.md5
 hack/benchmark/assets/checksum/glove-100-angular.md5
@@ -698,6 +699,8 @@ hack/benchmark/internal/starter/gateway/vald/vald_test.go
 hack/benchmark/internal/starter/starter.go
 hack/benchmark/metrics/metrics.go
 hack/benchmark/src/singleflight/singleflight_bench_test.go
+hack/cspell/main.go
+hack/cspell/main_test.go
 hack/docker/gen/main.go
 hack/git/hooks/pre-commit
 hack/go.mod.default
@@ -1422,6 +1425,15 @@ k8s/gateway/gateway/lb/hpa.yaml
 k8s/gateway/gateway/lb/pdb.yaml
 k8s/gateway/gateway/lb/priorityclass.yaml
 k8s/gateway/gateway/lb/svc.yaml
+k8s/gateway/gateway/mirror/clusterrole.yaml
+k8s/gateway/gateway/mirror/clusterrolebinding.yaml
+k8s/gateway/gateway/mirror/configmap.yaml
+k8s/gateway/gateway/mirror/deployment.yaml
+k8s/gateway/gateway/mirror/hpa.yaml
+k8s/gateway/gateway/mirror/pdb.yaml
+k8s/gateway/gateway/mirror/priorityclass.yaml
+k8s/gateway/gateway/mirror/serviceaccount.yaml
+k8s/gateway/gateway/mirror/svc.yaml
 k8s/index/job/correction/configmap.yaml
 k8s/index/job/correction/cronjob.yaml
 k8s/index/job/creation/configmap.yaml
@@ -1594,6 +1606,8 @@ pkg/agent/internal/kvs/kvs.go
 pkg/agent/internal/kvs/kvs_test.go
 pkg/agent/internal/kvs/option.go
 pkg/agent/internal/kvs/option_test.go
+pkg/agent/internal/memstore/data_manager.go
+pkg/agent/internal/memstore/data_manager_test.go
 pkg/agent/internal/metadata/metadata.go
 pkg/agent/internal/metadata/metadata_test.go
 pkg/agent/internal/vqueue/option.go
@@ -1897,13 +1911,20 @@ rust/bin/agent/src/handler/search.rs
 rust/bin/agent/src/handler/update.rs
 rust/bin/agent/src/handler/upsert.rs
 rust/bin/agent/src/main.rs
-rust/libs/ngt-rs/Cargo.toml
-rust/libs/ngt-rs/build.rs
-rust/libs/ngt-rs/src/input.cpp
-rust/libs/ngt-rs/src/input.h
-rust/libs/ngt-rs/src/lib.rs
-rust/libs/ngt/Cargo.toml
-rust/libs/ngt/src/lib.rs
+rust/libs/algorithm/Cargo.toml
+rust/libs/algorithm/src/lib.rs
+rust/libs/algorithms/faiss/Cargo.toml
+rust/libs/algorithms/faiss/src/lib.rs
+rust/libs/algorithms/ngt/Cargo.toml
+rust/libs/algorithms/ngt/build.rs
+rust/libs/algorithms/ngt/src/input.cpp
+rust/libs/algorithms/ngt/src/input.h
+rust/libs/algorithms/ngt/src/lib.rs
+rust/libs/observability/Cargo.toml
+rust/libs/observability/src/config.rs
+rust/libs/observability/src/lib.rs
+rust/libs/observability/src/macros.rs
+rust/libs/observability/src/observability.rs
 rust/libs/proto/Cargo.toml
 rust/libs/proto/src/core.v1.tonic.rs
 rust/libs/proto/src/discoverer.v1.tonic.rs
@@ -1965,7 +1986,6 @@ versions/PROTOBUF_VERSION
 versions/REVIEWDOG_VERSION
 versions/RUST_VERSION
 versions/TELEPRESENCE_VERSION
-versions/VALDCLI_VERSION
 versions/VALD_VERSION
 versions/YQ_VERSION
 versions/ZLIB_VERSION

.github/ISSUE_TEMPLATE/bug_report.md

@@ -23,10 +23,10 @@ assignees: ""
 <!--- Please change the versions below along with your environment -->
 
 - Vald Version: v1.7.13
-- Go Version: v1.23.0
-- Rust Version: v1.80.0
-- Docker Version: v27.1.1
-- Kubernetes Version: v1.30.3
-- Helm Version: v3.15.3
+- Go Version: v1.23.1
+- Rust Version: v1.81.0
+- Docker Version: v27.2.1
+- Kubernetes Version: v1.31.0
+- Helm Version: v3.16.0
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0

.github/ISSUE_TEMPLATE/security_issue_report.md

@@ -17,10 +17,10 @@ assignees: ""
 <!--- Please change the versions below along with your environment -->
 
 - Vald Version: v1.7.13
-- Go Version: v1.23.0
-- Rust Version: v1.80.0
-- Docker Version: v27.1.1
-- Kubernetes Version: v1.30.3
-- Helm Version: v3.15.3
+- Go Version: v1.23.1
+- Rust Version: v1.81.0
+- Docker Version: v27.2.1
+- Kubernetes Version: v1.31.0
+- Helm Version: v3.16.0
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,11 +16,11 @@
 
 <!--- Please change the versions below along with your environment -->
 - Vald Version: v1.7.13
-- Go Version: v1.23.0
-- Rust Version: v1.80.0
-- Docker Version: v27.1.1
-- Kubernetes Version: v1.30.3
-- Helm Version: v3.15.3
+- Go Version: v1.23.1
+- Rust Version: v1.81.0
+- Docker Version: v27.2.1
+- Kubernetes Version: v1.31.0
+- Helm Version: v3.16.0
 - NGT Version: v2.2.4
 - Faiss Version: v1.8.0
 

.github/helm/values/values-correction.yaml

@@ -16,7 +16,7 @@
 
 defaults:
   logging:
-    level: info
+    level: debug
   networkPolicy:
     enabled: true
 gateway:

.github/workflows/_release-pr.yml → .github/workflows/_release-pr.yaml

@@ -32,7 +32,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
   detect-ci-container:
-    uses: ./.github/workflows/_detect-ci-container.yml
+    uses: ./.github/workflows/_detect-ci-container.yaml
     secrets: inherit
   create:
     needs:

.github/workflows/build-binaries.yml → .github/workflows/build-binaries.yaml

@@ -25,7 +25,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
   detect-ci-container:
-    uses: ./.github/workflows/_detect-ci-container.yml
+    uses: ./.github/workflows/_detect-ci-container.yaml
   build-linux:
     runs-on: ubuntu-latest
     needs: [detect-ci-container]

.github/workflows/build-protobuf.yml → .github/workflows/build-protobuf.yaml

@@ -34,7 +34,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
   detect-ci-container:
-    uses: ./.github/workflows/_detect-ci-container.yml
+    uses: ./.github/workflows/_detect-ci-container.yaml
   build:
     runs-on: ubuntu-latest
     needs: [detect-ci-container]

.github/workflows/check-conflict.yml → .github/workflows/check-conflict.yaml

@@ -35,7 +35,7 @@ jobs:
           git config --global --add safe.directory ${GITHUB_WORKSPACE}
       - name: Check conflict
         run: |
-          if grep -r "<<<< HEAD" . --exclude-dir=.git --exclude=check-conflict.yml; then
+          if grep -r "<<<< HEAD" . --exclude-dir=.git --exclude=check-conflict.yaml; then
             PR_COMMENTS=`curl ${API_URL}?per_page=10000`
             BODY=`echo -E "${PR_COMMENTS}" | jq 'last(.[] | select(.user.login == "vdaas-ci") | select(.body | test("^\\\\*\\\\*\\\\[WARNING:CONFLICT")) | .body)' -r`
 

.github/workflows/codeql-analysis.yml → .github/workflows/codeql-analysis.yaml

@@ -31,7 +31,7 @@ on:
       - "release/v*.*"
       - "!release/v*.*.*"
     paths:
-      - ".github/workflows/codeql-analysis.yml"
+      - ".github/workflows/codeql-analysis.yaml"
       - "**.go"
   schedule:
     - cron: "0 1 * * *"
@@ -42,7 +42,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
   detect-ci-container:
-    uses: ./.github/workflows/_detect-ci-container.yml
+    uses: ./.github/workflows/_detect-ci-container.yaml
   codeql-build:
     name: CodeQL
     runs-on: ubuntu-latest

.github/workflows/coverage.yml → .github/workflows/coverage.yaml

@@ -19,13 +19,13 @@ on:
     branches:
       - main
     paths:
-      - ".github/workflows/coverage.yml"
+      - ".github/workflows/coverage.yaml"
       - "internal/**"
       - "pkg/**"
       - "cmd/**"
   pull_request:
     paths:
-      - ".github/workflows/coverage.yml"
+      - ".github/workflows/coverage.yaml"
       - "internal/**"
       - "pkg/**"
       - "cmd/**"
@@ -36,7 +36,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/dump-context
   detect-ci-container:
-    uses: ./.github/workflows/_detect-ci-container.yml
+    uses: ./.github/workflows/_detect-ci-container.yaml
   coverage:
     runs-on: ubuntu-latest
     needs: [detect-ci-container]