Add rollout restart agent e2e test

.github/actions/setup-e2e/action.yaml

@@ -32,6 +32,10 @@ inputs:
     description: "If k3d is not required, set this to false"
     required: false
     default: "true"
+  require_k3d_storage:
+    description: "If local-path-storage is not required, set this to false"
+    required: false
+    default: "false"
   require_minikube:
     description: "If minikube is not required, set this to true and set require_k3d to false"
     required: false
@@ -85,6 +89,7 @@ runs:
       with:
         agents: 3
         ingress_port: ${{ inputs.ingress_port }}
+        storage: ${{ inputs.require_k3d_storage }}
     - name: Setup Minikube environment
       if: ${{ inputs.require_minikube == 'true' }}
       shell: bash

.github/actions/setup-k3d/action.yaml

@@ -36,6 +36,10 @@ inputs:
     description: "Number of agents"
     required: false
     default: "3"
+  storage:
+    description: "If true, the local-path-storage will be deployed"
+    required: false
+    default: "false"
   options:
     description: "Options for k3d cluster create command"
     required: false
@@ -124,6 +128,11 @@ runs:
         echo $KUBECONFIG
         cat $KUBECONFIG
         cat /etc/hosts
+    - name: Set local path storage
+      if: ${{ inputs.storage == 'true' }}
+      shell: bash
+      run: |
+        make k3d/storage
     - name: Show Kubernetes Cluster Info
       shell: bash
       run: |

.github/helm/values/values-rollout-agent.yaml

@@ -0,0 +1,84 @@
+#
+# Copyright (C) 2019-2025 vdaas.org vald team <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+defaults:
+  logging:
+    level: debug
+  networkPolicy:
+    enabled: true
+gateway:
+  lb:
+    enabled: true
+    minReplicas: 1
+    hpa:
+      enabled: false
+    resources:
+      requests:
+        cpu: 100m
+        memory: 50Mi
+    gateway_config:
+      index_replica: 2
+agent:
+  minReplicas: 3
+  maxReplicas: 3
+  podManagementPolicy: Parallel
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+  # We recommend to set this value long enough to ensure the backup speed of PV, since the Index is backed up at the end of the pod.
+  terminationGracePeriodSeconds: 600
+  # This is the persistent volume settings.
+  # Please change it according to your environment.
+  persistentVolume:
+    enabled: true
+    accessMode: ReadWriteOncePod
+    storageClass: local-path
+    size: 500Mi
+  ngt:
+    auto_index_duration_limit: 2m
+    auto_index_check_duration: 30s
+    auto_index_length: 500
+    dimension: 784
+    enable_in_memory_mode: false
+    index_path: "/var/ngt/index"
+discoverer:
+  minReplicas: 1
+  hpa:
+    enabled: false
+  resources:
+    requests:
+      cpu: 100m
+      memory: 50Mi
+manager:
+  index:
+    replicas: 1
+    resources:
+      requests:
+        cpu: 100m
+        memory: 30Mi
+    indexer:
+      auto_index_duration_limit: 2m
+      auto_index_check_duration: 30s
+      auto_index_length: 1000
+    corrector:
+      enabled: true
+      # suspend because you do not want corrector to start automatically in CI
+      # instead run it manually
+      suspend: true
+      schedule: "1 2 3 4 5"

.github/workflows/dockers-agent-ngt-image.yaml

@@ -269,4 +269,5 @@ jobs:
     uses: ./.github/workflows/_docker-image.yaml
     with:
       target: agent-ngt
+      platforms: linux/amd64,linux/arm64
     secrets: inherit

.github/workflows/e2e.yaml

@@ -348,6 +348,49 @@ jobs:
             e2e/readreplica
         env:
           POD_NAME: ${{ steps.deploy_vald_readreplica.outputs.POD_NAME }}
+  e2e-stream-crud-with-rollout-restart-agent:
+    name: "E2E test (Stream CRUD) with rollout restart agent"
+    needs: [detect-ci-container]
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    container:
+      image: ghcr.io/vdaas/vald/vald-ci-container:${{ needs.detect-ci-container.outputs.TAG }}
+      options: "--add-host host.docker.internal:host-gateway"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set Git config
+        run: |
+          git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - name: Setup E2E environment
+        id: setup_e2e
+        uses: ./.github/actions/setup-e2e
+        with:
+          require_k3d_storage: true
+      - name: Deploy Vald
+        id: deploy_vald
+        uses: ./.github/actions/e2e-deploy-vald
+        with:
+          helm_extra_options: ${{ steps.setup_e2e.outputs.HELM_EXTRA_OPTIONS }}
+          values: .github/helm/values/values-rollout-agent.yaml
+          wait_for_selector: app=vald-lb-gateway
+      - name: Run E2E CRUD with rollout restart agent
+        run: |
+          make hack/benchmark/assets/dataset/${{ env.DATASET }}
+          make E2E_BIND_PORT=8081 \
+            E2E_DATASET_NAME=${{ env.DATASET }} \
+            E2E_INSERT_COUNT=1000 \
+            E2E_SEARCH_COUNT=1000 \
+            E2E_SEARCH_BY_ID_COUNT=1000 \
+            E2E_GET_OBJECT_COUNT=100 \
+            E2E_UPDATE_COUNT=100 \
+            E2E_UPSERT_COUNT=100 \
+            E2E_REMOVE_COUNT=100 \
+            E2E_WAIT_FOR_CREATE_INDEX_DURATION=3m \
+            E2E_TARGET_POD_NAME=${POD_NAME} \
+            E2E_TARGET_NAMESPACE=default \
+            e2e/rollout/restart/agent
+        env:
+          POD_NAME: ${{ steps.deploy_vald.outputs.POD_NAME }}
   e2e-stream-crud-with-mirror:
     name: "E2E test (Stream CRUD) with mirror"
     needs: [detect-ci-container]
@@ -417,6 +460,7 @@ jobs:
       - e2e-stream-crud-under-index-management-jobs
       - e2e-stream-crud-with-mirror
       - e2e-stream-crud-with-readreplica
+      - e2e-stream-crud-with-rollout-restart-agent
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

Makefile

@@ -367,6 +367,7 @@ E2E_UPDATE_COUNT                   ?= 10
 E2E_UPSERT_COUNT                   ?= 10
 E2E_WAIT_FOR_CREATE_INDEX_DURATION ?= 8m
 E2E_WAIT_FOR_START_TIMEOUT         ?= 10m
+E2E_WAIT_FOR_RESOURCE_READY        ?= 3m
 E2E_SEARCH_FROM                    ?= 0
 E2E_SEARCH_BY_ID_FROM              ?= 0
 E2E_INSERT_FROM                    ?= 0

Makefile.d/e2e.mk

@@ -89,6 +89,11 @@ e2e/index/job/correction:
 e2e/readreplica:
 	$(call run-e2e-crud-test,-run TestE2EReadReplica)
 
+.PHONY: e2e/rollaout/restart/agent
+## run rollout-restart agent e2e
+e2e/rollout/restart/agent:
+	$(call run-e2e-crud-test,-run TestE2EAgentRolloutRestart)
+
 .PHONY: e2e/maxdim
 ## run e2e/maxdim
 e2e/maxdim:

Makefile.d/functions.mk

@@ -163,6 +163,7 @@ define run-e2e-crud-test
 	    -search-by-id-from=$(E2E_SEARCH_BY_ID_FROM) \
 	    -get-object-from=$(E2E_GET_OBJECT_FROM) \
 	    -wait-after-insert=$(E2E_WAIT_FOR_CREATE_INDEX_DURATION) \
+	    -wait-resource-ready=$(E2E_WAIT_FOR_RESOURCE_READY) \
 	    -portforward=$(E2E_PORTFORWARD_ENABLED) \
 	    -portforward-pod-name=$(E2E_TARGET_POD_NAME) \
 	    -portforward-pod-port=$(E2E_TARGET_PORT) \

dockers/agent/core/agent/Dockerfile

@@ -94,4 +94,4 @@ LABEL maintainer="vdaas.org vald team <[email protected]>"
 COPY --from=builder /usr/bin/agent /usr/bin/agent
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/agent"]
+ENTRYPOINT ["/usr/bin/agent"]

dockers/agent/core/faiss/Dockerfile

@@ -96,4 +96,4 @@ COPY --from=builder /usr/bin/faiss /usr/bin/faiss
 COPY cmd/agent/core/faiss/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/faiss"]
+ENTRYPOINT ["/usr/bin/faiss"]

dockers/agent/core/ngt/Dockerfile

@@ -95,4 +95,4 @@ COPY --from=builder /usr/bin/ngt /usr/bin/ngt
 COPY cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/ngt"]
+ENTRYPOINT ["/usr/bin/ngt"]

dockers/agent/sidecar/Dockerfile

@@ -85,4 +85,4 @@ LABEL maintainer="vdaas.org vald team <[email protected]>"
 COPY --from=builder /usr/bin/sidecar /usr/bin/sidecar
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/sidecar"]
+ENTRYPOINT ["/usr/bin/sidecar"]

dockers/binfmt/Dockerfile

@@ -17,4 +17,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM tonistiigi/binfmt:master AS builder
+FROM tonistiigi/binfmt:master AS builder

dockers/buildbase/Dockerfile

@@ -17,4 +17,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM ubuntu:devel AS builder
+FROM ubuntu:devel AS builder

dockers/buildkit/Dockerfile

@@ -17,4 +17,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM moby/buildkit:master AS builder
+FROM moby/buildkit:master AS builder

dockers/buildkit/syft/scanner/Dockerfile

@@ -17,4 +17,4 @@
 #
 
 # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go
-FROM docker/buildkit-syft-scanner:edge AS scanner
+FROM docker/buildkit-syft-scanner:edge AS scanner

dockers/ci/base/Dockerfile

@@ -128,4 +128,4 @@ RUN --mount=type=bind,target=.,rw \
     && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
 # skipcq: DOK-DL3002
 USER root:root
-ENTRYPOINT ["/bin/bash"]
+ENTRYPOINT ["/bin/bash"]

dockers/dev/Dockerfile

@@ -141,4 +141,4 @@ RUN --mount=type=bind,target=.,rw \
     && make faiss/install \
     && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/*
 # skipcq: DOK-DL3002
-USER root:root
+USER root:root

dockers/discoverer/k8s/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/discoverer /usr/bin/discoverer
 COPY cmd/discoverer/k8s/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/discoverer"]
+ENTRYPOINT ["/usr/bin/discoverer"]

dockers/example/client/Dockerfile

@@ -93,4 +93,4 @@ LABEL maintainer="vdaas.org vald team <[email protected]>"
 COPY --from=builder /usr/bin/client /usr/bin/client
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/client"]
+ENTRYPOINT ["/usr/bin/client"]

dockers/gateway/filter/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/filter /usr/bin/filter
 COPY cmd/gateway/filter/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/filter"]
+ENTRYPOINT ["/usr/bin/filter"]

dockers/gateway/lb/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/lb /usr/bin/lb
 COPY cmd/gateway/lb/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/lb"]
+ENTRYPOINT ["/usr/bin/lb"]

dockers/gateway/mirror/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/mirror /usr/bin/mirror
 COPY cmd/gateway/mirror/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/mirror"]
+ENTRYPOINT ["/usr/bin/mirror"]

dockers/index/job/correction/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/index-correction /usr/bin/index-correction
 COPY cmd/index/job/correction/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-correction"]
+ENTRYPOINT ["/usr/bin/index-correction"]

dockers/index/job/creation/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/index-creation /usr/bin/index-creation
 COPY cmd/index/job/creation/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-creation"]
+ENTRYPOINT ["/usr/bin/index-creation"]

dockers/index/job/deletion/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/index-deletion /usr/bin/index-deletion
 COPY cmd/index/job/deletion/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-deletion"]
+ENTRYPOINT ["/usr/bin/index-deletion"]

dockers/index/job/readreplica/rotate/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/readreplica-rotate /usr/bin/readreplica-rotate
 COPY cmd/index/job/readreplica/rotate/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/readreplica-rotate"]
+ENTRYPOINT ["/usr/bin/readreplica-rotate"]

dockers/index/job/save/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/index-save /usr/bin/index-save
 COPY cmd/index/job/save/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-save"]
+ENTRYPOINT ["/usr/bin/index-save"]

dockers/index/operator/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/index-operator /usr/bin/index-operator
 COPY cmd/index/operator/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index-operator"]
+ENTRYPOINT ["/usr/bin/index-operator"]

dockers/manager/index/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/index /usr/bin/index
 COPY cmd/manager/index/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/index"]
+ENTRYPOINT ["/usr/bin/index"]

dockers/operator/helm/Dockerfile

@@ -107,4 +107,4 @@ COPY --from=builder /opt/helm/charts/vald /opt/helm/charts/vald
 COPY --from=builder /opt/helm/charts/vald-helm-operator /opt/helm/charts/vald-helm-operator
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"]
+ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"]

dockers/tools/benchmark/job/Dockerfile

@@ -94,4 +94,4 @@ COPY --from=builder /usr/bin/job /usr/bin/job
 COPY cmd/tools/benchmark/job/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/job"]
+ENTRYPOINT ["/usr/bin/job"]

dockers/tools/benchmark/operator/Dockerfile

@@ -86,4 +86,4 @@ COPY --from=builder /usr/bin/operator /usr/bin/operator
 COPY cmd/tools/benchmark/operator/sample.yaml /etc/server/config.yaml
 # skipcq: DOK-DL3002
 USER nonroot:nonroot
-ENTRYPOINT ["/usr/bin/operator"]
+ENTRYPOINT ["/usr/bin/operator"]