diff --git a/.gitfiles b/.gitfiles index 64f792a39c..56c5bfe305 100644 --- a/.gitfiles +++ b/.gitfiles @@ -140,7 +140,6 @@ Makefile Makefile.d/actions.mk Makefile.d/bench.mk Makefile.d/build.mk -Makefile.d/client.mk Makefile.d/dependencies.mk Makefile.d/docker.mk Makefile.d/e2e.mk @@ -1904,6 +1903,11 @@ rust/libs/ngt-rs/src/input.h rust/libs/ngt-rs/src/lib.rs rust/libs/ngt/Cargo.toml rust/libs/ngt/src/lib.rs +rust/libs/observability/Cargo.toml +rust/libs/observability/src/config.rs +rust/libs/observability/src/lib.rs +rust/libs/observability/src/macros.rs +rust/libs/observability/src/observability.rs rust/libs/proto/Cargo.toml rust/libs/proto/src/core.v1.tonic.rs rust/libs/proto/src/discoverer.v1.tonic.rs @@ -1965,7 +1969,6 @@ versions/PROTOBUF_VERSION versions/REVIEWDOG_VERSION versions/RUST_VERSION versions/TELEPRESENCE_VERSION -versions/VALDCLI_VERSION versions/VALD_VERSION versions/YQ_VERSION versions/ZLIB_VERSION diff --git a/.github/helm/values/values-correction.yaml b/.github/helm/values/values-correction.yaml index 888931ca6f..0632c3d2f7 100644 --- a/.github/helm/values/values-correction.yaml +++ b/.github/helm/values/values-correction.yaml @@ -16,7 +16,7 @@ defaults: logging: - level: info + level: debug networkPolicy: enabled: true gateway: diff --git a/Makefile b/Makefile index 4a5e001f71..edc14dbba5 100644 --- a/Makefile +++ b/Makefile @@ -103,7 +103,6 @@ PROTOBUF_VERSION := $(eval PROTOBUF_VERSION := $(shell cat versions/PRO REVIEWDOG_VERSION := $(eval REVIEWDOG_VERSION := $(shell cat versions/REVIEWDOG_VERSION))$(REVIEWDOG_VERSION) RUST_VERSION := $(eval RUST_VERSION := $(shell cat versions/RUST_VERSION))$(RUST_VERSION) TELEPRESENCE_VERSION := $(eval TELEPRESENCE_VERSION := $(shell cat versions/TELEPRESENCE_VERSION))$(TELEPRESENCE_VERSION) -VALDCLI_VERSION := $(eval VALDCLI_VERSION := $(shell cat versions/VALDCLI_VERSION))$(VALDCLI_VERSION) YQ_VERSION := $(eval YQ_VERSION := $(shell cat versions/YQ_VERSION))$(YQ_VERSION) ZLIB_VERSION := $(eval ZLIB_VERSION := $(shell cat versions/ZLIB_VERSION))$(ZLIB_VERSION) @@ -455,7 +454,6 @@ init: \ tools/install: \ helm/install \ kind/install \ - valdcli/install \ telepresence/install \ textlint/install @@ -620,10 +618,6 @@ version/helm: version/yq: @echo $(YQ_VERSION) -.PHONY: version/valdcli -version/valdcli: - @echo $(VALDCLI_VERSION) - .PHONY: version/telepresence version/telepresence: @echo $(TELEPRESENCE_VERSION) @@ -766,7 +760,6 @@ changelog/next/print: include Makefile.d/actions.mk include Makefile.d/bench.mk include Makefile.d/build.mk -include Makefile.d/client.mk include Makefile.d/dependencies.mk include Makefile.d/docker.mk include Makefile.d/e2e.mk @@ -775,7 +768,7 @@ include Makefile.d/helm.mk include Makefile.d/k3d.mk include Makefile.d/k8s.mk include Makefile.d/kind.mk +include Makefile.d/minikube.mk include Makefile.d/proto.mk include Makefile.d/test.mk include Makefile.d/tools.mk -include Makefile.d/minikube.mk diff --git a/Makefile.d/client.mk b/Makefile.d/client.mk deleted file mode 100644 index 1c5c7aa9bf..0000000000 --- a/Makefile.d/client.mk +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright (C) 2019-2024 vdaas.org vald team -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -.PHONY: valdcli/install -## install valdcli -valdcli/install: $(BINDIR)/valdcli - -ifeq ($(UNAME),Darwin) -$(BINDIR)/valdcli: - mkdir -p $(BINDIR) - curl -fsSLO https://github.com/rinx/vald-client-clj/releases/download/$(VALDCLI_VERSION)/valdcli-macos.zip - unzip valdcli-macos.zip - rm -f valdcli-macos.zip - mv valdcli $(BINDIR)/valdcli -else -$(BINDIR)/valdcli: - mkdir -p $(BINDIR) - curl -fsSLO https://github.com/rinx/vald-client-clj/releases/download/$(VALDCLI_VERSION)/valdcli-linux-static.zip - unzip valdcli-linux-static.zip - rm -f valdcli-linux-static.zip - mv valdcli $(BINDIR)/valdcli -endif - -.PHONY: valdcli/xpanes/insert -## insert randomized vectors using valdcli and xpanes -valdcli/xpanes/insert: - xpanes -c "valdcli rand-vecs -n $(NUMBER) -d $(DIMENSION) --gaussian --gaussian-mean $(MEAN) --gaussian-stddev $(STDDEV) --with-ids | valdcli -h $(HOST) -p $(PORT) stream-insert --elapsed-time" $$(seq 1 $(NUMPANES)) - -.PHONY: valdcli/xpanes/search -## search randomized vectors using valdcli and xpanes -valdcli/xpanes/search: - xpanes -c "valdcli rand-vecs -n $(NUMBER) -d $(DIMENSION) --gaussian --gaussian-mean $(MEAN) --gaussian-stddev $(STDDEV) | valdcli -h $(HOST) -p $(PORT) stream-search --elapsed-time" $$(seq 1 $(NUMPANES)) diff --git a/Makefile.d/dependencies.mk b/Makefile.d/dependencies.mk index 928ed8c9d2..596e9ec1b2 100644 --- a/Makefile.d/dependencies.mk +++ b/Makefile.d/dependencies.mk @@ -39,7 +39,6 @@ update/libs: \ update/rust \ update/telepresence \ update/vald \ - update/valdcli \ update/yq \ update/zlib @@ -227,11 +226,6 @@ update/hdf5: update/vald: curl -fsSL https://api.github.com/repos/$(REPO)/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALD_VERSION -.PHONY: update/valdcli -## update vald client library made by clojure self version -update/valdcli: - curl -fsSL https://api.github.com/repos/$(REPO)-client-clj/releases/latest | grep -Po '"tag_name": "\K.*?(?=")' > $(ROOTDIR)/versions/VALDCLI_VERSION - .PHONY: update/template ## update PULL_REQUEST_TEMPLATE and ISSUE_TEMPLATE update/template: diff --git a/README.md b/README.md index f383078d65..a03458bf5b 100755 --- a/README.md +++ b/README.md @@ -264,7 +264,9 @@ make init ## Contributors + [![All Contributors](https://img.shields.io/badge/all_contributors-18-orange.svg?style=flat-square)](#contributors) + Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): diff --git a/apis/docs/v1/docs.md b/apis/docs/v1/docs.md index 2289344848..08bf1d81d6 100644 --- a/apis/docs/v1/docs.md +++ b/apis/docs/v1/docs.md @@ -1664,11 +1664,12 @@ Search service provides ways to search indexed vectors. Update service provides ways to update indexed vectors. -| Method Name | Request Type | Response Type | Description | -| ------------ | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- | -| Update | [.payload.v1.Update.Request](#payload-v1-Update-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update an indexed vector. | -| StreamUpdate | [.payload.v1.Update.Request](#payload-v1-Update-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update multiple indexed vectors by bidirectional streaming. | -| MultiUpdate | [.payload.v1.Update.MultiRequest](#payload-v1-Update-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple indexed vectors in a single request. | +| Method Name | Request Type | Response Type | Description | +| --------------- | ------------------------------------------------------------------ | ----------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| Update | [.payload.v1.Update.Request](#payload-v1-Update-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update an indexed vector. | +| StreamUpdate | [.payload.v1.Update.Request](#payload-v1-Update-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update multiple indexed vectors by bidirectional streaming. | +| MultiUpdate | [.payload.v1.Update.MultiRequest](#payload-v1-Update-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple indexed vectors in a single request. | +| UpdateTimestamp | [.payload.v1.Object.Timestamp](#payload-v1-Object-Timestamp) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update timestamp an indexed vector. | diff --git a/apis/grpc/v1/vald/update.pb.go b/apis/grpc/v1/vald/update.pb.go index ad05fbe50c..f98b9d1b26 100644 --- a/apis/grpc/v1/vald/update.pb.go +++ b/apis/grpc/v1/vald/update.pb.go @@ -46,7 +46,7 @@ var file_v1_vald_update_proto_rawDesc = []byte{ 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x18, 0x76, 0x31, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2f, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x9f, 0x02, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, + 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x32, 0x81, 0x03, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x55, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, @@ -64,32 +64,41 @@ var file_v1_vald_update_proto_rawDesc = []byte{ 0x2e, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x1b, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x15, 0x3a, 0x01, 0x2a, 0x22, 0x10, 0x2f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, - 0x2f, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x42, 0x53, 0x0a, 0x1a, 0x6f, 0x72, 0x67, - 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x2e, 0x61, 0x70, 0x69, 0x2e, - 0x76, 0x31, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x42, 0x0a, 0x56, 0x61, 0x6c, 0x64, 0x55, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x50, 0x01, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61, 0x70, 0x69, - 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x31, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2f, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x12, 0x60, 0x0a, 0x0f, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x1c, 0x2e, 0x70, + 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x1a, 0x1b, 0x2e, 0x70, 0x61, 0x79, + 0x6c, 0x6f, 0x61, 0x64, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x2e, 0x4c, + 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x12, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0c, 0x3a, + 0x01, 0x2a, 0x22, 0x07, 0x2f, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x42, 0x53, 0x0a, 0x1a, 0x6f, + 0x72, 0x67, 0x2e, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x2e, 0x61, 0x70, + 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x76, 0x61, 0x6c, 0x64, 0x42, 0x0a, 0x56, 0x61, 0x6c, 0x64, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x50, 0x01, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x76, 0x64, 0x61, 0x61, 0x73, 0x2f, 0x76, 0x61, 0x6c, 0x64, 0x2f, 0x61, + 0x70, 0x69, 0x73, 0x2f, 0x67, 0x72, 0x70, 0x63, 0x2f, 0x76, 0x31, 0x2f, 0x76, 0x61, 0x6c, 0x64, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var file_v1_vald_update_proto_goTypes = []any{ (*payload.Update_Request)(nil), // 0: payload.v1.Update.Request (*payload.Update_MultiRequest)(nil), // 1: payload.v1.Update.MultiRequest - (*payload.Object_Location)(nil), // 2: payload.v1.Object.Location - (*payload.Object_StreamLocation)(nil), // 3: payload.v1.Object.StreamLocation - (*payload.Object_Locations)(nil), // 4: payload.v1.Object.Locations + (*payload.Object_Timestamp)(nil), // 2: payload.v1.Object.Timestamp + (*payload.Object_Location)(nil), // 3: payload.v1.Object.Location + (*payload.Object_StreamLocation)(nil), // 4: payload.v1.Object.StreamLocation + (*payload.Object_Locations)(nil), // 5: payload.v1.Object.Locations } var file_v1_vald_update_proto_depIdxs = []int32{ 0, // 0: vald.v1.Update.Update:input_type -> payload.v1.Update.Request 0, // 1: vald.v1.Update.StreamUpdate:input_type -> payload.v1.Update.Request 1, // 2: vald.v1.Update.MultiUpdate:input_type -> payload.v1.Update.MultiRequest - 2, // 3: vald.v1.Update.Update:output_type -> payload.v1.Object.Location - 3, // 4: vald.v1.Update.StreamUpdate:output_type -> payload.v1.Object.StreamLocation - 4, // 5: vald.v1.Update.MultiUpdate:output_type -> payload.v1.Object.Locations - 3, // [3:6] is the sub-list for method output_type - 0, // [0:3] is the sub-list for method input_type + 2, // 3: vald.v1.Update.UpdateTimestamp:input_type -> payload.v1.Object.Timestamp + 3, // 4: vald.v1.Update.Update:output_type -> payload.v1.Object.Location + 4, // 5: vald.v1.Update.StreamUpdate:output_type -> payload.v1.Object.StreamLocation + 5, // 6: vald.v1.Update.MultiUpdate:output_type -> payload.v1.Object.Locations + 3, // 7: vald.v1.Update.UpdateTimestamp:output_type -> payload.v1.Object.Location + 4, // [4:8] is the sub-list for method output_type + 0, // [0:4] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name diff --git a/apis/grpc/v1/vald/update_vtproto.pb.go b/apis/grpc/v1/vald/update_vtproto.pb.go index 60f92fd0a8..6fc00c758c 100644 --- a/apis/grpc/v1/vald/update_vtproto.pb.go +++ b/apis/grpc/v1/vald/update_vtproto.pb.go @@ -48,6 +48,8 @@ type UpdateClient interface { StreamUpdate(ctx context.Context, opts ...grpc.CallOption) (Update_StreamUpdateClient, error) // A method to update multiple indexed vectors in a single request. MultiUpdate(ctx context.Context, in *payload.Update_MultiRequest, opts ...grpc.CallOption) (*payload.Object_Locations, error) + // A method to update timestamp an indexed vector. + UpdateTimestamp(ctx context.Context, in *payload.Object_Timestamp, opts ...grpc.CallOption) (*payload.Object_Location, error) } type updateClient struct { @@ -113,6 +115,17 @@ func (c *updateClient) MultiUpdate( return out, nil } +func (c *updateClient) UpdateTimestamp( + ctx context.Context, in *payload.Object_Timestamp, opts ...grpc.CallOption, +) (*payload.Object_Location, error) { + out := new(payload.Object_Location) + err := c.cc.Invoke(ctx, "/vald.v1.Update/UpdateTimestamp", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // UpdateServer is the server API for Update service. // All implementations must embed UnimplementedUpdateServer // for forward compatibility @@ -123,6 +136,8 @@ type UpdateServer interface { StreamUpdate(Update_StreamUpdateServer) error // A method to update multiple indexed vectors in a single request. MultiUpdate(context.Context, *payload.Update_MultiRequest) (*payload.Object_Locations, error) + // A method to update timestamp an indexed vector. + UpdateTimestamp(context.Context, *payload.Object_Timestamp) (*payload.Object_Location, error) mustEmbedUnimplementedUpdateServer() } @@ -144,6 +159,12 @@ func (UnimplementedUpdateServer) MultiUpdate( ) (*payload.Object_Locations, error) { return nil, status.Errorf(codes.Unimplemented, "method MultiUpdate not implemented") } + +func (UnimplementedUpdateServer) UpdateTimestamp( + context.Context, *payload.Object_Timestamp, +) (*payload.Object_Location, error) { + return nil, status.Errorf(codes.Unimplemented, "method UpdateTimestamp not implemented") +} func (UnimplementedUpdateServer) mustEmbedUnimplementedUpdateServer() {} // UnsafeUpdateServer may be embedded to opt out of forward compatibility for this service. @@ -223,6 +244,26 @@ func _Update_MultiUpdate_Handler( return interceptor(ctx, in, info, handler) } +func _Update_UpdateTimestamp_Handler( + srv any, ctx context.Context, dec func(any) error, interceptor grpc.UnaryServerInterceptor, +) (any, error) { + in := new(payload.Object_Timestamp) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(UpdateServer).UpdateTimestamp(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vald.v1.Update/UpdateTimestamp", + } + handler := func(ctx context.Context, req any) (any, error) { + return srv.(UpdateServer).UpdateTimestamp(ctx, req.(*payload.Object_Timestamp)) + } + return interceptor(ctx, in, info, handler) +} + // Update_ServiceDesc is the grpc.ServiceDesc for Update service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -238,6 +279,10 @@ var Update_ServiceDesc = grpc.ServiceDesc{ MethodName: "MultiUpdate", Handler: _Update_MultiUpdate_Handler, }, + { + MethodName: "UpdateTimestamp", + Handler: _Update_UpdateTimestamp_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/apis/grpc/v1/vald/vald.go b/apis/grpc/v1/vald/vald.go index a67e7c97ef..c1255fac42 100644 --- a/apis/grpc/v1/vald/vald.go +++ b/apis/grpc/v1/vald/vald.go @@ -95,6 +95,7 @@ const ( UpdateObjectRPCName = "UpdateObject" StreamUpdateObjectRPCName = "StreamUpdateObject" MultiUpdateObjectRPCName = "MultiUpdateObject" + UpdateTimestampRPCName = "UpdateTimestamp" UpsertRPCName = "Upsert" StreamUpsertRPCName = "StreamUpsert" diff --git a/apis/proto/v1/vald/update.proto b/apis/proto/v1/vald/update.proto index 4cab1cabf0..ccd715efd7 100644 --- a/apis/proto/v1/vald/update.proto +++ b/apis/proto/v1/vald/update.proto @@ -46,4 +46,12 @@ service Update { body: "*" }; } + + // A method to update timestamp an indexed vector. + rpc UpdateTimestamp(payload.v1.Object.Timestamp) returns (payload.v1.Object.Location) { + option (google.api.http) = { + post: "/update" + body: "*" + }; + } } diff --git a/apis/swagger/v1/vald/update.swagger.json b/apis/swagger/v1/vald/update.swagger.json index d295febc19..85c729d702 100644 --- a/apis/swagger/v1/vald/update.swagger.json +++ b/apis/swagger/v1/vald/update.swagger.json @@ -14,8 +14,8 @@ "paths": { "/update": { "post": { - "summary": "A method to update an indexed vector.", - "operationId": "Update_Update", + "summary": "A method to update timestamp an indexed vector.", + "operationId": "Update_UpdateTimestamp", "responses": { "200": { "description": "A successful response.", @@ -33,11 +33,11 @@ "parameters": [ { "name": "body", - "description": "Represent the update request.", + "description": "Represent a vector meta data.", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/v1UpdateRequest" + "$ref": "#/definitions/v1ObjectTimestamp" } } ], @@ -212,6 +212,21 @@ }, "description": "Represent the vector location." }, + "v1ObjectTimestamp": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The vector ID." + }, + "timestamp": { + "type": "string", + "format": "int64", + "description": "timestamp represents when this vector inserted." + } + }, + "description": "Represent a vector meta data." + }, "v1UpdateConfig": { "type": "object", "properties": { diff --git a/dockers/agent/core/agent/Dockerfile b/dockers/agent/core/agent/Dockerfile index f3195dde42..d901bd0f7a 100644 --- a/dockers/agent/core/agent/Dockerfile +++ b/dockers/agent/core/agent/Dockerfile @@ -36,11 +36,11 @@ ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=agent/core/agent ENV REPO=vald -ENV RUST_HOME=/usr/loacl/lib/rust +ENV RUST_HOME=/usr/local/lib/rust ENV TZ=Etc/UTC ENV USER=root -ENV CARGO_HOME=${RUST_HOME}/cargo ENV RUSTUP_HOME=${RUST_HOME}/rustup +ENV CARGO_HOME=${RUST_HOME}/cargo ENV PATH=${CARGO_HOME}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH} WORKDIR ${HOME}/rust/src/github.com/${ORG}/${REPO} SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -91,4 +91,4 @@ LABEL maintainer="vdaas.org vald team " COPY --from=builder /usr/bin/agent /usr/bin/agent # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/agent"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/agent"] diff --git a/dockers/agent/core/faiss/Dockerfile b/dockers/agent/core/faiss/Dockerfile index 463e13494b..3ee7076aad 100644 --- a/dockers/agent/core/faiss/Dockerfile +++ b/dockers/agent/core/faiss/Dockerfile @@ -93,4 +93,4 @@ COPY --from=builder /usr/bin/faiss /usr/bin/faiss COPY cmd/agent/core/faiss/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/faiss"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/faiss"] diff --git a/dockers/agent/core/ngt/Dockerfile b/dockers/agent/core/ngt/Dockerfile index cf0bbcdce2..55b55d7e43 100644 --- a/dockers/agent/core/ngt/Dockerfile +++ b/dockers/agent/core/ngt/Dockerfile @@ -92,4 +92,4 @@ COPY --from=builder /usr/bin/ngt /usr/bin/ngt COPY cmd/agent/core/ngt/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/ngt"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/ngt"] diff --git a/dockers/agent/sidecar/Dockerfile b/dockers/agent/sidecar/Dockerfile index 2b06e565fd..0f9c0e0760 100644 --- a/dockers/agent/sidecar/Dockerfile +++ b/dockers/agent/sidecar/Dockerfile @@ -82,4 +82,4 @@ LABEL maintainer="vdaas.org vald team " COPY --from=builder /usr/bin/sidecar /usr/bin/sidecar # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/sidecar"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/sidecar"] diff --git a/dockers/binfmt/Dockerfile b/dockers/binfmt/Dockerfile index 47284d17e2..f2d73909bd 100644 --- a/dockers/binfmt/Dockerfile +++ b/dockers/binfmt/Dockerfile @@ -16,4 +16,4 @@ # # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go -FROM tonistiigi/binfmt:master AS builder \ No newline at end of file +FROM tonistiigi/binfmt:master AS builder diff --git a/dockers/buildbase/Dockerfile b/dockers/buildbase/Dockerfile index 6457b01457..301a31e010 100644 --- a/dockers/buildbase/Dockerfile +++ b/dockers/buildbase/Dockerfile @@ -16,4 +16,4 @@ # # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go -FROM ubuntu:devel AS builder \ No newline at end of file +FROM ubuntu:devel AS builder diff --git a/dockers/buildkit/Dockerfile b/dockers/buildkit/Dockerfile index 9dd722ea25..99c71c61e2 100644 --- a/dockers/buildkit/Dockerfile +++ b/dockers/buildkit/Dockerfile @@ -16,4 +16,4 @@ # # DO_NOT_EDIT this Dockerfile is generated by https://github.com/vdaas/vald/blob/main/hack/docker/gen/main.go -FROM moby/buildkit:master AS builder \ No newline at end of file +FROM moby/buildkit:master AS builder diff --git a/dockers/ci/base/Dockerfile b/dockers/ci/base/Dockerfile index 09fccb1e35..270b2f1876 100644 --- a/dockers/ci/base/Dockerfile +++ b/dockers/ci/base/Dockerfile @@ -41,7 +41,7 @@ ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=ci/base ENV REPO=vald -ENV RUST_HOME=/usr/loacl/lib/rust +ENV RUST_HOME=/usr/local/lib/rust ENV TZ=Etc/UTC ENV USER=root ENV RUSTUP_HOME=${RUST_HOME}/rustup @@ -113,7 +113,6 @@ RUN --mount=type=bind,target=.,rw \ && make kubelinter/install \ && make reviewdog/install \ && make tparse/install \ - && make valdcli/install \ && make yq/install \ && make minikube/install \ && make stern/install \ @@ -123,4 +122,4 @@ RUN --mount=type=bind,target=.,rw \ && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/* # skipcq: DOK-DL3002 USER root:root -ENTRYPOINT ["/bin/bash"] \ No newline at end of file +ENTRYPOINT ["/bin/bash"] diff --git a/dockers/dev/Dockerfile b/dockers/dev/Dockerfile index 502f386e97..061c48169f 100644 --- a/dockers/dev/Dockerfile +++ b/dockers/dev/Dockerfile @@ -41,11 +41,11 @@ ENV LC_ALL=en_US.UTF-8 ENV ORG=vdaas ENV PKG=dev ENV REPO=vald -ENV RUST_HOME=/usr/loacl/lib/rust +ENV RUST_HOME=/usr/local/lib/rust ENV TZ=Etc/UTC ENV USER=root -ENV CARGO_HOME=${RUST_HOME}/cargo ENV RUSTUP_HOME=${RUST_HOME}/rustup +ENV CARGO_HOME=${RUST_HOME}/cargo ENV PATH=${CARGO_HOME}/bin:${GOPATH}/bin:${GOROOT}/bin:${RUSTUP_HOME}/bin:/usr/local/bin:${PATH} WORKDIR ${GOPATH}/src/github.com/${ORG}/${REPO} SHELL ["/bin/bash", "-o", "pipefail", "-c"] @@ -133,7 +133,6 @@ RUN --mount=type=bind,target=.,rw \ && make kubelinter/install \ && make reviewdog/install \ && make tparse/install \ - && make valdcli/install \ && make yq/install \ && make minikube/install \ && make stern/install \ @@ -142,4 +141,4 @@ RUN --mount=type=bind,target=.,rw \ && make faiss/install \ && rm -rf ${GOPATH}/src/github.com/${ORG}/${REPO}/* # skipcq: DOK-DL3002 -USER root:root \ No newline at end of file +USER root:root diff --git a/dockers/discoverer/k8s/Dockerfile b/dockers/discoverer/k8s/Dockerfile index 19e5953ee2..42d7b476f7 100644 --- a/dockers/discoverer/k8s/Dockerfile +++ b/dockers/discoverer/k8s/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/discoverer /usr/bin/discoverer COPY cmd/discoverer/k8s/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/discoverer"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/discoverer"] diff --git a/dockers/gateway/filter/Dockerfile b/dockers/gateway/filter/Dockerfile index b523b5e5b8..bd6b3dadd3 100644 --- a/dockers/gateway/filter/Dockerfile +++ b/dockers/gateway/filter/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/filter /usr/bin/filter COPY cmd/gateway/filter/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/filter"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/filter"] diff --git a/dockers/gateway/lb/Dockerfile b/dockers/gateway/lb/Dockerfile index 2f8c91768b..f6c3f0b5c5 100644 --- a/dockers/gateway/lb/Dockerfile +++ b/dockers/gateway/lb/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/lb /usr/bin/lb COPY cmd/gateway/lb/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/lb"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/lb"] diff --git a/dockers/gateway/mirror/Dockerfile b/dockers/gateway/mirror/Dockerfile index 9b97231c74..f3effce23f 100644 --- a/dockers/gateway/mirror/Dockerfile +++ b/dockers/gateway/mirror/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/mirror /usr/bin/mirror COPY cmd/gateway/mirror/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/mirror"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/mirror"] diff --git a/dockers/index/job/correction/Dockerfile b/dockers/index/job/correction/Dockerfile index 01e3818c56..06b7642c1a 100644 --- a/dockers/index/job/correction/Dockerfile +++ b/dockers/index/job/correction/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/index-correction /usr/bin/index-correction COPY cmd/index/job/correction/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-correction"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/index-correction"] diff --git a/dockers/index/job/creation/Dockerfile b/dockers/index/job/creation/Dockerfile index d656b3ad22..6ea3eda4b6 100644 --- a/dockers/index/job/creation/Dockerfile +++ b/dockers/index/job/creation/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/index-creation /usr/bin/index-creation COPY cmd/index/job/creation/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-creation"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/index-creation"] diff --git a/dockers/index/job/readreplica/rotate/Dockerfile b/dockers/index/job/readreplica/rotate/Dockerfile index bdb0ec7664..432085dbb4 100644 --- a/dockers/index/job/readreplica/rotate/Dockerfile +++ b/dockers/index/job/readreplica/rotate/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/readreplica-rotate /usr/bin/readreplica-rotate COPY cmd/index/job/readreplica/rotate/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/readreplica-rotate"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/readreplica-rotate"] diff --git a/dockers/index/job/save/Dockerfile b/dockers/index/job/save/Dockerfile index fdd674abd4..1b0797a81d 100644 --- a/dockers/index/job/save/Dockerfile +++ b/dockers/index/job/save/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/index-save /usr/bin/index-save COPY cmd/index/job/save/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-save"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/index-save"] diff --git a/dockers/index/operator/Dockerfile b/dockers/index/operator/Dockerfile index 23e9aae514..c2f652283f 100644 --- a/dockers/index/operator/Dockerfile +++ b/dockers/index/operator/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/index-operator /usr/bin/index-operator COPY cmd/index/operator/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index-operator"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/index-operator"] diff --git a/dockers/manager/index/Dockerfile b/dockers/manager/index/Dockerfile index edecb98a6c..8eb554a390 100644 --- a/dockers/manager/index/Dockerfile +++ b/dockers/manager/index/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/index /usr/bin/index COPY cmd/manager/index/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/index"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/index"] diff --git a/dockers/operator/helm/Dockerfile b/dockers/operator/helm/Dockerfile index db24094790..d5edc4f6e9 100644 --- a/dockers/operator/helm/Dockerfile +++ b/dockers/operator/helm/Dockerfile @@ -104,4 +104,4 @@ COPY --from=builder /opt/helm/charts/vald /opt/helm/charts/vald COPY --from=builder /opt/helm/charts/vald-helm-operator /opt/helm/charts/vald-helm-operator # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/helm-operator", "run", "--watches-file=/opt/helm/watches.yaml"] diff --git a/dockers/tools/benchmark/job/Dockerfile b/dockers/tools/benchmark/job/Dockerfile index 65da31d468..d77a1181fd 100644 --- a/dockers/tools/benchmark/job/Dockerfile +++ b/dockers/tools/benchmark/job/Dockerfile @@ -91,4 +91,4 @@ COPY --from=builder /usr/bin/job /usr/bin/job COPY cmd/tools/benchmark/job/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/job"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/job"] diff --git a/dockers/tools/benchmark/operator/Dockerfile b/dockers/tools/benchmark/operator/Dockerfile index 283773b11e..efe0a34e9f 100644 --- a/dockers/tools/benchmark/operator/Dockerfile +++ b/dockers/tools/benchmark/operator/Dockerfile @@ -83,4 +83,4 @@ COPY --from=builder /usr/bin/operator /usr/bin/operator COPY cmd/tools/benchmark/operator/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/operator"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/operator"] diff --git a/dockers/tools/cli/loadtest/Dockerfile b/dockers/tools/cli/loadtest/Dockerfile index ffbea81b05..d061b486f9 100644 --- a/dockers/tools/cli/loadtest/Dockerfile +++ b/dockers/tools/cli/loadtest/Dockerfile @@ -91,4 +91,4 @@ COPY --from=builder /usr/bin/loadtest /usr/bin/loadtest COPY cmd/tools/cli/loadtest/sample.yaml /etc/server/config.yaml # skipcq: DOK-DL3002 USER nonroot:nonroot -ENTRYPOINT ["/usr/bin/loadtest"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/loadtest"] diff --git a/example/client/go.mod b/example/client/go.mod index f64e6546d4..cc7f72f0f2 100644 --- a/example/client/go.mod +++ b/example/client/go.mod @@ -11,9 +11,9 @@ replace ( golang.org/x/crypto => golang.org/x/crypto v0.26.0 golang.org/x/net => golang.org/x/net v0.28.0 golang.org/x/text => golang.org/x/text v0.17.0 - google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240805194559-2c9e96a0b5d4 - google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4 - google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 google.golang.org/grpc => google.golang.org/grpc v1.65.0 google.golang.org/protobuf => google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 => gopkg.in/yaml.v2 v2.4.0 diff --git a/example/client/go.sum b/example/client/go.sum index d91b037bda..db50f99b1e 100644 --- a/example/client/go.sum +++ b/example/client/go.sum @@ -30,10 +30,10 @@ golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946 h1:vJpL69PeUullhJyKtTjHjENEmZU3BkO4e+fod7nKzgM= gonum.org/v1/hdf5 v0.0.0-20210714002203-8c5d23bc6946/go.mod h1:BQUWDHIAygjdt1HnUPQ0eWqLN2n5FwJycrpYUVUOx2I= -google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4 h1:ABEBT/sZ7We8zd7A5f3KO6zMQe+s3901H7l8Whhijt0= -google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 h1:OsSGQeIIsyOEOimVxLEIL4rwGcnrjOydQaiA2bOnZUM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 h1:+/tmTy5zAieooKIXfzDm9KiA3Bv6JBwriRN9LY+yayk= +google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 h1:V71AcdLZr2p8dC9dbOIMCpqi4EmRl8wUwnJzXXLmbmc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= diff --git a/go.mod b/go.mod index 333ed08f29..c9e699b97d 100644 --- a/go.mod +++ b/go.mod @@ -5,17 +5,17 @@ go 1.22.6 replace ( cloud.google.com/go => cloud.google.com/go v0.115.0 cloud.google.com/go/bigquery => cloud.google.com/go/bigquery v1.62.0 - cloud.google.com/go/compute => cloud.google.com/go/compute v1.27.4 + cloud.google.com/go/compute => cloud.google.com/go/compute v1.27.5 cloud.google.com/go/datastore => cloud.google.com/go/datastore v1.17.1 cloud.google.com/go/firestore => cloud.google.com/go/firestore v1.16.0 - cloud.google.com/go/iam => cloud.google.com/go/iam v1.1.12 - cloud.google.com/go/kms => cloud.google.com/go/kms v1.18.4 - cloud.google.com/go/monitoring => cloud.google.com/go/monitoring v1.20.3 + cloud.google.com/go/iam => cloud.google.com/go/iam v1.1.13 + cloud.google.com/go/kms => cloud.google.com/go/kms v1.18.5 + cloud.google.com/go/monitoring => cloud.google.com/go/monitoring v1.20.4 cloud.google.com/go/pubsub => cloud.google.com/go/pubsub v1.41.0 - cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager v1.13.5 + cloud.google.com/go/secretmanager => cloud.google.com/go/secretmanager v1.13.6 cloud.google.com/go/storage => cloud.google.com/go/storage v1.43.0 - cloud.google.com/go/trace => cloud.google.com/go/trace v1.10.11 - code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20240806182212-6cf545ebdd6b + cloud.google.com/go/trace => cloud.google.com/go/trace v1.10.12 + code.cloudfoundry.org/bytefmt => code.cloudfoundry.org/bytefmt v0.0.0-20240808182453-a379845013d9 contrib.go.opencensus.io/exporter/aws => contrib.go.opencensus.io/exporter/aws v0.0.0-20230502192102-15967c811cec contrib.go.opencensus.io/exporter/prometheus => contrib.go.opencensus.io/exporter/prometheus v0.4.2 contrib.go.opencensus.io/integrations/ocsql => contrib.go.opencensus.io/integrations/ocsql v0.1.7 @@ -63,7 +63,7 @@ replace ( github.com/aws/aws-sdk-go-v2/service/secretsmanager => github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.4 github.com/aws/aws-sdk-go-v2/service/sns => github.com/aws/aws-sdk-go-v2/service/sns v1.31.3 github.com/aws/aws-sdk-go-v2/service/sqs => github.com/aws/aws-sdk-go-v2/service/sqs v1.34.3 - github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm v1.52.3 + github.com/aws/aws-sdk-go-v2/service/ssm => github.com/aws/aws-sdk-go-v2/service/ssm v1.52.4 github.com/aws/aws-sdk-go-v2/service/sso => github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 github.com/aws/aws-sdk-go-v2/service/sts => github.com/aws/aws-sdk-go-v2/service/sts v1.30.3 github.com/aws/smithy-go => github.com/aws/smithy-go v1.20.3 @@ -192,7 +192,7 @@ replace ( github.com/jstemmer/go-junit-report => github.com/jstemmer/go-junit-report v1.0.0 github.com/kisielk/errcheck => github.com/kisielk/errcheck v1.7.0 github.com/kisielk/gotool => github.com/kisielk/gotool v1.0.0 - github.com/klauspost/compress => github.com/klauspost/compress v1.17.10-0.20240805110405-8b81499bfd70 + github.com/klauspost/compress => github.com/klauspost/compress v1.17.10-0.20240812095115-3868468e621b github.com/klauspost/cpuid/v2 => github.com/klauspost/cpuid/v2 v2.2.8 github.com/kpango/fastime => github.com/kpango/fastime v1.1.9 github.com/kpango/fuid => github.com/kpango/fuid v0.0.0-20221203053508-503b5ad89aa1 @@ -295,8 +295,8 @@ replace ( go.uber.org/zap => go.uber.org/zap v1.27.0 gocloud.dev => gocloud.dev v0.38.0 golang.org/x/crypto => golang.org/x/crypto v0.26.0 - golang.org/x/exp => golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 - golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20240719175910-8a7402abbf56 + golang.org/x/exp => golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa + golang.org/x/exp/typeparams => golang.org/x/exp/typeparams v0.0.0-20240808152545-0cdaa3abc0fa golang.org/x/image => golang.org/x/image v0.19.0 golang.org/x/lint => golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 golang.org/x/mobile => golang.org/x/mobile v0.0.0-20240806205939-81131f6468ab @@ -304,7 +304,7 @@ replace ( golang.org/x/net => golang.org/x/net v0.28.0 golang.org/x/oauth2 => golang.org/x/oauth2 v0.22.0 golang.org/x/sync => golang.org/x/sync v0.8.0 - golang.org/x/sys => golang.org/x/sys v0.23.0 + golang.org/x/sys => golang.org/x/sys v0.24.0 golang.org/x/term => golang.org/x/term v0.23.0 golang.org/x/text => golang.org/x/text v0.17.0 golang.org/x/time => golang.org/x/time v0.6.0 @@ -316,9 +316,9 @@ replace ( gonum.org/v1/plot => gonum.org/v1/plot v0.14.0 google.golang.org/api => google.golang.org/api v0.191.0 google.golang.org/appengine => google.golang.org/appengine v1.6.8 - google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240805194559-2c9e96a0b5d4 - google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4 - google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 + google.golang.org/genproto => google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 + google.golang.org/genproto/googleapis/api => google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 + google.golang.org/genproto/googleapis/rpc => google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 google.golang.org/grpc => google.golang.org/grpc v1.65.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc => google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 google.golang.org/protobuf => google.golang.org/protobuf v1.34.2 @@ -336,10 +336,10 @@ replace ( k8s.io/client-go => k8s.io/client-go v0.30.3 k8s.io/component-base => k8s.io/component-base v0.30.3 k8s.io/klog/v2 => k8s.io/klog/v2 v2.130.1 - k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e + k8s.io/kube-openapi => k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f k8s.io/kubernetes => k8s.io/kubernetes v0.30.3 k8s.io/metrics => k8s.io/metrics v0.30.3 - nhooyr.io/websocket => nhooyr.io/websocket v1.8.11 + nhooyr.io/websocket => nhooyr.io/websocket v1.8.17 rsc.io/pdf => rsc.io/pdf v0.1.1 sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.18.4 sigs.k8s.io/json => sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd @@ -399,7 +399,7 @@ require ( golang.org/x/net v0.28.0 golang.org/x/oauth2 v0.22.0 golang.org/x/sync v0.8.0 - golang.org/x/sys v0.23.0 + golang.org/x/sys v0.24.0 golang.org/x/text v0.17.0 golang.org/x/time v0.6.0 golang.org/x/tools v0.24.0 @@ -422,7 +422,7 @@ require ( require ( cloud.google.com/go v0.115.0 // indirect - cloud.google.com/go/auth v0.7.3 // indirect + cloud.google.com/go/auth v0.8.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect cloud.google.com/go/iam v1.1.12 // indirect @@ -506,14 +506,14 @@ require ( go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.26.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect golang.org/x/exp/typeparams v0.0.0-20240213143201-ec583247a57a // indirect golang.org/x/image v0.19.0 // indirect golang.org/x/mod v0.20.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/api v0.189.0 // indirect + google.golang.org/api v0.191.0 // indirect google.golang.org/genproto v0.0.0-20240730163845-b1a4ccb954bf // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 106b02efec..04313bdd06 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,9 @@ cloud.google.com/go/auth v0.6.0/go.mod h1:b4acV+jLQDyjwm4OXHYjNvRi4jvGBzHWJRtJcy cloud.google.com/go/auth v0.6.1/go.mod h1:eFHG7zDzbXHKmjJddFG/rBlcGp6t25SwRUiEQSlO4x4= cloud.google.com/go/auth v0.7.0/go.mod h1:D+WqdrpcjmiCgWrXmLLxOVq1GACoE36chW6KXoEvuIw= cloud.google.com/go/auth v0.7.2/go.mod h1:VEc4p5NNxycWQTMQEDQF0bd6aTMb6VgYDXEwiJJQAbs= -cloud.google.com/go/auth v0.7.3 h1:98Vr+5jMaCZ5NZk6e/uBgf60phTk/XN84r8QEWB9yjY= cloud.google.com/go/auth v0.7.3/go.mod h1:HJtWUx1P5eqjy/f6Iq5KeytNpbAcGolPhOgyop2LlzA= +cloud.google.com/go/auth v0.8.0 h1:y8jUJLl/Fg+qNBWxP/Hox2ezJvjkrPb952PC1p0G6A4= +cloud.google.com/go/auth v0.8.0/go.mod h1:qGVp/Y3kDRSDZ5gFD/XPUfYQ9xW1iI7q8RIRoCyBbJc= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/auth/oauth2adapt v0.2.3 h1:MlxF+Pd3OmSudg/b1yZ5lJwoXCEaeedAguodky1PcKI= cloud.google.com/go/auth/oauth2adapt v0.2.3/go.mod h1:tMQXOfZzFuNuUxOypHlQEXgdfX5cuhwU+ffUuXRJE8I= @@ -45,7 +46,7 @@ cloud.google.com/go/channel v1.17.11/go.mod h1:gjWCDBcTGQce/BSMoe2lAqhlq0dIRiZuk cloud.google.com/go/cloudbuild v1.16.5/go.mod h1:HXLpZ8QeYZgmDIWpbl9Gs22p6o6uScgQ/cV9HF9cIZU= cloud.google.com/go/clouddms v1.7.10/go.mod h1:PzHELq0QDyA7VaD9z6mzh2mxeBz4kM6oDe8YxMxd4RA= cloud.google.com/go/cloudtasks v1.12.12/go.mod h1:8UmM+duMrQpzzRREo0i3x3TrFjsgI/3FQw3664/JblA= -cloud.google.com/go/compute v1.27.4/go.mod h1:7JZS+h21ERAGHOy5qb7+EPyXlQwzshzrx1x6L9JhTqU= +cloud.google.com/go/compute v1.27.5/go.mod h1:DfwDGujFTdSeiE8b8ZqadF/uxHFBz+ekGsk8Zfi9dTA= cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= @@ -84,12 +85,12 @@ cloud.google.com/go/gkehub v0.14.11/go.mod h1:CsmDJ4qbBnSPkoBltEubK6qGOjG0xNfeeT cloud.google.com/go/gkemulticloud v1.2.4/go.mod h1:PjTtoKLQpIRztrL+eKQw8030/S4c7rx/WvHydDJlpGE= cloud.google.com/go/grafeas v0.3.6/go.mod h1:to6ECAPgRO2xeqD8ISXHc70nObJuaKZThreQOjeOH3o= cloud.google.com/go/gsuiteaddons v1.6.11/go.mod h1:U7mk5PLBzDpHhgHv5aJkuvLp9RQzZFpa8hgWAB+xVIk= -cloud.google.com/go/iam v1.1.12 h1:JixGLimRrNGcxvJEQ8+clfLxPlbeZA6MuRJ+qJNQ5Xw= -cloud.google.com/go/iam v1.1.12/go.mod h1:9LDX8J7dN5YRyzVHxwQzrQs9opFFqn0Mxs9nAeB+Hhg= +cloud.google.com/go/iam v1.1.13 h1:7zWBXG9ERbMLrzQBRhFliAV+kjcRToDTgQT3CTwYyv4= +cloud.google.com/go/iam v1.1.13/go.mod h1:K8mY0uSXwEXS30KrnVb+j54LB/ntfZu1dr+4zFMNbus= cloud.google.com/go/iap v1.9.10/go.mod h1:pO0FEirrhMOT1H0WVwpD5dD9r3oBhvsunyBQtNXzzc0= cloud.google.com/go/ids v1.4.11/go.mod h1:+ZKqWELpJm8WcRRsSvKZWUdkriu4A3XsLLzToTv3418= cloud.google.com/go/iot v1.7.11/go.mod h1:0vZJOqFy9kVLbUXwTP95e0dWHakfR4u5IWqsKMGIfHk= -cloud.google.com/go/kms v1.18.4/go.mod h1:SG1bgQ3UWW6/KdPo9uuJnzELXY5YTTMJtDYvajiQ22g= +cloud.google.com/go/kms v1.18.5/go.mod h1:yXunGUGzabH8rjUPImp2ndHiGolHeWJJ0LODLedicIY= cloud.google.com/go/language v1.13.0/go.mod h1:B9FbD17g1EkilctNGUDAdSrBHiFOlKNErLljO7jplDU= cloud.google.com/go/lifesciences v0.9.11/go.mod h1:NMxu++FYdv55TxOBEvLIhiAvah8acQwXsz79i9l9/RY= cloud.google.com/go/logging v1.11.0/go.mod h1:5LDiJC/RxTt+fHc1LAt20R9TKiUTReDg6RuuFOZ67+A= @@ -108,7 +109,7 @@ cloud.google.com/go/maps v1.11.6/go.mod h1:MOS/NN0L6b7Kumr8bLux9XTpd8+D54DYxBMUj cloud.google.com/go/mediatranslation v0.8.11/go.mod h1:3sNEm0fx61eHk7rfzBzrljVV9XKr931xI3OFacQBVFg= cloud.google.com/go/memcache v1.10.11/go.mod h1:ubJ7Gfz/xQawQY5WO5pht4Q0dhzXBFeEszAeEJnwBHU= cloud.google.com/go/metastore v1.13.10/go.mod h1:RPhMnBxUmTLT1fN7fNbPqtH5EoGHueDxubmJ1R1yT84= -cloud.google.com/go/monitoring v1.20.3/go.mod h1:GPIVIdNznIdGqEjtRKQWTLcUeRnPjZW85szouimiczU= +cloud.google.com/go/monitoring v1.20.4/go.mod h1:v7F/UcLRw15EX7xq565N7Ae5tnYEE28+Cl717aTXG4c= cloud.google.com/go/networkconnectivity v1.14.10/go.mod h1:f7ZbGl4CV08DDb7lw+NmMXQTKKjMhgCEEwFbEukWuOY= cloud.google.com/go/networkmanagement v1.13.6/go.mod h1:WXBijOnX90IFb6sberjnGrVtZbgDNcPDUYOlGXmG8+4= cloud.google.com/go/networksecurity v0.9.11/go.mod h1:4xbpOqCwplmFgymAjPFM6ZIplVC6+eQ4m7sIiEq9oJA= @@ -134,7 +135,7 @@ cloud.google.com/go/resourcesettings v1.7.4/go.mod h1:seBdLuyeq+ol2u9G2+74GkSjQa cloud.google.com/go/retail v1.17.4/go.mod h1:oPkL1FzW7D+v/hX5alYIx52ro2FY/WPAviwR1kZZTMs= cloud.google.com/go/run v1.4.0/go.mod h1:4G9iHLjdOC+CQ0CzA0+6nLeR6NezVPmlj+GULmb0zE4= cloud.google.com/go/scheduler v1.10.12/go.mod h1:6DRtOddMWJ001HJ6MS148rtLSh/S2oqd2hQC3n5n9fQ= -cloud.google.com/go/secretmanager v1.13.5/go.mod h1:/OeZ88l5Z6nBVilV0SXgv6XJ243KP2aIhSWRMrbvDCQ= +cloud.google.com/go/secretmanager v1.13.6/go.mod h1:x2ySyOrqv3WGFRFn2Xk10iHmNmvmcEVSSqc30eb1bhw= cloud.google.com/go/security v1.17.4/go.mod h1:KMuDJH+sEB3KTODd/tLJ7kZK+u2PQt+Cfu0oAxzIhgo= cloud.google.com/go/securitycenter v1.33.1/go.mod h1:jeFisdYUWHr+ig72T4g0dnNCFhRwgwGoQV6GFuEwafw= cloud.google.com/go/servicedirectory v1.11.11/go.mod h1:pnynaftaj9LmRLIc6t3r7r7rdCZZKKxui/HaF/RqYfs= @@ -147,7 +148,7 @@ cloud.google.com/go/storagetransfer v1.10.10/go.mod h1:8+nX+WgQ2ZJJnK8e+RbK/zCXk cloud.google.com/go/talent v1.6.12/go.mod h1:nT9kNVuJhZX2QgqKZS6t6eCWZs5XEBYRBv6bIMnPmo4= cloud.google.com/go/texttospeech v1.7.11/go.mod h1:Ua125HU+WT2IkIo5MzQtuNpNEk72soShJQVdorZ1SAE= cloud.google.com/go/tpu v1.6.11/go.mod h1:W0C4xaSj1Ay3VX/H96FRvLt2HDs0CgdRPVI4e7PoCDk= -cloud.google.com/go/trace v1.10.11/go.mod h1:fUr5L3wSXerNfT0f1bBg08W4axS2VbHGgYcfH4KuTXU= +cloud.google.com/go/trace v1.10.12/go.mod h1:tYkAIta/gxgbBZ/PIzFxSH5blajgX4D00RpQqCG/GZs= cloud.google.com/go/translate v1.10.3/go.mod h1:GW0vC1qvPtd3pgtypCv4k4U8B7EdgK9/QEF2aJEUovs= cloud.google.com/go/translate v1.10.7/go.mod h1:mH/+8tvcItuy1cOWqU+/Y3iFHgkVUObNIQYI/kiFFiY= cloud.google.com/go/video v1.22.0/go.mod h1:CxPshUNAb1ucnzbtruEHlAal9XY+SPG2cFqC/woJzII= @@ -159,8 +160,8 @@ cloud.google.com/go/vpcaccess v1.7.11/go.mod h1:a2cuAiSCI4TVK0Dt6/dRjf22qQvfY+po cloud.google.com/go/webrisk v1.9.11/go.mod h1:mK6M8KEO0ZI7VkrjCq3Tjzw4vYq+3c4DzlMUDVaiswE= cloud.google.com/go/websecurityscanner v1.6.11/go.mod h1:vhAZjksELSg58EZfUQ1BMExD+hxqpn0G0DuyCZQjiTg= cloud.google.com/go/workflows v1.12.10/go.mod h1:RcKqCiOmKs8wFUEf3EwWZPH5eHc7Oq0kamIyOUCk0IE= -code.cloudfoundry.org/bytefmt v0.0.0-20240806182212-6cf545ebdd6b h1:C+YxvUtePwiSzniQagI/yDdWjh1Lx1TkstHBGN7OTnA= -code.cloudfoundry.org/bytefmt v0.0.0-20240806182212-6cf545ebdd6b/go.mod h1:9aVxojRyikUaWddFMb0A9tgpGMtDPhk1pnCfhY0/fA4= +code.cloudfoundry.org/bytefmt v0.0.0-20240808182453-a379845013d9 h1:8KlrGCtoaWaaxVxi9KzED38kNIWa1qafh9bNSVZ6otk= +code.cloudfoundry.org/bytefmt v0.0.0-20240808182453-a379845013d9/go.mod h1:eF2ZbltNI7Pv+8Cuyeksu9up5FN5konuH0trDJBuscw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20221208032759-85de2813cf6b/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= eliasnaur.com/font v0.0.0-20230308162249-dd43949cb42d/go.mod h1:OYVuxibdk9OSLX8vAqydtRPP87PyTFcT9uH3MlEGBQA= @@ -484,8 +485,8 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/errcheck v1.7.0/go.mod h1:1kLL+jV4e+CFfueBmI1dSK2ADDyQnlrnrY/FqKluHJQ= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.17.10-0.20240805110405-8b81499bfd70 h1:3rXiP8hu9oCOQ7ZcJO7x4Dh/r0yCkwpj7GGOB5fA20U= -github.com/klauspost/compress v1.17.10-0.20240805110405-8b81499bfd70/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.10-0.20240812095115-3868468e621b h1:4A/j6hb0Sd3VXqhNtgmUlcPy353Qaa0aIfAPcBrI1n8= +github.com/klauspost/compress v1.17.10-0.20240812095115-3868468e621b/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kpango/fastime v1.1.9 h1:xVQHcqyPt5M69DyFH7g1EPRns1YQNap9d5eLhl/Jy84= @@ -684,14 +685,14 @@ gocloud.dev v0.38.0 h1:SpxfaOc/Fp4PeO8ui7wRcCZV0EgXZ+IWcVSLn6ZMSw0= gocloud.dev v0.38.0/go.mod h1:3XjKvd2E5iVNu/xFImRzjN0d/fkNHe4s0RiKidpEUMQ= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/exp/shiny v0.0.0-20220827204233-334a2380cb91/go.mod h1:VjAR7z0ngyATZTELrBSkxOOHhhlnVUxDye4mcjx5h/8= golang.org/x/exp/shiny v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0= golang.org/x/exp/shiny v0.0.0-20230817173708-d852ddb80c63/go.mod h1:UH99kUObWAZkDnWqppdQe5ZhPYESUw8I0zVV1uWBR+0= golang.org/x/exp/shiny v0.0.0-20240707233637-46b078467d37/go.mod h1:3F+MieQB7dRYLTmnncoFbb1crS5lfQoTfDgQy6K4N0o= -golang.org/x/exp/typeparams v0.0.0-20240719175910-8a7402abbf56 h1:i+QrZdyNyfLEnWjd5T6LQZvQP3xk2XiNs3sQgN7QDGE= -golang.org/x/exp/typeparams v0.0.0-20240719175910-8a7402abbf56/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20240808152545-0cdaa3abc0fa h1:54T+HVkPu4D3lltpEHyI3Fs2pG/GqjGkXLgyKOmifXk= +golang.org/x/exp/typeparams v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.19.0 h1:D9FX4QWkLfkeqaC62SonffIIuYdOk/UE2XKUBgRIBIQ= golang.org/x/image v0.19.0/go.mod h1:y0zrRqlQRWQ5PXaYCOMLTW2fpsxZ8Qh9I/ohnInJEys= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -704,8 +705,8 @@ golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= @@ -728,13 +729,13 @@ gonum.org/v1/plot v0.14.0/go.mod h1:MLdR9424SJed+5VqC6MsouEpig9pZX2VZ57H9ko2bXU= google.golang.org/api v0.191.0 h1:cJcF09Z+4HAB2t5qTQM1ZtfL/PemsLFkcFG67qq2afk= google.golang.org/api v0.191.0/go.mod h1:tD5dsFGxFza0hnQveGfVk9QQYKcfp+VzgRqyXFxE0+E= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto v0.0.0-20240805194559-2c9e96a0b5d4 h1:g+rQ3aqOyXK/0qwnC5TGUXnyIeipstP5SsniB9uPJ2c= -google.golang.org/genproto v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:7uvplUBj4RjHAxIZ//98LzOvrQ04JBkaixRmCMI29hc= -google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4 h1:ABEBT/sZ7We8zd7A5f3KO6zMQe+s3901H7l8Whhijt0= -google.golang.org/genproto/googleapis/api v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= +google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988 h1:CT2Thj5AuPV9phrYMtzX11k+XkzMGfRAet42PmoTATM= +google.golang.org/genproto v0.0.0-20240812133136-8ffd90a71988/go.mod h1:7uvplUBj4RjHAxIZ//98LzOvrQ04JBkaixRmCMI29hc= +google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988 h1:+/tmTy5zAieooKIXfzDm9KiA3Bv6JBwriRN9LY+yayk= +google.golang.org/genproto/googleapis/api v0.0.0-20240812133136-8ffd90a71988/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= google.golang.org/genproto/googleapis/bytestream v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:5/MT647Cn/GGhwTpXC7QqcaR5Cnee4v4MKCU1/nwnIQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4 h1:OsSGQeIIsyOEOimVxLEIL4rwGcnrjOydQaiA2bOnZUM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240805194559-2c9e96a0b5d4/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988 h1:V71AcdLZr2p8dC9dbOIMCpqi4EmRl8wUwnJzXXLmbmc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240812133136-8ffd90a71988/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= @@ -764,8 +765,8 @@ k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e h1:OnKkExfhk4yxMqvBSPzUfhv3zQ96FWJ+UOZzLrAFyAo= -k8s.io/kube-openapi v0.0.0-20240730131305-7a9a4e85957e/go.mod h1:0CVn9SVo8PeW5/JgsBZZIFmmTk5noOM8WXf2e1tCihE= +k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f h1:bnWtxXWdAl5bVOCEPoNdvMkyj6cTW3zxHuwKIakuV9w= +k8s.io/kube-openapi v0.0.0-20240812233141-91dab695df6f/go.mod h1:G0W3eI9gG219NHRq3h5uQaRBl4pj4ZpwzRP5ti8y770= k8s.io/metrics v0.30.3 h1:gKCpte5zykrOmQhZ8qmsxyJslMdiLN+sqbBfIWNpbGM= k8s.io/metrics v0.30.3/go.mod h1:W06L2nXRhOwPkFYDJYWdEIS3u6JcJy3ebIPYbndRs6A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= diff --git a/hack/docker/gen/main.go b/hack/docker/gen/main.go index 7750270efc..34e073c420 100644 --- a/hack/docker/gen/main.go +++ b/hack/docker/gen/main.go @@ -223,6 +223,9 @@ const ( organization = "vdaas" repository = "vald" defaultBinaryDir = "/usr/bin" + usrLocal = "/usr/local" + usrLocalBinaryDir = usrLocal + "/bin" + usrLocalLibDir = usrLocal + "/lib" defaultBuilderImage = "ghcr.io/vdaas/vald/vald-buildbase" defaultBuilderTag = "nightly" defaultLanguage = "en_US.UTF-8" @@ -230,7 +233,8 @@ const ( defaultRuntimeImage = "gcr.io/distroless/static" defaultRuntimeTag = "nonroot" defaultRuntimeUser = "nonroot:nonroot" - defaultBuildUser = "root:root" + rootUser = "root" + defaultBuildUser = rootUser + ":" + rootUser defaultBuildStageName = "builder" maintainerKey = "MAINTAINER" minimumArgumentLength = 2 @@ -274,28 +278,28 @@ var ( defaultEnvironments = map[string]string{ "DEBIAN_FRONTEND": "noninteractive", - "HOME": "/root", - "USER": "root", + "HOME": "/" + rootUser, + "USER": rootUser, "INITRD": "No", "LANG": defaultLanguage, "LANGUAGE": defaultLanguage, "LC_ALL": defaultLanguage, "ORG": organization, "TZ": "Etc/UTC", - "PATH": "${PATH}:/usr/local/bin", + "PATH": "${PATH}:" + usrLocalBinaryDir, "REPO": repository, } goDefaultEnvironments = map[string]string{ "GOROOT": "/opt/go", "GOPATH": "/go", "GO111MODULE": "on", - "PATH": "${PATH}:${GOROOT}/bin:${GOPATH}/bin:/usr/local/bin", + "PATH": "${PATH}:${GOROOT}/bin:${GOPATH}/bin:" + usrLocalBinaryDir, } rustDefaultEnvironments = map[string]string{ - "RUST_HOME": "/usr/loacl/lib/rust", + "RUST_HOME": usrLocalLibDir + "/rust", "RUSTUP_HOME": "${RUST_HOME}/rustup", "CARGO_HOME": "${RUST_HOME}/cargo", - "PATH": "${PATH}:${RUSTUP_HOME}/bin:${CARGO_HOME}/bin:/usr/local/bin", + "PATH": "${PATH}:${RUSTUP_HOME}/bin:${CARGO_HOME}/bin:" + usrLocalBinaryDir, } clangDefaultEnvironments = map[string]string{ "CC": "gcc", @@ -372,7 +376,6 @@ var ( "make kubelinter/install", "make reviewdog/install", "make tparse/install", - "make valdcli/install", "make yq/install", "make minikube/install", "make stern/install", @@ -597,7 +600,7 @@ func main() { "OPERATOR_SDK_VERSION": "latest", }, ExtraCopies: []string{ - "--from=operator /usr/local/bin/${APP_NAME} {{$.BinDir}}/${APP_NAME}", + "--from=operator " + usrLocalBinaryDir + "/${APP_NAME} {{$.BinDir}}/${APP_NAME}", }, ExtraImages: []string{ "quay.io/operator-framework/helm-operator:${OPERATOR_SDK_VERSION} AS operator", @@ -790,9 +793,9 @@ func main() { data.RootDir = "${HOME}" data.Environments["ROOTDIR"] = os.Args[1] } - if strings.Contains(data.BuildUser, "root") { - data.Environments["HOME"] = "/root" - data.Environments["USER"] = "root" + if strings.Contains(data.BuildUser, rootUser) { + data.Environments["HOME"] = "/" + rootUser + data.Environments["USER"] = rootUser } else { user := data.BuildUser if strings.Contains(user, ":") { diff --git a/internal/client/v1/client/vald/vald.go b/internal/client/v1/client/vald/vald.go index 00957b7b7b..280aa78be8 100644 --- a/internal/client/v1/client/vald/vald.go +++ b/internal/client/v1/client/vald/vald.go @@ -499,6 +499,28 @@ func (c *client) MultiUpdate( return res, nil } +func (c *client) UpdateTimestamp( + ctx context.Context, in *payload.Object_Timestamp, opts ...grpc.CallOption, +) (res *payload.Object_Location, err error) { + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "internal/client/"+vald.UpdateTimestampRPCName), apiName+"/"+vald.UpdateTimestampRPCName) + defer func() { + if span != nil { + span.End() + } + }() + _, err = c.c.RoundRobin(ctx, func(ctx context.Context, + conn *grpc.ClientConn, + copts ...grpc.CallOption, + ) (any, error) { + res, err = vald.NewValdClient(conn).UpdateTimestamp(ctx, in, append(copts, opts...)...) + return nil, err + }) + if err != nil { + return nil, err + } + return res, nil +} + func (c *client) Upsert( ctx context.Context, in *payload.Upsert_Request, opts ...grpc.CallOption, ) (res *payload.Object_Location, err error) { @@ -1066,6 +1088,18 @@ func (c *singleClient) Update( return c.vc.Update(ctx, in, opts...) } +func (c *singleClient) UpdateTimestamp( + ctx context.Context, in *payload.Object_Timestamp, opts ...grpc.CallOption, +) (res *payload.Object_Location, err error) { + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "internal/singleClient/"+vald.UpdateTimestampRPCName), apiName+"/"+vald.UpdateTimestampRPCName) + defer func() { + if span != nil { + span.End() + } + }() + return c.vc.UpdateTimestamp(ctx, in, opts...) +} + func (c *singleClient) StreamUpdate( ctx context.Context, opts ...grpc.CallOption, ) (res vald.Update_StreamUpdateClient, err error) { diff --git a/internal/errors/agent.go b/internal/errors/agent.go index da16986ade..a25d13d043 100644 --- a/internal/errors/agent.go +++ b/internal/errors/agent.go @@ -115,4 +115,9 @@ var ( // ErrWriteOperationToReadReplica represents an error that when a write operation is made to read replica. ErrWriteOperationToReadReplica = New("write operation to read replica is not possible") + + // ErrInvalidTimestamp represents a function to generate an error that the timestamp is invalid. + ErrInvalidTimestamp = func(ts int64) error { + return Errorf("invalid timestamp detected", ts) + } ) diff --git a/pkg/gateway/lb/handler/grpc/handler.go b/pkg/gateway/lb/handler/grpc/handler.go index acfdfad8f3..555c3d739b 100644 --- a/pkg/gateway/lb/handler/grpc/handler.go +++ b/pkg/gateway/lb/handler/grpc/handler.go @@ -361,55 +361,17 @@ func (s *server) SearchByID( } return nil, err } - vec, err := s.getObject(ctx, uuid) + vec, err := s.GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: uuid, + }, + }) if err != nil { - var ( - attrs trace.Attributes - st *status.Status - msg string - ) - switch { - case errors.Is(err, errors.ErrInvalidUUID(uuid)): - err = status.WrapWithInvalidArgument( - vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API invalid argument for uuid \""+uuid+"\" detected", - err, - reqInfo, - resInfo, - &errdetails.BadRequest{ - FieldViolations: []*errdetails.BadRequestFieldViolation{ - { - Field: "uuid", - Description: err.Error(), - }, - }, - }, - ) - attrs = trace.StatusCodeInvalidArgument(err.Error()) - case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): - err = status.WrapWithInternal(vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API connection not found", err, reqInfo, resInfo) - attrs = trace.StatusCodeInternal(err.Error()) - case errors.Is(err, context.Canceled): - err = status.WrapWithCanceled(vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API canceled", err, reqInfo, resInfo) - attrs = trace.StatusCodeCancelled(err.Error()) - case errors.Is(err, context.DeadlineExceeded): - err = status.WrapWithDeadlineExceeded(vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API deadline exceeded", err, reqInfo, resInfo) - attrs = trace.StatusCodeDeadlineExceeded(err.Error()) - case errors.Is(err, errors.ErrObjectIDNotFound(uuid)), errors.Is(err, errors.ErrObjectNotFound(nil, uuid)): - err = nil - default: - st, msg, err = status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) - attrs = trace.FromGRPCStatus(st.Code(), msg) - if st == nil || st.Code() == codes.NotFound { - err = nil - } - } - if err != nil { - if span != nil { - span.RecordError(err) - span.SetAttributes(attrs...) - span.SetStatus(trace.StatusError, err.Error()) - } - return nil, err + st, msg, err := status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.SearchByIDRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if span != nil && st != nil && st.Code() != codes.NotFound { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), st.Message())...) + span.SetStatus(trace.StatusError, err.Error()) } // try search by using agent's SearchByID method this operation is emergency fallback, the search quality is not same as usual SearchByID operation. res, err = s.doSearch(ctx, req.GetConfig(), func(ctx context.Context, fcfg *payload.Search_Config, vc vald.Client, copts ...grpc.CallOption) (*payload.Search_Response, error) { @@ -851,55 +813,17 @@ func (s *server) LinearSearchByID( } return nil, err } - vec, err := s.getObject(ctx, uuid) + vec, err := s.GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: uuid, + }, + }) if err != nil { - var ( - attrs trace.Attributes - st *status.Status - msg string - ) - switch { - case errors.Is(err, errors.ErrInvalidUUID(uuid)): - err = status.WrapWithInvalidArgument( - vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API invalid argument for uuid \""+uuid+"\" detected", - err, - reqInfo, - resInfo, - &errdetails.BadRequest{ - FieldViolations: []*errdetails.BadRequestFieldViolation{ - { - Field: "uuid", - Description: err.Error(), - }, - }, - }, - ) - attrs = trace.StatusCodeInvalidArgument(err.Error()) - case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): - err = status.WrapWithInternal(vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API connection not found", err, reqInfo, resInfo) - attrs = trace.StatusCodeInternal(err.Error()) - case errors.Is(err, context.Canceled): - err = status.WrapWithCanceled(vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API canceled", err, reqInfo, resInfo) - attrs = trace.StatusCodeCancelled(err.Error()) - case errors.Is(err, context.DeadlineExceeded): - err = status.WrapWithDeadlineExceeded(vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API deadline exceeded", err, reqInfo, resInfo) - attrs = trace.StatusCodeDeadlineExceeded(err.Error()) - case errors.Is(err, errors.ErrObjectIDNotFound(uuid)), errors.Is(err, errors.ErrObjectNotFound(nil, uuid)): - err = nil - default: - st, msg, err = status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) - attrs = trace.FromGRPCStatus(st.Code(), msg) - if st == nil || st.Code() == codes.NotFound { - err = nil - } - } - if err != nil { - if span != nil { - span.RecordError(err) - span.SetAttributes(attrs...) - span.SetStatus(trace.StatusError, err.Error()) - } - return nil, err + st, msg, err := status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.LinearSearchByIDRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if span != nil && st != nil && st.Code() != codes.NotFound { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), st.Message())...) + span.SetStatus(trace.StatusError, err.Error()) } // try search by using agent's LinearSearchByID method this operation is emergency fallback, the search quality is not same as usual LinearSearchByID operation. res, err = s.doSearch(ctx, req.GetConfig(), func(ctx context.Context, fcfg *payload.Search_Config, vc vald.Client, copts ...grpc.CallOption) (*payload.Search_Response, error) { @@ -1778,54 +1702,16 @@ func (s *server) Update( } if !req.GetConfig().GetSkipStrictExistCheck() { - vec, err := s.getObject(ctx, uuid) - if err != nil || vec == nil { - var ( - attrs trace.Attributes - st *status.Status - msg string - ) - switch { - case errors.Is(err, errors.ErrInvalidUUID(uuid)): - err = status.WrapWithInvalidArgument( - vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API invalid argument for uuid \""+uuid+"\" detected", - err, - reqInfo, - resInfo, - &errdetails.BadRequest{ - FieldViolations: []*errdetails.BadRequestFieldViolation{ - { - Field: "uuid", - Description: err.Error(), - }, - }, - }, - ) - attrs = trace.StatusCodeInvalidArgument(err.Error()) - case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): - err = status.WrapWithInternal(vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API connection not found", err, reqInfo, resInfo) - attrs = trace.StatusCodeInternal(err.Error()) - case errors.Is(err, context.Canceled): - err = status.WrapWithCanceled(vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API canceled", err, reqInfo, resInfo) - attrs = trace.StatusCodeCancelled(err.Error()) - case errors.Is(err, context.DeadlineExceeded): - err = status.WrapWithDeadlineExceeded(vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API deadline exceeded", err, reqInfo, resInfo) - attrs = trace.StatusCodeDeadlineExceeded(err.Error()) - case errors.Is(err, errors.ErrObjectIDNotFound(uuid)), errors.Is(err, errors.ErrObjectNotFound(nil, uuid)): - err = status.WrapWithNotFound(vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API uuid "+uuid+"'s object not found", err, reqInfo, resInfo) - attrs = trace.StatusCodeNotFound(err.Error()) - default: - code := codes.Unknown - if err == nil { - err = errors.ErrObjectIDNotFound(uuid) - code = codes.NotFound - } - st, msg, err = status.ParseError(err, code, vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) - attrs = trace.FromGRPCStatus(st.Code(), msg) - } - if span != nil { + vec, err := s.GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: uuid, + }, + }) + if err != nil { + st, msg, err := status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.UpdateRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if span != nil && st != nil { span.RecordError(err) - span.SetAttributes(attrs...) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), msg)...) span.SetStatus(trace.StatusError, err.Error()) } return nil, err @@ -2121,6 +2007,221 @@ func (s *server) MultiUpdate( return locs, errs } +func (s *server) UpdateTimestamp( + ctx context.Context, req *payload.Object_Timestamp, +) (res *payload.Object_Location, err error) { + ctx, span := trace.StartSpan(grpc.WithGRPCMethod(ctx, vald.PackageName+"."+vald.UpdateRPCServiceName+"/"+vald.UpdateTimestampRPCName), apiName+"/"+vald.UpdateTimestampRPCName) + defer func() { + if span != nil { + span.End() + } + }() + uuid := req.GetId() + reqInfo := &errdetails.RequestInfo{ + RequestId: uuid, + ServingData: errdetails.Serialize(req), + } + resInfo := &errdetails.ResourceInfo{ + ResourceType: errdetails.ValdGRPCResourceTypePrefix + "/vald.v1." + vald.UpdateTimestampRPCName + "." + vald.GetObjectRPCName, + ResourceName: fmt.Sprintf("%s: %s(%s) to %v", apiName, s.name, s.ip, s.gateway.Addrs(ctx)), + } + if len(uuid) == 0 { + err = errors.ErrInvalidMetaDataConfig + err = status.WrapWithInvalidArgument(vald.UpdateTimestampRPCName+" API invalid uuid", err, reqInfo, resInfo, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "invalid id", + Description: err.Error(), + }, + }, + }) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + ts := req.GetTimestamp() + if ts < 0 { + err = errors.ErrInvalidTimestamp(ts) + err = status.WrapWithInvalidArgument(vald.UpdateTimestampRPCName+" API invalid vector argument", err, reqInfo, resInfo, + &errdetails.BadRequest{ + FieldViolations: []*errdetails.BadRequestFieldViolation{ + { + Field: "timestamp", + Description: err.Error(), + }, + }, + }, info.Get()) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeInvalidArgument(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + var ( + mu sync.RWMutex + aeCount atomic.Uint64 + updated atomic.Uint64 + ls = make([]string, 0, s.replica) + visited = make(map[string]bool, s.replica) + locs = &payload.Object_Location{ + Uuid: uuid, + Ips: make([]string, 0, s.replica), + } + ) + err = s.gateway.BroadCast(ctx, service.WRITE, func(ctx context.Context, target string, vc vald.Client, copts ...grpc.CallOption) (err error) { + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "BroadCast/"+target), apiName+"/"+vald.UpdateRPCName+"/"+target) + defer func() { + if span != nil { + span.End() + } + }() + loc, err := vc.UpdateTimestamp(ctx, req, copts...) + if err != nil { + st, ok := status.FromError(err) + if ok && st != nil { + if st.Code() != codes.AlreadyExists && + st.Code() != codes.Canceled && + st.Code() != codes.DeadlineExceeded && + st.Code() != codes.InvalidArgument && + st.Code() != codes.NotFound && + st.Code() != codes.OK && + st.Code() != codes.Unimplemented { + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), fmt.Sprintf("UpdateTimestamp operation for Agent %s failed,\terror: %v", target, err))...) + span.SetStatus(trace.StatusError, err.Error()) + } + return err + } + if st.Code() == codes.AlreadyExists { + host, _, err := net.SplitHostPort(target) + if err != nil { + host = target + } + aeCount.Add(1) + mu.Lock() + visited[target] = true + locs.Ips = append(locs.GetIps(), host) + ls = append(ls, host) + mu.Unlock() + + } + } + return nil + } + if loc != nil { + updated.Add(1) + mu.Lock() + visited[target] = true + locs.Ips = append(locs.GetIps(), loc.GetIps()...) + ls = append(ls, loc.GetName()) + mu.Unlock() + } + return nil + }) + switch { + case err != nil: + st, msg, err := status.ParseError(err, codes.Internal, + "failed to parse "+vald.UpdateRPCName+" gRPC error response", reqInfo, resInfo, info.Get()) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), msg)...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + case len(locs.Ips) <= 0: + err = errors.ErrIndexNotFound + err = status.WrapWithNotFound(vald.UpdateRPCName+" API update target not found", err, reqInfo, resInfo) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeNotFound(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + case updated.Load()+aeCount.Load() < uint64(s.replica): + shortage := s.replica - int(updated.Load()+aeCount.Load()) + vec, err := s.GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: uuid, + }, + }) + if err != nil { + st, msg, err := status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.UpdateTimestampRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if span != nil && st != nil { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), msg)...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + + err = s.gateway.DoMulti(ctx, shortage, func(ctx context.Context, target string, vc vald.Client, copts ...grpc.CallOption) (err error) { + mu.RLock() + tf, ok := visited[target] + mu.RUnlock() + if tf && ok { + return errors.Errorf("target: %s already inserted will skip", target) + } + ctx, span := trace.StartSpan(grpc.WrapGRPCMethod(ctx, "DoMulti/"+target), apiName+"/"+vald.InsertRPCName+"/"+target) + defer func() { + if span != nil { + span.End() + } + }() + loc, err := vc.Insert(ctx, &payload.Insert_Request{ + Vector: vec, + Config: &payload.Insert_Config{ + SkipStrictExistCheck: true, + Timestamp: ts, + }, + }, copts...) + if err != nil { + st, ok := status.FromError(err) + if ok && st != nil && span != nil { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), fmt.Sprintf("Shortage index Insert for Update operation for Agent %s failed,\terror: %v", target, err))...) + span.SetStatus(trace.StatusError, err.Error()) + } + return err + } + if loc != nil { + updated.Add(1) + mu.Lock() + locs.Ips = append(locs.GetIps(), loc.GetIps()...) + ls = append(ls, loc.GetName()) + mu.Unlock() + } + return nil + }) + if err != nil { + st, msg, err := status.ParseError(err, codes.Unknown, vald.InsertRPCName+" API for "+vald.UpdateRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if span != nil && st != nil { + span.RecordError(err) + span.SetAttributes(trace.FromGRPCStatus(st.Code(), msg)...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + } + case updated.Load() == 0 && aeCount.Load() > 0: + err = status.WrapWithAlreadyExists(vald.UpdateRPCName+" API update target same vector already exists", errors.ErrSameVectorAlreadyExists(uuid, nil, nil), reqInfo, resInfo) + if span != nil { + span.RecordError(err) + span.SetAttributes(trace.StatusCodeAlreadyExists(err.Error())...) + span.SetStatus(trace.StatusError, err.Error()) + } + return nil, err + + } + slices.Sort(ls) + locs.Name = strings.Join(ls, ",") + return locs, nil +} + func (s *server) Upsert( ctx context.Context, req *payload.Upsert_Request, ) (loc *payload.Object_Location, err error) { @@ -2180,48 +2281,23 @@ func (s *server) Upsert( } var shouldInsert bool if !req.GetConfig().GetSkipStrictExistCheck() { - vec, err := s.getObject(ctx, uuid) - var ( - attrs trace.Attributes - st *status.Status - msg string - ) + vec, err := s.GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: uuid, + }, + }) + var attrs trace.Attributes if err != nil || vec == nil { - switch { - case errors.Is(err, errors.ErrInvalidUUID(uuid)): - err = status.WrapWithInvalidArgument( - vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API invalid argument for uuid \""+uuid+"\" detected", - err, - reqInfo, - resInfo, - &errdetails.BadRequest{ - FieldViolations: []*errdetails.BadRequestFieldViolation{ - { - Field: "uuid", - Description: err.Error(), - }, - }, - }, - ) - attrs = trace.StatusCodeInvalidArgument(err.Error()) - case errors.Is(err, errors.ErrGRPCClientConnNotFound("*")): - err = status.WrapWithInternal(vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API connection not found", err, reqInfo, resInfo) - attrs = trace.StatusCodeInternal(err.Error()) - case errors.Is(err, context.Canceled): - err = status.WrapWithCanceled(vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API canceled", err, reqInfo, resInfo) - attrs = trace.StatusCodeCancelled(err.Error()) - case errors.Is(err, context.DeadlineExceeded): - err = status.WrapWithDeadlineExceeded(vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API deadline exceeded", err, reqInfo, resInfo) - attrs = trace.StatusCodeDeadlineExceeded(err.Error()) - case errors.Is(err, errors.ErrObjectIDNotFound(uuid)), errors.Is(err, errors.ErrObjectNotFound(nil, uuid)): - err = nil - shouldInsert = true - default: - st, msg, err = status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + var ( + st *status.Status + msg string + ) + st, msg, err = status.ParseError(err, codes.Unknown, vald.GetObjectRPCName+" API for "+vald.UpsertRPCName+" API uuid "+uuid+"'s request returned error", reqInfo, resInfo) + if st != nil { attrs = trace.FromGRPCStatus(st.Code(), msg) - if st != nil && st.Code() == codes.NotFound { - err = nil + if st.Code() == codes.NotFound { shouldInsert = true + err = nil } } } else if conv.F32stos(vec.GetVector()) == conv.F32stos(req.GetVector().GetVector()) { @@ -2236,7 +2312,6 @@ func (s *server) Upsert( } return nil, err } - } else { id, err := s.exists(ctx, uuid) if err != nil { diff --git a/pkg/gateway/lb/service/gateway.go b/pkg/gateway/lb/service/gateway.go index 53c259c9e1..2d806b20fc 100644 --- a/pkg/gateway/lb/service/gateway.go +++ b/pkg/gateway/lb/service/gateway.go @@ -24,6 +24,7 @@ import ( "github.com/vdaas/vald/apis/grpc/v1/vald" "github.com/vdaas/vald/internal/client/v1/client/discoverer" + vc "github.com/vdaas/vald/internal/client/v1/client/vald" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/net/grpc" "github.com/vdaas/vald/internal/observability/trace" @@ -94,7 +95,7 @@ func (g *gateway) BroadCast( case <-ictx.Done(): return nil default: - err = f(ictx, addr, vald.NewValdClient(conn), copts...) + err = f(ictx, addr, vc.NewValdClient(conn), copts...) if err != nil { return err } @@ -129,7 +130,7 @@ func (g *gateway) DoMulti( copts ...grpc.CallOption, ) (err error) { if atomic.LoadUint32(&cur) < limit { - err = f(ictx, addr, vald.NewValdClient(conn), copts...) + err = f(ictx, addr, vc.NewValdClient(conn), copts...) if err != nil { return err } @@ -147,7 +148,7 @@ func (g *gateway) DoMulti( if atomic.LoadUint32(&cur) < limit { _, ok := visited.Load(addr) if !ok { - err = f(ictx, addr, vald.NewValdClient(conn), copts...) + err = f(ictx, addr, vc.NewValdClient(conn), copts...) if err != nil { return err } diff --git a/pkg/index/job/correction/service/corrector.go b/pkg/index/job/correction/service/corrector.go index 41a71b2274..53cb22a3b8 100644 --- a/pkg/index/job/correction/service/corrector.go +++ b/pkg/index/job/correction/service/corrector.go @@ -136,15 +136,22 @@ func (c *correct) Start(ctx context.Context) (err error) { return err } counts := detail.GetCounts() - agents := make([]string, 0, detail.GetLiveAgents()) - for agent, count := range counts { - log.Infof("index info: addr(%s), stored(%d), uncommitted(%d), indexing=%t, saving=%t", agent, count.GetStored(), count.GetUncommitted(), count.GetIndexing(), count.GetSaving()) + agents := make([]string, 0, len(counts)) + for agent := range counts { agents = append(agents, agent) } slices.SortFunc(agents, func(left, right string) int { - return cmp.Compare(counts[left].GetStored(), counts[right].GetStored()) + return cmp.Compare(counts[right].GetStored(), counts[left].GetStored()) }) + for _, agent := range agents { + count, ok := counts[agent] + if ok && count != nil { + log.Infof("index info: addr(%s), stored(%d), uncommitted(%d), indexing=%t, saving=%t", agent, count.GetStored(), count.GetUncommitted(), count.GetIndexing(), count.GetSaving()) + } + } + log.Infof("sorted agents: %v,\tdiscovered agents: %v", agents, c.discoverer.GetAddrs(ctx)) + errs := make([]error, 0, len(agents)) emptyReq := new(payload.Object_List_Request) @@ -177,24 +184,26 @@ func (c *correct) Start(ctx context.Context) (err error) { uncommitted uint32 indexing bool saving bool + debugMsg string ) count, ok := counts[addr] if ok && count != nil { stored = count.GetStored() uncommitted = count.GetUncommitted() + indexing = count.GetIndexing() + saving = count.GetSaving() + debugMsg = fmt.Sprintf("agent %s (total index detail = stored: %d, uncommitted: %d, indexing=%t, saving=%t), stream concurrency: %d, processing %d/%d, replicas: size(%d) = addrs%v", addr, stored, uncommitted, indexing, saving, c.streamListConcurrency, corrected, len(agents), len(replicas), replicas) if stored+uncommitted == 0 { // id no indices in agent skip process + log.Warnf("skipping index correction process due to zero index detected for %s", debugMsg) return nil } - indexing = count.GetIndexing() - saving = count.GetSaving() } - debugMsg := fmt.Sprintf("agent %s (stored: %d, uncommitted: %d, indexing=%t, saving=%t), stream concurrency: %d, processing %d/%d, replicas: size(%d) = addrs%v", addr, stored, uncommitted, indexing, saving, c.streamListConcurrency, corrected, len(agents), len(replicas), replicas) eg, egctx := errgroup.WithContext(ctx) eg.SetLimit(c.streamListConcurrency) ctx, cancel := context.WithCancelCause(egctx) - stream, err := vald.NewObjectClient(conn).StreamListObject(ctx, emptyReq, copts...) + stream, err := vc.NewValdClient(conn).StreamListObject(ctx, emptyReq, copts...) if err != nil { return err } @@ -225,27 +234,12 @@ func (c *correct) Start(ctx context.Context) (err error) { } else { cancel(errors.ErrStreamListObjectStreamFinishedUnexpectedly(err)) } - } else { + } else if res != nil && res.GetVector() != nil && res.GetVector().GetId() != "" && res.GetVector().GetTimestamp() < start.UnixNano() { eg.Go(safety.RecoverFunc(func() (err error) { vec := res.GetVector() - if vec == nil || vec.GetId() == "" { - st := res.GetStatus() - if st != nil { - log.Errorf("invalid vector id: %s detected and returned status code: %d, message: %s, details: %v, debug: %s", vec.GetId(), st.GetCode(), st.GetMessage(), st.GetDetails(), debugMsg) - } - return errors.ErrFailedToReceiveVectorFromStream - } - - // skip if the vector is inserted after correction start - if vec.GetTimestamp() > start.UnixNano() { - log.Debugf("index correction process for ID: %s skipped due to newer timestamp detected. job started at %s but object timestamp is %s", - vec.GetId(), - start.Format(time.RFC3339Nano), - time.Unix(0, vec.GetTimestamp()).Format(time.RFC3339Nano)) - return nil - } - + ts := vec.GetTimestamp() id := vec.GetId() + _, ok, err := c.checkedList.Get(id) if err != nil { log.Errorf("failed to perform Get from check list but still try to finish processing without cache: %v", err) @@ -264,161 +258,250 @@ func (c *correct) Start(ctx context.Context) (err error) { // Therefore, the process is only to correct the missing replicas. if len(replicas) <= 0 { diff := c.indexReplica - 1 - addrs := c.discoverer.GetAddrs(egctx) // correct index replica shortage if diff > 0 { - log.Infof("replica shortage(diff=%d) of vector id: %s detected from last %s. inserting to other agents = %v", diff, id, debugMsg, addrs) - if len(addrs) == 0 { - return errors.ErrNoAvailableAgentToInsert - } - req := &payload.Insert_Request{ - Vector: vec, - // TODO: this should be deleted after Config.Timestamp deprecation - Config: &payload.Insert_Config{ - Timestamp: vec.GetTimestamp(), - }, - } - for _, daddr := range addrs { - if diff > 0 && daddr != addr { - _, err := c.discoverer.GetClient().Do(grpc.WithGRPCMethod(egctx, vald.PackageName+"."+vald.InsertRPCServiceName+"/"+vald.InsertRPCName), daddr, func(ctx context.Context, - conn *grpc.ClientConn, - copts ...grpc.CallOption, - ) (any, error) { - client := vald.NewValdClient(conn) - _, err := client.Insert(ctx, req, copts...) - if err != nil { - if st, ok := status.FromError(err); !ok { - log.Errorf("gRPC call returned not a gRPC status error: %v", err) - return nil, err - } else if st.Code() == codes.AlreadyExists { - obj, err := client.GetObject(ctx, &payload.Object_VectorRequest{ - Id: &payload.Object_ID{ - Id: id, - }, - }, copts...) - if err != nil { - if st, ok = status.FromError(err); !ok { - log.Errorf("gRPC call returned not a gRPC status error: %v", err) - return nil, err - } else if st.Code() == codes.NotFound { - return nil, nil - } else if st.Code() == codes.Canceled { - return nil, nil - } - return nil, err - } - if obj.GetTimestamp() < vec.GetTimestamp() { - _, err := client.Update(ctx, &payload.Update_Request{ - Vector: vec, - // TODO: this should be deleted after Config.Timestamp deprecation - Config: &payload.Update_Config{ - // TODO: Decrementing because it's gonna be incremented befor being pushed - // to vqueue in the agent. This is a not ideal workaround for the current vqueue implementation - // so we should consider refactoring vqueue. - Timestamp: vec.GetTimestamp() - 1, - }, - }, copts...) - if err != nil { - if st, ok = status.FromError(err); !ok { - log.Errorf("gRPC call returned not a gRPC status error: %v", err) - return nil, err - } else if st.Code() == codes.NotFound { - return nil, nil - } else if st.Code() == codes.Canceled { - return nil, nil - } - return nil, err - } - c.correctedOldIndexCount.Add(1) - } - diff-- - c.correctedReplicationCount.Add(1) - return nil, nil - } else if st.Code() == codes.Canceled { - return nil, nil - } - return nil, err - } - diff-- - c.correctedReplicationCount.Add(1) - return nil, nil - }) - if err != nil { - log.Error(fmt.Errorf("failed to insert object to agent(%s): %w", daddr, err)) - } - } - } + return c.correctShortage(egctx, id, addr, debugMsg, vec, make(map[string]*payload.Object_Timestamp), diff) } return nil } - var ( - latest int64 - mu sync.Mutex - found = make(map[string]*payload.Object_Timestamp, len(addr)) - latestAgent = addr - ) // load index replica from other agents and store it to found map - if err := c.discoverer.GetClient().OrderedRangeConcurrent(egctx, replicas, len(replicas), - func(ctx context.Context, addr string, conn *grpc.ClientConn, copts ...grpc.CallOption) error { - ots, err := vald.NewObjectClient(conn).GetTimestamp(ctx, &payload.Object_TimestampRequest{ - Id: &payload.Object_ID{ - Id: id, - }, - }) - if err != nil { - if st, ok := status.FromError(err); !ok { - log.Errorf("gRPC call GetTimestamp to agent: %s, id: %s returned not a gRPC status error: %v", addr, id, err) - return err - } else if st.Code() == codes.NotFound { - // when replica of agent > index replica, this happens - return nil - } else if st.Code() == codes.Canceled { - return nil - } else { - log.Errorf("failed to GetTimestamp with unexpected error. agent: %s, id: %s, code: %v, message: %s", addr, id, st.Code(), st.Message()) - return err - } + found, skipped, latest, latestAgent, err := c.loadReplicaInfo(egctx, addr, id, replicas, counts, ts, start) + if err != nil { + return err + } + if len(found) != 0 && ((len(replicas) > 0 && len(skipped) == 0) || (len(skipped) > 0 && len(skipped) < len(replicas))) { + // current object timestamp is not latest get latest object from other agent index replica + if ts < latest && latestAgent != addr { + latestObject := c.getLatestObject(egctx, id, addr, latestAgent, latest) + if latestObject != nil && latestObject.GetVector() != nil && latestObject.GetId() != "" && latestObject.GetTimestamp() >= latest { + vec = latestObject } + } + c.correctTimestamp(ctx, id, vec, found) + } else if len(skipped) > 0 { + log.Debugf("timestamp correction for index id %s skipped, replica %s, skipped agents: %v", id, addr, skipped) + } + diff := c.indexReplica - (len(found) + 1) + if diff > 0 { // correct index replica shortage + return c.correctShortage(egctx, id, addr, debugMsg, vec, found, diff) + } else if diff < 0 { // correct index replica oversupply + return c.correctOversupply(egctx, id, addr, debugMsg, found, diff) + } + return nil + })) + } + } + } + }); err != nil { + // This only happens when ErrGRPCClientConnNotFound is returned. + // In other cases, OrderedRange continues processing, so error is used to keep track of the error status of correction. + errs = append(errs, err) + } + if len(errs) != 0 { + return errors.Join(errs...) + } - // skip if the vector is inserted after correction start - if ots.GetTimestamp() > start.UnixNano() { - log.Debugf("timestamp of vector(id: %s, timestamp: %v) is newer than correction start time(%v). skipping...", - ots.GetId(), - ots.GetTimestamp(), - start.UnixNano(), - ) - return nil - } - mu.Lock() - found[addr] = ots - if latest < ots.GetTimestamp() { - latest = ots.GetTimestamp() - if latest > vec.GetTimestamp() { - latestAgent = addr - } - } - mu.Unlock() - return nil + return nil +} + +func (c *correct) PreStop(_ context.Context) error { + log.Info("removing persistent cache files...") + return c.checkedList.Close(true) +} + +func (c *correct) NumberOfCheckedIndex() uint64 { + return c.checkedIndexCount.Load() +} + +func (c *correct) NumberOfCorrectedOldIndex() uint64 { + return c.correctedOldIndexCount.Load() +} + +func (c *correct) NumberOfCorrectedReplication() uint64 { + return c.correctedReplicationCount.Load() +} + +func (c *correct) loadReplicaInfo( + ctx context.Context, + originAddr, id string, + replicas []string, + counts map[string]*payload.Info_Index_Count, + ts int64, + start time.Time, +) ( + found map[string]*payload.Object_Timestamp, + skipped []string, + latest int64, + latestAgent string, + err error, +) { + var mu sync.Mutex + latestAgent = originAddr + skipped = make([]string, 0, len(replicas)) + found = make(map[string]*payload.Object_Timestamp, c.indexReplica-1) + err = c.discoverer.GetClient().OrderedRangeConcurrent(ctx, replicas, len(replicas), + func(ctx context.Context, addr string, conn *grpc.ClientConn, copts ...grpc.CallOption) error { + if originAddr == addr { + return nil + } + count, ok := counts[addr] // counts is read-only we don't need to lock. + if ok && count != nil && count.GetStored() == 0 && count.GetUncommitted() == 0 { + mu.Lock() + skipped = append(skipped, addr) + mu.Unlock() + return nil + } + + ots, err := vc.NewValdClient(conn).GetTimestamp(ctx, &payload.Object_TimestampRequest{ + Id: &payload.Object_ID{ + Id: id, + }, + }) + if err != nil { + if st, ok := status.FromError(err); !ok || st == nil { + log.Errorf("gRPC call GetTimestamp to agent: %s, id: %s returned not a gRPC status error: %v", addr, id, err) + return err + } else if st.Code() == codes.NotFound { + // when replica of agent > index replica, this happens + return nil + } else if st.Code() == codes.Canceled { + return nil + } else { + log.Errorf("failed to GetTimestamp with unexpected error. agent: %s, id: %s, code: %v, message: %s", addr, id, st.Code(), st.Message()) + return err + } + } + + if ots == nil { + // not found + return nil + } + + // skip if the vector is inserted after correction start + if ots.GetTimestamp() > start.UnixNano() { + log.Debugf("timestamp of vector(id: %s, timestamp: %v) is newer than correction start time(%v). skipping...", + ots.GetId(), + ots.GetTimestamp(), + start.UnixNano(), + ) + return nil + } + mu.Lock() + found[addr] = ots + if latest < ots.GetTimestamp() { + latest = ots.GetTimestamp() + if latest > ts { + latestAgent = addr + } + } + mu.Unlock() + return nil + }, + ) + return +} + +func (c *correct) getLatestObject( + ctx context.Context, id, addr, latestAgent string, latest int64, +) (latestObject *payload.Object_Vector) { + _, err := c.discoverer.GetClient().Do(grpc.WithGRPCMethod(ctx, vald.PackageName+"."+vald.ObjectRPCServiceName+"/"+vald.GetObjectRPCName), latestAgent, func(ctx context.Context, + conn *grpc.ClientConn, + copts ...grpc.CallOption, + ) (any, error) { + obj, err := vc.NewValdClient(conn).GetObject(ctx, &payload.Object_VectorRequest{ + Id: &payload.Object_ID{ + Id: id, + }, + }, copts...) + if err != nil { + if st, ok := status.FromError(err); !ok || st == nil { + log.Errorf("gRPC call returned not a gRPC status error: %v", err) + return nil, err + } else if st.Code() == codes.NotFound { + return nil, nil + } else if st.Code() == codes.Canceled { + return nil, nil + } + return nil, err + } + if ots == nil { + // not found + return nil, nil + } + if obj.GetTimestamp() >= latest && obj.GetId() != "" && obj.GetVector() != nil { + latestObject = obj + } + return obj, nil + }) + if err != nil { + log.Errorf("failed to load latest object id: %s, agent: %s, timestamp: %d, error: %v", id, addr, latest, err) + } + if latestObject != nil && latestObject.GetTimestamp() < latest { + latestObject.Timestamp = latest + } + return latestObject +} + +func (c *correct) correctTimestamp( + ctx context.Context, + id string, + latestObject *payload.Object_Vector, + found map[string]*payload.Object_Timestamp, +) { + tss := time.Unix(0, latestObject.GetTimestamp()).Format(time.RFC3339Nano) // timestamp string + for addr, ots := range found { // correct timestamp inconsistency + if latestObject.GetTimestamp() > ots.GetTimestamp() { + log.Infof("timestamp inconsistency detected with vector(id: %s, timestamp: %s). updating with the latest vector(id: %s, timestamp: %s)", + ots.GetId(), + time.Unix(0, ots.GetTimestamp()).Format(time.RFC3339Nano), + latestObject.GetId(), + tss, + ) + _, err := c.discoverer.GetClient().Do(grpc.WithGRPCMethod(ctx, vald.PackageName+"."+vald.UpdateRPCServiceName+"/"+vald.UpdateRPCName), addr, func(ctx context.Context, + conn *grpc.ClientConn, + copts ...grpc.CallOption, + ) (any, error) { + client := vc.NewValdClient(conn) + // TODO: use UpdateTimestamp when it's implemented because here we just want to update only the timestamp but not the vector + _, err := client.Update(ctx, &payload.Update_Request{ + Vector: latestObject, + // TODO: this should be deleted after Config.Timestamp deprecation + Config: &payload.Update_Config{ + // TODO: Decrementing because it's gonna be incremented befor being pushed + // to vqueue in the agent. This is a not ideal workaround for the current vqueue implementation + // so we should consider refactoring vqueue. + Timestamp: latestObject.GetTimestamp() - 1, + }, + }, copts...) + if err != nil { + if st, ok := status.FromError(err); !ok || st == nil { + log.Errorf("gRPC call returned not a gRPC status error: %v", err) + return nil, err + } else if st.Code() == codes.NotFound { + _, err = client.Insert(ctx, &payload.Insert_Request{ + Vector: latestObject, + // TODO: this should be deleted after Config.Timestamp deprecation + Config: &payload.Insert_Config{ + // TODO: Decrementing because it's gonna be incremented befor being pushed + // to vqueue in the agent. This is a not ideal workaround for the current vqueue implementation + // so we should consider refactoring vqueue. + Timestamp: latestObject.GetTimestamp(), }, - ); err != nil { - return err - } - latestObject := vec - - // current object timestamp is not latest get latest object from other agent index replica - if vec.GetTimestamp() < latest && latestAgent != addr { - _, err := c.discoverer.GetClient().Do(grpc.WithGRPCMethod(egctx, vald.PackageName+"."+vald.ObjectRPCServiceName+"/"+vald.GetObjectRPCName), latestAgent, func(ctx context.Context, - conn *grpc.ClientConn, - copts ...grpc.CallOption, - ) (any, error) { - obj, err := vald.NewObjectClient(conn).GetObject(ctx, &payload.Object_VectorRequest{ + }, copts...) + if err != nil { + if st, ok = status.FromError(err); !ok || st == nil { + log.Errorf("gRPC call returned not a gRPC status error: %v", err) + return nil, err + } else if st.Code() == codes.AlreadyExists { + var obj *payload.Object_Vector + obj, err = client.GetObject(ctx, &payload.Object_VectorRequest{ Id: &payload.Object_ID{ Id: id, }, }, copts...) if err != nil { - if st, ok := status.FromError(err); !ok { + if st, ok = status.FromError(err); !ok || st == nil { log.Errorf("gRPC call returned not a gRPC status error: %v", err) return nil, err } else if st.Code() == codes.NotFound { @@ -428,34 +511,8 @@ func (c *correct) Start(ctx context.Context) (err error) { } return nil, err } - if obj.GetTimestamp() >= latest && obj.GetId() != "" && obj.GetVector() != nil { - latestObject = obj - } - return obj, nil - }) - if err != nil { - log.Error(fmt.Errorf("failed to load latest object id: %s, agent: %s, timestamp: %d, error: %w", id, addr, latest, err)) - } - } - if latestObject.Timestamp < latest { - latestObject.Timestamp = latest - } - tss := time.Unix(0, latestObject.GetTimestamp()).Format(time.RFC3339Nano) // timestamp string - for addr, ots := range found { // correct timestamp inconsistency - if latestObject.GetTimestamp() > ots.GetTimestamp() { - log.Infof("timestamp inconsistency detected with vector(id: %s, timestamp: %s). updating with the latest vector(id: %s, timestamp: %s)", - ots.GetId(), - time.Unix(0, ots.GetTimestamp()).Format(time.RFC3339Nano), - latestObject.GetId(), - tss, - ) - _, err := c.discoverer.GetClient().Do(grpc.WithGRPCMethod(egctx, vald.PackageName+"."+vald.UpdateRPCServiceName+"/"+vald.UpdateRPCName), addr, func(ctx context.Context, - conn *grpc.ClientConn, - copts ...grpc.CallOption, - ) (any, error) { - client := vald.NewValdClient(conn) - // TODO: use UpdateTimestamp when it's implemented because here we just want to update only the timestamp but not the vector - _, err := client.Update(ctx, &payload.Update_Request{ + if obj != nil && obj.GetTimestamp() < latestObject.GetTimestamp() { + _, err = client.Update(ctx, &payload.Update_Request{ Vector: latestObject, // TODO: this should be deleted after Config.Timestamp deprecation Config: &payload.Update_Config{ @@ -466,247 +523,186 @@ func (c *correct) Start(ctx context.Context) (err error) { }, }, copts...) if err != nil { - if st, ok := status.FromError(err); !ok { + if st, ok = status.FromError(err); !ok || st == nil { log.Errorf("gRPC call returned not a gRPC status error: %v", err) return nil, err } else if st.Code() == codes.NotFound { - _, err = client.Insert(ctx, &payload.Insert_Request{ - Vector: latestObject, - // TODO: this should be deleted after Config.Timestamp deprecation - Config: &payload.Insert_Config{ - // TODO: Decrementing because it's gonna be incremented befor being pushed - // to vqueue in the agent. This is a not ideal workaround for the current vqueue implementation - // so we should consider refactoring vqueue. - Timestamp: latestObject.GetTimestamp(), - }, - }, copts...) - if err != nil { - if st, ok = status.FromError(err); !ok { - log.Errorf("gRPC call returned not a gRPC status error: %v", err) - return nil, err - } else if st.Code() == codes.AlreadyExists { - obj, err := client.GetObject(ctx, &payload.Object_VectorRequest{ - Id: &payload.Object_ID{ - Id: id, - }, - }, copts...) - if err != nil { - if st, ok = status.FromError(err); !ok { - log.Errorf("gRPC call returned not a gRPC status error: %v", err) - return nil, err - } else if st.Code() == codes.NotFound { - return nil, nil - } else if st.Code() == codes.Canceled { - return nil, nil - } - return nil, err - } - if obj.GetTimestamp() < latestObject.GetTimestamp() { - _, err = client.Update(ctx, &payload.Update_Request{ - Vector: latestObject, - // TODO: this should be deleted after Config.Timestamp deprecation - Config: &payload.Update_Config{ - // TODO: Decrementing because it's gonna be incremented befor being pushed - // to vqueue in the agent. This is a not ideal workaround for the current vqueue implementation - // so we should consider refactoring vqueue. - Timestamp: latestObject.GetTimestamp() - 1, - }, - }, copts...) - if err != nil { - if st, ok = status.FromError(err); !ok { - log.Errorf("gRPC call returned not a gRPC status error: %v", err) - return nil, err - } else if st.Code() == codes.NotFound { - return nil, nil - } else if st.Code() == codes.Canceled { - return nil, nil - } - return nil, err - } - } - return nil, nil - } else if st.Code() == codes.Canceled { - return nil, nil - } - return nil, err - } - c.correctedOldIndexCount.Add(1) return nil, nil } else if st.Code() == codes.Canceled { return nil, nil } return nil, err } - log.Infof("vector successfully updated. address: %s, uuid: %s, timestamp: %s", addr, latestObject.GetId(), tss) c.correctedOldIndexCount.Add(1) - return nil, nil - }) - if err != nil { - log.Error(fmt.Errorf("failed to fix timestamp to %s for id %s agent %s error: %w", tss, id, addr, err)) } + return nil, nil + } else if st.Code() == codes.Canceled { + return nil, nil } + return nil, err } - currentNumberOfIndexReplica := len(found) + 1 - diff := c.indexReplica - currentNumberOfIndexReplica - addrs := c.discoverer.GetAddrs(egctx) - if diff > 0 { // correct index replica shortage - log.Infof("replica shortage(diff=%d) of vector id: %s detected for %s. inserting to other agents = %v", diff, id, debugMsg, addrs) - if len(addrs) == 0 { - return errors.ErrNoAvailableAgentToInsert - } - req := &payload.Insert_Request{ - Vector: latestObject, - // TODO: this should be deleted after Config.Timestamp deprecation - Config: &payload.Insert_Config{ - Timestamp: latestObject.GetTimestamp(), - }, - } - for _, daddr := range addrs { - if diff > 0 && daddr != addr { - _, ok := found[daddr] - if !ok { - _, err := c.discoverer.GetClient().Do(grpc.WithGRPCMethod(egctx, vald.PackageName+"."+vald.InsertRPCServiceName+"/"+vald.InsertRPCName), daddr, func(ctx context.Context, - conn *grpc.ClientConn, - copts ...grpc.CallOption, - ) (any, error) { - client := vald.NewValdClient(conn) - _, err := client.Insert(ctx, req, copts...) - if err != nil { - if st, ok := status.FromError(err); !ok { - log.Errorf("gRPC call returned not a gRPC status error: %v", err) - return nil, err - } else if st.Code() == codes.AlreadyExists { - obj, err := client.GetObject(ctx, &payload.Object_VectorRequest{ - Id: &payload.Object_ID{ - Id: id, - }, - }, copts...) - if err != nil { - if st, ok = status.FromError(err); !ok { - log.Errorf("gRPC call returned not a gRPC status error: %v", err) - return nil, err - } else if st.Code() == codes.NotFound { - return nil, nil - } else if st.Code() == codes.Canceled { - return nil, nil - } - return nil, err - } - if obj.GetTimestamp() < latestObject.GetTimestamp() { - _, err = client.Update(ctx, &payload.Update_Request{ - Vector: latestObject, - // TODO: this should be deleted after Config.Timestamp deprecation - Config: &payload.Update_Config{ - // TODO: Decrementing because it's gonna be incremented befor being pushed - // to vqueue in the agent. This is a not ideal workaround for the current vqueue implementation - // so we should consider refactoring vqueue. - Timestamp: latestObject.GetTimestamp() - 1, - }, - }, copts...) - if err != nil { - if st, ok = status.FromError(err); !ok { - log.Errorf("gRPC call returned not a gRPC status error: %v", err) - return nil, err - } else if st.Code() == codes.NotFound { - return nil, nil - } else if st.Code() == codes.Canceled { - return nil, nil - } - return nil, err - } - c.correctedOldIndexCount.Add(1) - } - diff-- - c.correctedReplicationCount.Add(1) - return nil, nil - } else if st.Code() == codes.Canceled { - return nil, nil - } - return nil, err - } - diff-- - c.correctedReplicationCount.Add(1) - return nil, nil - }) - if err != nil { - log.Error(fmt.Errorf("failed to insert object to agent(%s): %w", daddr, err)) - } - } - } - } - } else if diff < 0 { // correct index replica oversupply - log.Infof("replica oversupply of vector %s. deleting...", id) - if len(addrs) == 0 { - return errors.ErrNoAvailableAgentToRemove - } - req := &payload.Remove_Request{ + c.correctedOldIndexCount.Add(1) + return nil, nil + } else if st.Code() == codes.Canceled { + return nil, nil + } + return nil, err + } + log.Infof("vector successfully updated. address: %s, uuid: %s, timestamp: %s", addr, latestObject.GetId(), tss) + c.correctedOldIndexCount.Add(1) + return nil, nil + }) + if err != nil { + log.Error(fmt.Errorf("failed to fix timestamp to %s for id %s agent %s error: %w", tss, id, addr, err)) + } + } + } +} + +func (c *correct) correctOversupply( + ctx context.Context, + id, selfAddr, debugMsg string, + found map[string]*payload.Object_Timestamp, + diff int, +) (err error) { + addrs := c.discoverer.GetAddrs(ctx) + log.Infof("replica oversupply(configured: %d, stored: %d, diff: %d) of vector id: %s detected for %s. deleting from agents = %v", c.indexReplica, len(found)+1, diff, id, debugMsg, found) + if len(addrs) == 0 { + return errors.ErrNoAvailableAgentToRemove + } + req := &payload.Remove_Request{ + Id: &payload.Object_ID{ + Id: id, + }, + } + for _, daddr := range addrs { + if diff < 0 { + _, ok := found[daddr] + if ok || daddr == selfAddr { + _, err := c.discoverer.GetClient().Do(grpc.WithGRPCMethod(ctx, vald.PackageName+"."+vald.RemoveRPCServiceName+"/"+vald.RemoveRPCName), daddr, func(ctx context.Context, + conn *grpc.ClientConn, + copts ...grpc.CallOption, + ) (any, error) { + _, err := vc.NewValdClient(conn).Remove(ctx, req, copts...) + if err != nil { + if st, ok := status.FromError(err); !ok || st == nil { + log.Errorf("gRPC call returned not a gRPC status error: %v", err) + return nil, err + } else if st.Code() == codes.NotFound { + diff++ + c.correctedReplicationCount.Add(1) + return nil, nil + } else if st.Code() == codes.Canceled { + return nil, nil + } + return nil, err + } + diff++ + c.correctedReplicationCount.Add(1) + return nil, nil + }) + if err != nil { + log.Error(fmt.Errorf("failed to delete object from agent(%s): %w", daddr, err)) + } + } + } + } + return nil +} + +func (c *correct) correctShortage( + ctx context.Context, + id, selfAddr, debugMsg string, + latestObject *payload.Object_Vector, + found map[string]*payload.Object_Timestamp, + diff int, +) (err error) { + addrs := c.discoverer.GetAddrs(ctx) + log.Infof("replica shortage(configured: %d, stored: %d, diff: %d) of vector id: %s detected for %s. inserting to other agents = %v", c.indexReplica, len(found)+1, diff, id, debugMsg, addrs) + if len(addrs) == 0 { + return errors.ErrNoAvailableAgentToInsert + } + req := &payload.Insert_Request{ + Vector: latestObject, + // TODO: this should be deleted after Config.Timestamp deprecation + Config: &payload.Insert_Config{ + Timestamp: latestObject.GetTimestamp(), + }, + } + for _, daddr := range addrs { + if diff > 0 && daddr != selfAddr { + _, ok := found[daddr] + if !ok { + _, err := c.discoverer.GetClient().Do(grpc.WithGRPCMethod(ctx, vald.PackageName+"."+vald.InsertRPCServiceName+"/"+vald.InsertRPCName), daddr, func(ctx context.Context, + conn *grpc.ClientConn, + copts ...grpc.CallOption, + ) (any, error) { + client := vc.NewValdClient(conn) + _, err := client.Insert(ctx, req, copts...) + if err != nil { + if st, ok := status.FromError(err); !ok || st == nil { + log.Errorf("gRPC call returned not a gRPC status error: %v", err) + return nil, err + } else if st.Code() == codes.AlreadyExists { + var obj *payload.Object_Vector + obj, err = client.GetObject(ctx, &payload.Object_VectorRequest{ Id: &payload.Object_ID{ Id: id, }, + }, copts...) + if err != nil { + if st, ok = status.FromError(err); !ok || st == nil { + log.Errorf("gRPC call returned not a gRPC status error: %v", err) + return nil, err + } else if st.Code() == codes.NotFound { + return nil, nil + } else if st.Code() == codes.Canceled { + return nil, nil + } + return nil, err } - for _, daddr := range addrs { - if diff < 0 { - _, ok := found[daddr] - if ok || daddr == addr { - _, err := c.discoverer.GetClient().Do(grpc.WithGRPCMethod(egctx, vald.PackageName+"."+vald.RemoveRPCServiceName+"/"+vald.RemoveRPCName), daddr, func(ctx context.Context, - conn *grpc.ClientConn, - copts ...grpc.CallOption, - ) (any, error) { - _, err := vald.NewRemoveClient(conn).Remove(ctx, req, copts...) - if err != nil { - if st, ok := status.FromError(err); !ok { - log.Errorf("gRPC call returned not a gRPC status error: %v", err) - return nil, err - } else if st.Code() == codes.NotFound { - diff++ - c.correctedReplicationCount.Add(1) - return nil, nil - } else if st.Code() == codes.Canceled { - return nil, nil - } - return nil, err - } - diff++ - c.correctedReplicationCount.Add(1) + if obj != nil { + if obj.GetTimestamp() < latestObject.GetTimestamp() { + _, err = client.Update(ctx, &payload.Update_Request{ + Vector: latestObject, + // TODO: this should be deleted after Config.Timestamp deprecation + Config: &payload.Update_Config{ + // TODO: Decrementing because it's gonna be incremented befor being pushed + // to vqueue in the agent. This is a not ideal workaround for the current vqueue implementation + // so we should consider refactoring vqueue. + Timestamp: latestObject.GetTimestamp() - 1, + }, + }, copts...) + if err != nil { + if st, ok = status.FromError(err); !ok || st == nil { + log.Errorf("gRPC call returned not a gRPC status error: %v", err) + return nil, err + } else if st.Code() == codes.NotFound { + return nil, nil + } else if st.Code() == codes.Canceled { return nil, nil - }) - if err != nil { - log.Error(fmt.Errorf("failed to delete object from agent(%s): %w", daddr, err)) } + return nil, err } } + diff-- + c.correctedReplicationCount.Add(1) } + return nil, nil + } else if st.Code() == codes.Canceled { + return nil, nil } - return nil - })) + return nil, err + } + diff-- + c.correctedReplicationCount.Add(1) + return nil, nil + }) + if err != nil { + log.Error(fmt.Errorf("failed to insert object to agent(%s): %w", daddr, err)) } } } - }); err != nil { - // This only happens when ErrGRPCClientConnNotFound is returned. - // In other cases, OrderedRange continues processing, so error is used to keep track of the error status of correction. - errs = append(errs, err) - } - if len(errs) != 0 { - return errors.Join(errs...) } - return nil } - -func (c *correct) PreStop(_ context.Context) error { - log.Info("removing persistent cache files...") - return c.checkedList.Close(true) -} - -func (c *correct) NumberOfCheckedIndex() uint64 { - return c.checkedIndexCount.Load() -} - -func (c *correct) NumberOfCorrectedOldIndex() uint64 { - return c.correctedOldIndexCount.Load() -} - -func (c *correct) NumberOfCorrectedReplication() uint64 { - return c.correctedReplicationCount.Load() -} diff --git a/pkg/index/job/correction/usecase/corrector.go b/pkg/index/job/correction/usecase/corrector.go index 976d3ad961..a55326abdb 100644 --- a/pkg/index/job/correction/usecase/corrector.go +++ b/pkg/index/job/correction/usecase/corrector.go @@ -188,7 +188,7 @@ func (r *run) Start(ctx context.Context) (<-chan error, error) { // main groutine to run the job r.eg.Go(safety.RecoverFunc(func() (err error) { defer func() { - log.Info("fiding my pid to kill myself") + log.Info("finding my pid to kill myself") p, err := os.FindProcess(os.Getpid()) if err != nil { // using Fatal to avoid this process to be zombie diff --git a/pkg/manager/index/service/indexer.go b/pkg/manager/index/service/indexer.go index 42e90b1c9b..88400fe75f 100644 --- a/pkg/manager/index/service/indexer.go +++ b/pkg/manager/index/service/indexer.go @@ -24,10 +24,10 @@ import ( "sync/atomic" "time" - agent "github.com/vdaas/vald/apis/grpc/v1/agent/core" "github.com/vdaas/vald/apis/grpc/v1/payload" - vald "github.com/vdaas/vald/apis/grpc/v1/vald" + agent "github.com/vdaas/vald/internal/client/v1/client/agent/core" "github.com/vdaas/vald/internal/client/v1/client/discoverer" + vald "github.com/vdaas/vald/internal/client/v1/client/vald" "github.com/vdaas/vald/internal/errors" "github.com/vdaas/vald/internal/log" "github.com/vdaas/vald/internal/net/grpc" diff --git a/rust/Cargo.lock b/rust/Cargo.lock index c6b8a0217d..fd84b823b0 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -181,9 +181,9 @@ checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "cc" -version = "1.1.8" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549" +checksum = "e9e8aabfac534be767c909e0690571677d49f41bd8465ae876fe043d52ba5292" [[package]] name = "cfg-if" @@ -213,9 +213,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cxx" @@ -568,9 +568,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -678,9 +678,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ "hermit-abi", "libc", @@ -1105,18 +1105,18 @@ checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.207" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "5665e14a49a4ea1b91029ba7d3bca9f299e1f7cfa194388ccc20f14743e784f2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.207" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "6aea2634c86b0e8ef2cfdc0c340baede54ec27b1e46febd7f80dffb2aa44a00e" dependencies = [ "proc-macro2", "quote", @@ -1125,9 +1125,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.121" +version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609" +checksum = "66ad62847a56b3dba58cc891acd13884b9c61138d330c0d7b6181713d4fce38d" dependencies = [ "itoa", "memchr", @@ -1210,9 +1210,9 @@ checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" [[package]] name = "syn" -version = "2.0.72" +version = "2.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" dependencies = [ "proc-macro2", "quote", @@ -1543,19 +1543,20 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", @@ -1568,9 +1569,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -1580,9 +1581,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1590,9 +1591,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", @@ -1603,15 +1604,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/rust/bin/agent/src/handler/update.rs b/rust/bin/agent/src/handler/update.rs index c738a3dfd5..f8981b4f24 100644 --- a/rust/bin/agent/src/handler/update.rs +++ b/rust/bin/agent/src/handler/update.rs @@ -45,4 +45,12 @@ impl update_server::Update for super::Agent { ) -> std::result::Result, tonic::Status> { todo!() } + + #[doc = " A method to update timestamp indexed vectors in a single request.\n"] + async fn update_timestamp( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status> { + todo!() + } } diff --git a/rust/libs/proto/src/vald.v1.tonic.rs b/rust/libs/proto/src/vald.v1.tonic.rs index 1ff23e3b46..654bb5701b 100644 --- a/rust/libs/proto/src/vald.v1.tonic.rs +++ b/rust/libs/proto/src/vald.v1.tonic.rs @@ -5450,6 +5450,35 @@ pub mod update_client { .insert(GrpcMethod::new("vald.v1.Update", "MultiUpdate")); self.inner.unary(req, path, codec).await } + /** A method to update timestamp an indexed vector. +*/ + pub async fn update_timestamp( + &mut self, + request: impl tonic::IntoRequest< + super::super::super::payload::v1::object::Timestamp, + >, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/vald.v1.Update/UpdateTimestamp", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("vald.v1.Update", "UpdateTimestamp")); + self.inner.unary(req, path, codec).await + } } } /// Generated server implementations. @@ -5497,6 +5526,15 @@ pub mod update_server { tonic::Response, tonic::Status, >; + /** A method to update timestamp an indexed vector. +*/ + async fn update_timestamp( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } #[derive(Debug)] pub struct UpdateServer { @@ -5727,6 +5765,55 @@ pub mod update_server { }; Box::pin(fut) } + "/vald.v1.Update/UpdateTimestamp" => { + #[allow(non_camel_case_types)] + struct UpdateTimestampSvc(pub Arc); + impl< + T: Update, + > tonic::server::UnaryService< + super::super::super::payload::v1::object::Timestamp, + > for UpdateTimestampSvc { + type Response = super::super::super::payload::v1::object::Location; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request< + super::super::super::payload::v1::object::Timestamp, + >, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::update_timestamp(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let inner = inner.0; + let method = UpdateTimestampSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } _ => { Box::pin(async move { Ok( diff --git a/versions/PROMETHEUS_STACK_VERSION b/versions/PROMETHEUS_STACK_VERSION index 14e3460225..c40b5fb3e9 100644 --- a/versions/PROMETHEUS_STACK_VERSION +++ b/versions/PROMETHEUS_STACK_VERSION @@ -1 +1 @@ -61.7.1 +61.8.0 diff --git a/versions/VALDCLI_VERSION b/versions/VALDCLI_VERSION deleted file mode 100644 index b84efa430e..0000000000 --- a/versions/VALDCLI_VERSION +++ /dev/null @@ -1 +0,0 @@ -v1.7.12