Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] add values.go implementation #1770

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -776,7 +776,7 @@
- Update get started with using kubernetes ingress (#1510)
- Fix command template (#1508)
- Update dependencies version (#1501)
- modify .pb.go & swagger (#1493)
- modify .pb.go & openapi (#1493)
- Add reshape vector proto, remove meta/backup proto (#1492)

:pencil2: Documents
Expand Down
25 changes: 14 additions & 11 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -51,17 +51,17 @@ TEMP_DIR := $(eval TEMP_DIR := $(shell mktemp -d))$(TEMP_DIR)

OPERATOR_SDK_VERSION := $(eval OPERATOR_SDK_VERSION := $(shell cat versions/OPERATOR_SDK_VERSION))$(OPERATOR_SDK_VERSION)

KIND_VERSION ?= v0.15.0
HELM_VERSION ?= v3.9.4
GOLANGCILINT_VERSION ?= v1.49.0
HELM_DOCS_VERSION ?= 1.11.0
YQ_VERSION ?= v4.27.3
VALDCLI_VERSION ?= v1.5.6
TELEPRESENCE_VERSION ?= 2.7.2
HELM_VERSION ?= v3.9.4
JAEGER_OPERATOR_VERSION ?= 2.30.0
KIND_VERSION ?= v0.15.0
KUBELINTER_VERSION ?= 0.4.0
GOLANGCILINT_VERSION ?= v1.49.0
REVIEWDOG_VERSION ?= v0.14.1
PROTOBUF_VERSION ?= 21.5
JAEGER_OPERATOR_VERSION ?= 2.30.0
REVIEWDOG_VERSION ?= v0.14.1
TELEPRESENCE_VERSION ?= 2.7.2
VALDCLI_VERSION ?= v1.5.6
YQ_VERSION ?= v4.27.3

SWAP_DEPLOYMENT_TYPE ?= deployment
SWAP_IMAGE ?= ""
Expand Down Expand Up @@ -93,7 +93,8 @@ BENCH_DATASET_HDF5_DIR = $(BENCH_DATASET_BASE_DIR)/$(BENCH_DATASET_HDF5_DIR_NAME
PROTOS := $(eval PROTOS := $(shell find apis/proto -type f -regex ".*\.proto"))$(PROTOS)
PROTOS_V1 := $(eval PROTOS_V1 := $(filter apis/proto/v1/%.proto,$(PROTOS)))$(PROTOS_V1)
PBGOS = $(PROTOS:apis/proto/%.proto=apis/grpc/%.pb.go)
SWAGGERS = $(PROTOS:apis/proto/%.proto=apis/swagger/%.swagger.json)
OPENAPISPECS = $(PROTOS:apis/proto/%.proto=apis/openapi/%.openapi.json)
OPENAPIJSONSCHEMAS = $(PROTOS:apis/proto/%.proto=apis/jsonschema/%.schema.json)
PBDOCS = apis/docs/v1/docs.md

ifeq ($(GOARCH),amd64)
Expand Down Expand Up @@ -194,7 +195,7 @@ GO_SOURCES = $(eval GO_SOURCES := $(shell find \
-not -path './hack/benchmark/internal/starter/gateway/*' \
-not -path './hack/gorules/*' \
-not -path './hack/license/*' \
-not -path './hack/swagger/*' \
-not -path './hack/openapi/*' \
-not -path './hack/tools/*' \
-not -path './tests/*' \
-type f \
Expand Down Expand Up @@ -224,7 +225,7 @@ GO_OPTION_SOURCES = $(eval GO_OPTION_SOURCES := $(shell find \
-not -path './hack/benchmark/internal/starter/gateway/*' \
-not -path './hack/gorules/*' \
-not -path './hack/license/*' \
-not -path './hack/swagger/*' \
-not -path './hack/openapi/*' \
-not -path './hack/tools/*' \
-not -path './tests/*' \
-type f \
Expand Down Expand Up @@ -317,7 +318,9 @@ clean:
./*.log \
./*.svg \
./apis/docs \
./apis/openapi \
./apis/swagger \
./apis/jsonschema \
./apis/grpc \
./bench \
./pprof \
Expand Down
35 changes: 25 additions & 10 deletions Makefile.d/proto.mk
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,25 @@ proto/all: \
proto/deps \
pbgo \
pbdoc \
swagger
openapi \
jsonschema

.PHONY: pbgo
pbgo: $(PBGOS)

.PHONY: swagger
swagger: $(SWAGGERS)
.PHONY: openapi
openapi: $(OPENAPISPECS)

.PHONY: jsonschema
jsonschema: $(OPENAPIJSONSCHEMAS)

.PHONY: pbdoc
pbdoc: $(PBDOCS)

.PHONY: proto/clean
## clean proto artifacts
proto/clean:
rm -rf apis/grpc apis/swagger apis/docs
rm -rf apis/grpc apis/openapi apis/docs

.PHONY: proto/paths/print
## print proto paths
Expand All @@ -48,7 +52,8 @@ proto/deps: \
$(GOPATH)/bin/protoc-gen-go-grpc \
$(GOPATH)/bin/protoc-gen-go-vtproto \
$(GOPATH)/bin/protoc-gen-grpc-gateway \
$(GOPATH)/bin/protoc-gen-swagger \
$(GOPATH)/bin/protoc-gen-jsonschema \
$(GOPATH)/bin/protoc-gen-openapi \
$(GOPATH)/bin/protoc-gen-validate \
$(GOPATH)/bin/prototool \
$(GOPATH)/bin/swagger \
Expand Down Expand Up @@ -112,8 +117,11 @@ $(GOPATH)/bin/protoc-gen-go-grpc:
$(GOPATH)/bin/protoc-gen-grpc-gateway:
$(call go-install, github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway)

$(GOPATH)/bin/protoc-gen-swagger:
$(call go-install, github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger)
$(GOPATH)/bin/protoc-gen-openapi:
$(call go-install, github.com/google/gnostic/cmd/protoc-gen-openapi)

$(GOPATH)/bin/protoc-gen-jsonschema:
$(call go-install, github.com/google/gnostic/cmd/protoc-gen-jsonschema)

$(GOPATH)/bin/protoc-gen-validate:
$(call go-install, github.com/envoyproxy/protoc-gen-validate)
Expand All @@ -140,12 +148,19 @@ $(PBGOS): \
find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%google.golang.org/grpc/status%github.com/vdaas/vald/internal/net/grpc/status%g"
find $(ROOTDIR)/apis/grpc/* -name '*.go' | xargs sed -i -E "s%\"io\"%\"github.com/vdaas/vald/internal/io\"%g"

$(SWAGGERS): \
$(OPENAPISPECS): \
$(PROTOS) \
proto/deps
@$(call green, "generating openapi.json files...")
$(call mkdir, $(dir $@))
$(call protoc-gen, $(patsubst apis/openapi/%.openapi.json,apis/proto/%.proto,$@), --openapi_out=enum_type=string:$(dir $@))

$(OPENAPIJSONSCHEMAS): \
$(PROTOS) \
proto/deps
@$(call green, "generating swagger.json files...")
@$(call green, "generating schema.json files...")
$(call mkdir, $(dir $@))
$(call protoc-gen, $(patsubst apis/swagger/%.swagger.json,apis/proto/%.proto,$@), --swagger_out=json_names_for_fields=true:$(dir $@))
$(call protoc-gen, $(patsubst apis/jsonschema/%.schema.json,apis/proto/%.proto,$@), --jsonschema_out=$(dir $@))

$(PBDOCS): \
$(PROTOS) \
Expand Down
142 changes: 71 additions & 71 deletions apis/docs/v1/docs.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,12 @@
- [Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest)
- [Upsert.Request](#payload-v1-Upsert-Request)

- [apis/proto/v1/vald/filter.proto](#apis_proto_v1_vald_filter-proto)
- [Filter](#vald-v1-Filter)

- [apis/proto/v1/vald/insert.proto](#apis_proto_v1_vald_insert-proto)
- [Insert](#vald-v1-Insert)

- [apis/proto/v1/vald/object.proto](#apis_proto_v1_vald_object-proto)
- [Object](#vald-v1-Object)

Expand All @@ -107,12 +113,6 @@
- [apis/proto/v1/vald/upsert.proto](#apis_proto_v1_vald_upsert-proto)
- [Upsert](#vald-v1-Upsert)

- [apis/proto/v1/vald/filter.proto](#apis_proto_v1_vald_filter-proto)
- [Filter](#vald-v1-Filter)

- [apis/proto/v1/vald/insert.proto](#apis_proto_v1_vald_insert-proto)
- [Insert](#vald-v1-Insert)

- [Scalar Value Types](#scalar-value-types)


Expand Down Expand Up @@ -1326,6 +1326,71 @@ Represent the upsert request.



<a name="apis_proto_v1_vald_filter-proto"></a>
<p align="right"><a href="#top">Top</a></p>

## apis/proto/v1/vald/filter.proto









<a name="vald-v1-Filter"></a>

### Filter
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[LanguageTool] reported by reviewdog 🐶
Possible typo: you repeated a word (ENGLISH_WORD_REPEAT_RULE)
Suggestions: Filter
Rule: https://community.languagetool.org/rule/show/ENGLISH_WORD_REPEAT_RULE?lang=en-US
Category: MISC

Filter service provides ways to connect to Vald through filter.

| Method Name | Request Type | Response Type | Description |
| ----------- | ------------ | ------------- | ------------|
| SearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) | [.payload.v1.Search.Response](#payload-v1-Search-Response) | A method to search object. |
| MultiSearchObject | [.payload.v1.Search.MultiObjectRequest](#payload-v1-Search-MultiObjectRequest) | [.payload.v1.Search.Responses](#payload-v1-Search-Responses) | A method to search multiple objects. |
| StreamSearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) stream | [.payload.v1.Search.StreamResponse](#payload-v1-Search-StreamResponse) stream | A method to search object by bidirectional streaming. |
| InsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method insert object. |
| StreamInsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | Represent the streaming RPC to insert object by bidirectional streaming. |
| MultiInsertObject | [.payload.v1.Insert.MultiObjectRequest](#payload-v1-Insert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert multiple objects. |
| UpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update object. |
| StreamUpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update object by bidirectional streaming. |
| MultiUpdateObject | [.payload.v1.Update.MultiObjectRequest](#payload-v1-Update-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple objects. |
| UpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to upsert object. |
| StreamUpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to upsert object by bidirectional streaming. |
| MultiUpsertObject | [.payload.v1.Upsert.MultiObjectRequest](#payload-v1-Upsert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to upsert multiple objects. |





<a name="apis_proto_v1_vald_insert-proto"></a>
<p align="right"><a href="#top">Top</a></p>

## apis/proto/v1/vald/insert.proto









<a name="vald-v1-Insert"></a>

### Insert
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[LanguageTool] reported by reviewdog 🐶
Possible typo: you repeated a word (ENGLISH_WORD_REPEAT_RULE)
Suggestions: Insert
Rule: https://community.languagetool.org/rule/show/ENGLISH_WORD_REPEAT_RULE?lang=en-US
Category: MISC

Insert service provides ways to add new vectors.

| Method Name | Request Type | Response Type | Description |
| ----------- | ------------ | ------------- | ------------|
| Insert | [.payload.v1.Insert.Request](#payload-v1-Insert-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to add a new single vector. |
| StreamInsert | [.payload.v1.Insert.Request](#payload-v1-Insert-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to add new multiple vectors by bidirectional streaming. |
| MultiInsert | [.payload.v1.Insert.MultiRequest](#payload-v1-Insert-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to add new multiple vectors in a single request. |





<a name="apis_proto_v1_vald_object-proto"></a>
<p align="right"><a href="#top">Top</a></p>

Expand Down Expand Up @@ -1475,71 +1540,6 @@ Upsert service provides ways to insert/update vectors.



<a name="apis_proto_v1_vald_filter-proto"></a>
<p align="right"><a href="#top">Top</a></p>

## apis/proto/v1/vald/filter.proto









<a name="vald-v1-Filter"></a>

### Filter
Filter service provides ways to connect to Vald through filter.

| Method Name | Request Type | Response Type | Description |
| ----------- | ------------ | ------------- | ------------|
| SearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) | [.payload.v1.Search.Response](#payload-v1-Search-Response) | A method to search object. |
| MultiSearchObject | [.payload.v1.Search.MultiObjectRequest](#payload-v1-Search-MultiObjectRequest) | [.payload.v1.Search.Responses](#payload-v1-Search-Responses) | A method to search multiple objects. |
| StreamSearchObject | [.payload.v1.Search.ObjectRequest](#payload-v1-Search-ObjectRequest) stream | [.payload.v1.Search.StreamResponse](#payload-v1-Search-StreamResponse) stream | A method to search object by bidirectional streaming. |
| InsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method insert object. |
| StreamInsertObject | [.payload.v1.Insert.ObjectRequest](#payload-v1-Insert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | Represent the streaming RPC to insert object by bidirectional streaming. |
| MultiInsertObject | [.payload.v1.Insert.MultiObjectRequest](#payload-v1-Insert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to insert multiple objects. |
| UpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to update object. |
| StreamUpdateObject | [.payload.v1.Update.ObjectRequest](#payload-v1-Update-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to update object by bidirectional streaming. |
| MultiUpdateObject | [.payload.v1.Update.MultiObjectRequest](#payload-v1-Update-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to update multiple objects. |
| UpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to upsert object. |
| StreamUpsertObject | [.payload.v1.Upsert.ObjectRequest](#payload-v1-Upsert-ObjectRequest) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to upsert object by bidirectional streaming. |
| MultiUpsertObject | [.payload.v1.Upsert.MultiObjectRequest](#payload-v1-Upsert-MultiObjectRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to upsert multiple objects. |





<a name="apis_proto_v1_vald_insert-proto"></a>
<p align="right"><a href="#top">Top</a></p>

## apis/proto/v1/vald/insert.proto









<a name="vald-v1-Insert"></a>

### Insert
Insert service provides ways to add new vectors.

| Method Name | Request Type | Response Type | Description |
| ----------- | ------------ | ------------- | ------------|
| Insert | [.payload.v1.Insert.Request](#payload-v1-Insert-Request) | [.payload.v1.Object.Location](#payload-v1-Object-Location) | A method to add a new single vector. |
| StreamInsert | [.payload.v1.Insert.Request](#payload-v1-Insert-Request) stream | [.payload.v1.Object.StreamLocation](#payload-v1-Object-StreamLocation) stream | A method to add new multiple vectors by bidirectional streaming. |
| MultiInsert | [.payload.v1.Insert.MultiRequest](#payload-v1-Insert-MultiRequest) | [.payload.v1.Object.Locations](#payload-v1-Object-Locations) | A method to add new multiple vectors in a single request. |





## Scalar Value Types

| .proto Type | Notes | C++ | Java | Python | Go | C# | PHP | Ruby |
Expand Down
24 changes: 24 additions & 0 deletions apis/jsonschema/v1/payload/Control.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
{
"title": "Control",
"$id": "Control.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"description": "Control related messages.",
"properties": {
},
"definitions": {
"Control_CreateIndexRequest": {
"title": "CreateIndexRequest",
"type": "object",
"description": "Represent the create index request.",
"properties": {
"poolSize": {
"title": "poolSize",
"type": "integer",
"description": "The pool size of the create index operation.",
"format": "uint32"
}
}
}
}
}
33 changes: 33 additions & 0 deletions apis/jsonschema/v1/payload/Discoverer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
{
"title": "Discoverer",
"$id": "Discoverer.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"description": "Discoverer related messages.",
"properties": {
},
"definitions": {
"Discoverer_Request": {
"title": "Request",
"type": "object",
"description": "Represent the dicoverer request.",
"properties": {
"name": {
"title": "name",
"type": "string",
"description": "The agent name to be discover."
},
"namespace": {
"title": "namespace",
"type": "string",
"description": "The namespace to be discover."
},
"node": {
"title": "node",
"type": "string",
"description": "The node to be discover."
}
}
}
}
}
9 changes: 9 additions & 0 deletions apis/jsonschema/v1/payload/Empty.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"title": "Empty",
"$id": "Empty.json",
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"description": "Represent an empty message.",
"properties": {
}
}
Loading