Skip to content

Commit

Permalink
Merge remote-tracking branch 'github-bk-bcs/master'
Browse files Browse the repository at this point in the history
* github-bk-bcs/master:
  feat: bscp 支持 KV 类型 --story=115444201 (#2806)
  • Loading branch information
wenxinlee2015 committed Dec 8, 2023
2 parents 7842c8d + 11c3382 commit 2d437cb
Show file tree
Hide file tree
Showing 66 changed files with 4,436 additions and 439 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@
|---------------------| ------------ | ------ | ---------------- |
| biz_id | uint32 || 业务ID |
| name | string || 应用名称。最大长度128个字符,仅允许使用中文、英文、数字、下划线、中划线,且必须以中文、英文、数字开头和结尾 |
| config_type | string || 应用配置类型(枚举值:file,目前仅支持file类型)。应用配置类型限制了本应用下所有配置的类型,如果选择file类型,本应用下所有配置只能为file类型 |
| mode | string || app的实例在消费配置时的模式(枚举值:normal、namespace),详情见下方描述。 |
| memo | string || 备注。最大长度256个字符,仅允许使用中文、英文、数字、下划线、中划线、空格,且必须以中文、英文、数字开头和结尾 |
| config_type | string || 应用配置类型(枚举值:file、kv,目前仅支持file、kv类型)。应用配置类型限制了本应用下所有配置的类型,如果选择file类型,本应用下所有配置只能为file类型 |
| mode | string || app的实例在消费配置时的模式(枚举值:normal、namespace),详情见下方描述。 |
| memo | string || 备注。最大长度256个字符,仅允许使用中文、英文、数字、下划线、中划线、空格,且必须以中文、英文、数字开头和结尾 |
| reload_type | string | 选填,仅在 config_type 为 file 类型下使用 | sidecar通知app重新Reload配置的方式(枚举值:file,目前仅支持file类型) |
| reload_file_path | string | 选填,仅在 reload_type 为 file 类型下使用 | Reload文件绝对路径(绝对路径 + 文件名),最大长度为128字节 |
| reload_file_path | string | 选填,仅在 reload_type 为 file 类型下使用 | Reload文件绝对路径(绝对路径 + 文件名),最大长度为128字节 |
| alias | string | 必填 | 服务别名。最大长度128个字符,仅允许使用中文、英文、数字、下划线、中划线,且必须以中文、英文、数字开头和结尾 |
| data_type | string | 选填,仅在config_type为kv 类型下生效 | 数据类型(枚举值:any、string、number、string、text、yaml、json、xml) |

#### 参数说明:

Expand Down Expand Up @@ -45,7 +47,7 @@ app工作的工作模式决定了该app下的实例消费配置数据的方式

bscp sidecar 会将下载好的配置信息,写到用户指定的 reload 文件(reload_file_path)当中,应用程序通过这个 reload 文件来获取最新的配置信息。

### 调用示例
### 调用示例(config_item)

```json
{
Expand All @@ -54,7 +56,22 @@ bscp sidecar 会将下载好的配置信息,写到用户指定的 reload 文
"mode": "normal",
"memo": "my_first_app",
"reload_type": "file",
"reload_file_path": "/data/reload.json"
"reload_file_path": "/data/reload.json",
"alias":"appAlias"
}
```

### 调用示例(kv)

```json
{
"biz_id": "myapp",
"name": "kv",
"config_type": "kv",
"mode": "normal",
"memo": "",
"data_type":"any",
"alias":"appAlias"
}
```

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
### 描述
该接口提供版本:v1.0.0+


更新应用。

Expand All @@ -10,9 +10,11 @@
| id | uint32 || 应用ID |
| biz_id | uint32 || 业务ID |
| name | string || 应用名称。最大长度128个字符,仅允许使用中文、英文、数字、下划线、中划线,且必须以中文、英文、数字开头和结尾 |
| memo | string || 备注。最大长度256个字符,仅允许使用中文、英文、数字、下划线、中划线、空格,且必须以中文、英文、数字开头和结尾 |
| memo | string || 备注。最大长度256个字符,仅允许使用中文、英文、数字、下划线、中划线、空格,且必须以中文、英文、数字开头和结尾 |
| reload_type | string | 选填,仅在 config_type 为 file 类型下使用 | sidecar通知app重新Reload配置的方式(枚举值:file,目前仅支持file类型) |
| reload_file_path | string | 选填,仅在 reload_type 为 file 类型下使用 | Reload文件绝对路径(绝对路径 + 文件名),最大长度为128字节 |
| alias | string | 必填 | 服务别名。最大长度128个字符,仅允许使用中文、英文、数字、下划线、中划线,且必须以中文、英文、数字开头和结尾 |
| data_type | string | 选填,仅在config_type为kv 类型下生效 | 数据类型(枚举值:any、string、number、string、text、yaml、json、xml) |

#### 参数说明:
##### reload_type:
Expand All @@ -22,15 +24,30 @@
###### File类型:
bscp sidecar 会将下载好的配置信息,写到用户指定的 reload 文件(reload_file_path)当中,应用程序通过这个 reload 文件来获取最新的配置信息。

### 调用示例


### 调用示例(config_items)
```json
{
"name": "update_app",
"memo": "my_update_app"
"memo": "my_update_app",
"alias":"appAlias"
}
```

### 调用示例(kv)

```json
{
"name": "update_app",
"memo": "my_update_app",
"config_type":"any",
"alias":"appAlias"
}
```

### 响应示例

```json
{
"code": 0,
Expand Down
8 changes: 6 additions & 2 deletions bcs-services/bcs-bscp/cmd/config-server/service/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ func (s *Service) CreateApp(ctx context.Context, req *pbcs.CreateAppReq) (*pbcs.
ConfigType: req.ConfigType,
Mode: req.Mode,
Memo: req.Memo,
Alias: req.Alias,
DataType: req.DataType,
Reload: &pbapp.Reload{
ReloadType: req.ReloadType,
FileReloadSpec: &pbapp.FileReloadSpec{
Expand Down Expand Up @@ -104,8 +106,10 @@ func (s *Service) UpdateApp(ctx context.Context, req *pbcs.UpdateAppReq) (*pbcs.
Id: req.Id,
BizId: req.BizId,
Spec: &pbapp.AppSpec{
Name: req.Name,
Memo: req.Memo,
Name: req.Name,
Memo: req.Memo,
Alias: req.Alias,
DataType: req.DataType,
Reload: &pbapp.Reload{
ReloadType: req.ReloadType,
FileReloadSpec: &pbapp.FileReloadSpec{
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Tencent is pleased to support the open source community by making Blueking Container Service available.
* Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
* Licensed under the MIT License (the "License"); you may not use this file except
* in compliance with the License. You may obtain a copy of the License at
* http://opensource.org/licenses/MIT
* Unless required by applicable law or agreed to in writing, software distributed under
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
* either express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/

package migrations

import (
"fmt"

"gorm.io/gorm"

"bscp.io/cmd/data-service/db-migration/migrator"
"bscp.io/pkg/dal/table"
)

func init() {
// add current migration to migrator
migrator.GetMigrator().AddMigration(&migrator.Migration{
Version: "20231123143015",
Name: "20231123143015_modify_app",
Mode: migrator.GormMode,
Up: mig20231123143015Up,
Down: mig20231123143015Down,
})
}

// mig20231123143015Up for up migration
func mig20231123143015Up(tx *gorm.DB) error {

// Applications: 服务
type Applications struct {
Alias string `gorm:"type:varchar(255) not null;"`
DataType string `gorm:"type:varchar(255) not null;"`
}

if err := tx.Set("gorm:table_options", "ENGINE=InnoDB CHARSET=utf8mb4").AutoMigrate(
&Applications{},
); err != nil {
return err
}

// set default value
var apps []table.App
tx.Model(&table.App{}).Find(&apps)
for _, app := range apps {
if app.Spec.Alias == "" {
app.Spec.Alias = fmt.Sprintf("%s_alias", app.Spec.Name)
}
if app.Spec.ConfigType == table.KV {
// 原有的kv类型app,数据类型全部记为any
app.Spec.DataType = table.KvAny
}
tx.Save(&apps)
}

return nil

}

// mig20231123143015Down for down migration
func mig20231123143015Down(tx *gorm.DB) error {

// Applications: 服务
type Applications struct {
Alias string `gorm:"type:varchar(255) not null;"`
DataType string `gorm:"type:varchar(255) not null;"`
}

// delete old column
if tx.Migrator().HasColumn(&Applications{}, "alias") {
if err := tx.Migrator().DropColumn(&Applications{}, "alias"); err != nil {
return err
}
}
if tx.Migrator().HasColumn(&Applications{}, "data_type") {
if err := tx.Migrator().DropColumn(&Applications{}, "data_type"); err != nil {
return err
}
}

return nil
}
59 changes: 58 additions & 1 deletion bcs-services/bcs-bscp/cmd/data-service/service/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ func (s *Service) CreateApp(ctx context.Context, req *pbds.CreateAppReq) (*pbds.
return nil, fmt.Errorf("app name %s already exists", req.Spec.Name)
}

if _, err := s.dao.App().GetByAlias(kt, req.BizId, req.Spec.Alias); err == nil {
return nil, fmt.Errorf("app alias %s already exists", req.Spec.Alias)
}

app := &table.App{
BizID: req.BizId,
Spec: req.Spec.AppSpec(),
Expand All @@ -77,7 +81,27 @@ func (s *Service) CreateApp(ctx context.Context, req *pbds.CreateAppReq) (*pbds.
func (s *Service) UpdateApp(ctx context.Context, req *pbds.UpdateAppReq) (*pbbase.EmptyResp, error) {
grpcKit := kit.FromGrpcContext(ctx)

app := &table.App{
old, err := s.dao.App().GetByAlias(grpcKit, req.BizId, req.Spec.Alias)
if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
logs.Errorf("get app failed, err: %v, rid: %s", err, grpcKit.Rid)
return nil, err
}
if !errors.Is(gorm.ErrRecordNotFound, err) && old.ID != req.Id {
return nil, fmt.Errorf("app alias %s already exists", req.Spec.Alias)
}

app, err := s.dao.App().Get(grpcKit, req.BizId, req.Id)
if err != nil {
logs.Errorf("get app failed, err: %v, rid: %s", err, grpcKit.Rid)
return nil, err
}
if app.Spec.ConfigType == table.KV {
if err := s.checkUpdateAppDataType(grpcKit, req, app); err != nil {
return nil, err
}
}

app = &table.App{
ID: req.Id,
BizID: req.BizId,
Spec: req.Spec.AppSpec(),
Expand All @@ -93,6 +117,39 @@ func (s *Service) UpdateApp(ctx context.Context, req *pbds.UpdateAppReq) (*pbbas
return new(pbbase.EmptyResp), nil
}

func (s *Service) checkUpdateAppDataType(kt *kit.Kit, req *pbds.UpdateAppReq, app *table.App) error {

if app.Spec.DataType == table.DataType(req.Spec.DataType) {
return nil
}

if req.Spec.DataType == string(table.KvAny) {
return nil
}

// 获取所有的kv
kvList, err := s.dao.Kv().ListAllByAppID(kt, app.ID, req.BizId)
if err != nil {
return err
}
if len(kvList) == 0 {
return nil
}

for _, kv := range kvList {
kvType, _, err := s.getKv(kt, req.BizId, kv.Attachment.AppID, kv.Spec.Version, kv.Spec.Key)
if err != nil {
return err
}

if string(kvType) != req.Spec.DataType {
return fmt.Errorf("the specified type does not match the actual configuration")
}
}

return nil
}

// DeleteApp delete application.
func (s *Service) DeleteApp(ctx context.Context, req *pbds.DeleteAppReq) (*pbbase.EmptyResp, error) {
grpcKit := kit.FromGrpcContext(ctx)
Expand Down
29 changes: 26 additions & 3 deletions bcs-services/bcs-bscp/cmd/data-service/service/kv.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,20 @@ func (s *Service) CreateKv(ctx context.Context, req *pbds.CreateKvReq) (*pbds.Cr
return nil, fmt.Errorf("kv same key %s already exists", req.Spec.Key)
}

app, err := s.dao.App().Get(kt, req.Attachment.BizId, req.Attachment.AppId)
if err != nil {
return nil, fmt.Errorf("get app fail,err : %v", req.Spec.Key)
}
if !checkKVTypeMatch(table.DataType(req.Spec.KvType), app.Spec.DataType) {
return nil, fmt.Errorf("kv type does not match the data type defined in the application")
}

opt := &types.UpsertKvOption{
BizID: req.Attachment.BizId,
AppID: req.Attachment.AppId,
Key: req.Spec.Key,
Value: req.Spec.Value,
KvType: types.KvType(req.Spec.KvType),
KvType: table.DataType(req.Spec.KvType),
}
version, err := s.vault.UpsertKv(kt, opt)
if err != nil {
Expand Down Expand Up @@ -69,11 +77,26 @@ func (s *Service) CreateKv(ctx context.Context, req *pbds.CreateKvReq) (*pbds.Cr

}

func checkKVTypeMatch(kvType, appKvType table.DataType) bool {
if appKvType == table.KvAny {
return true
}
return kvType == appKvType
}

// UpdateKv is used to update key-value data.
func (s *Service) UpdateKv(ctx context.Context, req *pbds.UpdateKvReq) (*pbbase.EmptyResp, error) {

kt := kit.FromGrpcContext(ctx)

app, err := s.dao.App().Get(kt, req.Attachment.BizId, req.Attachment.AppId)
if err != nil {
return nil, fmt.Errorf("get app fail,err : %v", req.Spec.Key)
}
if checkKVTypeMatch(table.DataType(req.Spec.KvType), app.Spec.DataType) {
return nil, fmt.Errorf("kv type does not match the data type defined in the application")
}

kv, err := s.dao.Kv().GetByKey(kt, req.Attachment.BizId, req.Attachment.AppId, req.Spec.Key)
if err != nil {
logs.Errorf("get kv (%d) failed, err: %v, rid: %s", req.Spec.Key, err, kt.Rid)
Expand Down Expand Up @@ -262,7 +285,7 @@ func (s *Service) BatchUpsertKvs(ctx context.Context, req *pbds.BatchUpsertKvsRe
return new(pbbase.EmptyResp), nil
}

func (s *Service) getKv(kt *kit.Kit, bizID, appID, version uint32, key string) (types.KvType, string, error) {
func (s *Service) getKv(kt *kit.Kit, bizID, appID, version uint32, key string) (table.DataType, string, error) {
opt := &types.GetKvByVersion{
BizID: bizID,
AppID: appID,
Expand Down Expand Up @@ -295,7 +318,7 @@ func (s *Service) doBatchUpsertVault(kt *kit.Kit, req *pbds.BatchUpsertKvsReq,
}
opt.KvType = kvType
} else {
opt.KvType = types.KvType(kv.KvSpec.KvType)
opt.KvType = table.DataType(kv.KvSpec.KvType)
}

version, err := s.vault.UpsertKv(kt, opt)
Expand Down
2 changes: 1 addition & 1 deletion bcs-services/bcs-bscp/cmd/data-service/service/release.go
Original file line number Diff line number Diff line change
Expand Up @@ -785,7 +785,7 @@ func (s *Service) genCreateReleasedKvMap(kt *kit.Kit, bizID, appID,
kvsMap := make(map[string]*types.CreateReleasedKvOption, len(kvs))
for _, kv := range kvs {

var kvType types.KvType
var kvType table.DataType
var value string

kvType, value, err = s.getKv(kt, bizID, appID, kv.Spec.Version, kv.Spec.Key)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package service
import (
"context"

"bscp.io/pkg/dal/table"
"bscp.io/pkg/kit"
"bscp.io/pkg/logs"
pbbase "bscp.io/pkg/protocol/core/base"
Expand Down Expand Up @@ -105,7 +106,7 @@ func (s *Service) ListReleasedKvs(ctx context.Context, req *pbds.ListReleasedKvR
}

func (s *Service) getReleasedKv(kt *kit.Kit, bizID, appID, version, releasedID uint32,
key string) (types.KvType, string, error) {
key string) (table.DataType, string, error) {

opt := &types.GetRKvOption{
BizID: bizID,
Expand Down
Loading

0 comments on commit 2d437cb

Please sign in to comment.