Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(metric): implement new pipeline dashboard endpoints #238

Draft
wants to merge 2 commits into
base: main
Choose a base branch
from
Draft

feat(metric): implement new pipeline dashboard endpoints #238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8444799
Revert "fix: restore pipeline dashboard enpoints (#230)"
jvallesm Aug 22, 2024
80d9d72
chore: reference proto contracts
jvallesm Aug 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ require (
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1
github.com/iancoleman/strcase v0.2.0
github.com/influxdata/influxdb-client-go/v2 v2.12.3
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240729070104-1b23ffa897eb
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240726044326-e2dcf1c317f4 // TODO reference commit once merged
github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a
github.com/instill-ai/x v0.4.0-alpha
github.com/knadh/koanf v1.5.0
Expand Down
6 changes: 4 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1090,8 +1090,10 @@ github.com/influxdata/influxdb-client-go/v2 v2.12.3 h1:28nRlNMRIV4QbtIUvxhWqaxn0
github.com/influxdata/influxdb-client-go/v2 v2.12.3/go.mod h1:IrrLUbCjjfkmRuaCiGQg4m2GbkaeJDcuWoxiWdQEbA0=
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240729070104-1b23ffa897eb h1:3NyTWRr4HDB0Hz83Bd66g7s0qCytHOA7oGL6TVe6UsA=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240729070104-1b23ffa897eb/go.mod h1:2blmpUwiTwxIDnrjIqT6FhR5ewshZZF554wzjXFvKpQ=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240725140016-18f3ff64c952 h1:gtc9vYwc5sEWIs5H1bdu0vx5IgZb0CKcvBa0VWtU8VE=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240725140016-18f3ff64c952/go.mod h1:2blmpUwiTwxIDnrjIqT6FhR5ewshZZF554wzjXFvKpQ=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240726044326-e2dcf1c317f4 h1:LTnrCSgLcgYLQR43NGrAswAQ+XroDAbLEvCMTK//AxU=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240726044326-e2dcf1c317f4/go.mod h1:2blmpUwiTwxIDnrjIqT6FhR5ewshZZF554wzjXFvKpQ=
github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a h1:gmy8BcCFDZQan40c/D3f62DwTYtlCwi0VrSax+pKffw=
github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a/go.mod h1:EpX3Yr661uWULtZf5UnJHfr5rw2PDyX8ku4Kx0UtYFw=
github.com/instill-ai/x v0.4.0-alpha h1:zQV2VLbSHjMv6gyBN/2mwwrvWk0/mJM6ZKS12AzjfQg=
Expand Down
87 changes: 15 additions & 72 deletions integration-test/grpc-public-user.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,84 +239,27 @@ export function CheckPublicGetRemainingCredit(header) {
}

export function CheckPublicMetrics(header) {
group(`Management Public API: List Pipeline Trigger Table Records`, () => {

let pipeline_id = randomString(10)
let pipeline_id = randomString(10)

client.connect(constant.mgmtPublicGRPCHost, {
plaintext: true
});


group(`Management Public API: List Pipeline Trigger Table Records`, () => {

let emptyPipelineTriggerTableRecordResponse = {
"pipelineTriggerTableRecords": [],
"nextPageToken": "",
"totalSize": 0
}

check(
http.request(
"GET",
`${constant.mgmtPublicHost}/metrics/vdp/pipeline/tables`,
null,
header,
),
{
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables response status is 200`]:
(r) => r.status === 200,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables response has pipelineTriggerTableRecords`]:
(r) => r.json().pipelineTriggerTableRecords !== undefined,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables response has nextPageToken`]:
(r) => r.json().totalSize !== undefined,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables response has totalSize`]:
(r) => r.json().nextPageToken !== undefined,
}
)
check(
http.request(
"GET",
`${constant.mgmtPublicHost}/metrics/vdp/pipeline/tables?filter=pipelineId=%22${pipeline_id}%22`,
null,
header,
),
{
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables with filter response status is 200`]:
(r) => r.status === 200,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables with filter response pipelineTriggerTableRecords length is 0`]:
(r) => r.json().pipelineTriggerTableRecords.length === 0,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables with filter response nextPageToken is empty`]:
(r) => r.json().nextPageToken === emptyPipelineTriggerTableRecordResponse.nextPageToken,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables with filter response totalSize is 0`]:
(r) => r.json().totalSize === emptyPipelineTriggerTableRecordResponse.totalSize,
}
)
})
group(`Management Public API: List Pipeline Trigger Chart Records`, () => {

let pipeline_id = randomString(10)

check(
http.request(
"GET",
`${constant.mgmtPublicHost}/metrics/vdp/pipeline/charts`,
null,
header,
),
{
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/charts response status is 200`]:
(r) => r.status === 200,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/charts response has pipelineTriggerRecords`]:
(r) => r.json().pipelineTriggerChartRecords !== undefined,
}
)
check(
http.request(
"GET",
`${constant.mgmtPublicHost}/metrics/vdp/pipeline/charts?filter=triggerMode=MODE_SYNC%20AND%20pipelineId=%22${pipeline_id}%22`,
null,
header,
),
{
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/charts with filter response status is 200`]:
(r) => r.status === 200,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/charts with filter response pipelineTriggerRecords length is 0`]:
(r) => r.json().pipelineTriggerChartRecords.length === 0,
}
)
})


});



client.close();
}
89 changes: 0 additions & 89 deletions integration-test/proto/core/mgmt/v1beta/metric.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@ message PipelineTriggerCount {
optional Status status = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
}

/*
// PipelineTriggerChartRecord represents a timeline of pipeline triggers. It
// contains a collection of (timestamp, count) pairs that represent the total
// pipeline triggers in a given time bucket.
Expand Down Expand Up @@ -69,7 +68,6 @@ message PipelineTriggerChartRecord {
// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
}
*/

// GetPipelineTriggerCountRequest represents a request to fetch the trigger
// count of a requester over a time period.
Expand All @@ -91,7 +89,6 @@ message GetPipelineTriggerCountResponse {
repeated PipelineTriggerCount pipeline_trigger_counts = 1;
}

/*
// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger chart records for a given requester, grouped by time buckets.
message ListPipelineTriggerChartRecordsRequest {
Expand Down Expand Up @@ -128,7 +125,6 @@ message ListPipelineTriggerChartRecordsResponse {
// ID, trigger mode, final status or other fields.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}
*/

// CreditConsumptionChartRecord represents a timeline of Instill Credit
// consumption. It contains a collection of (timestamp, amount) pairs that
Expand Down Expand Up @@ -173,88 +169,3 @@ message ListCreditConsumptionChartRecordsResponse {
// consumption by source.
reserved 2;
}

// Deprecated messages, to be removed with the new dashboard implementation.

// PipelineTriggerTableRecord contains pipeline trigger metrics, aggregated by
// pipeline ID.
message PipelineTriggerTableRecord {
// Pipeline ID.
string pipeline_id = 1;
// Pipeline UUID.
string pipeline_uid = 2;
// Number of triggers with `STATUS_COMPLETED`.
int32 trigger_count_completed = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Number of triggers with `STATUS_ERRORED`.
int32 trigger_count_errored = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
// Version for the triggered pipeline if it is a release pipeline.
string pipeline_release_id = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Release UUID for the triggered pipeline if it is a release pipeline.
string pipeline_release_uid = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// ListPipelineTriggerTableRecordsRequest represents a request to list the
// pipeline triggers metrics, aggregated by pipeline ID.
message ListPipelineTriggerTableRecordsRequest {
// The maximum number of results to return. If this parameter is unspecified,
// at most 100 pipelines will be returned. The cap value for this parameter
// is 1000 (i.e. any value above that will be coerced to 1000).
optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL];
// Page token.
optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL];
// Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter
// expression.
// - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`.
optional string filter = 3 [(google.api.field_behavior) = OPTIONAL];
}

// ListPipelineTriggerTableRecordsResponse contains the pipeline metrics.
message ListPipelineTriggerTableRecordsResponse {
// A list of pipeline trigger tables.
repeated PipelineTriggerTableRecord pipeline_trigger_table_records = 1;
// Next page token.
string next_page_token = 2;
// Total number of pipeline trigger records
int32 total_size = 3;
}

// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger metrics, aggregated by pipeline ID and time frame.
message ListPipelineTriggerChartRecordsRequest {
// Aggregation window in nanoseconds.
int32 aggregation_window = 1;
// Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter
// expression.
// - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`.
optional string filter = 2 [(google.api.field_behavior) = OPTIONAL];
}

// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// A list of pipeline trigger records.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}

// PipelineTriggerChartRecord contains pipeline trigger metrics, aggregated by
// pipeline ID and time frame.
message PipelineTriggerChartRecord {
// Pipeline ID.
string pipeline_id = 1;
// Pipeline UUID.
string pipeline_uid = 2;
// Trigger mode.
Mode trigger_mode = 3;
// Final status.
Status status = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
// Time buckets.
repeated google.protobuf.Timestamp time_buckets = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregated trigger count in each time bucket.
repeated int32 trigger_counts = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// Total computation time duration in each time bucket.
repeated float compute_time_duration = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
// Version for the triggered pipeline if it is a release pipeline.
string pipeline_release_id = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
// Release UUID for the triggered pipeline if it is a release pipeline.
string pipeline_release_uid = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
}
21 changes: 3 additions & 18 deletions integration-test/proto/core/mgmt/v1beta/mgmt_public_service.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -290,26 +290,11 @@ service MgmtPublicService {
rpc GetPipelineTriggerCount(GetPipelineTriggerCountRequest) returns (GetPipelineTriggerCountResponse) {
option (google.api.http) = {get: "/v1beta/metrics/vdp/pipeline/trigger-count"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Metric"};
// This endpoint will remain hidden until the new dashboard is implemented
// in the frontend. Until then, the server might return empty data.
option (google.api.method_visibility).restriction = "INTERNAL";
}

// List pipeline trigger metrics
//
// Returns a paginated list of pipeline executions aggregated by pipeline ID.
// NOTE: This method is deprecated and will be retired soon.
rpc ListPipelineTriggerTableRecords(ListPipelineTriggerTableRecordsRequest) returns (ListPipelineTriggerTableRecordsResponse) {
option (google.api.http) = {get: "/v1beta/metrics/vdp/pipeline/tables"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Metric"};
option deprecated = true;
}

// List pipeline trigger time charts
//
// Returns a timeline of pipline trigger counts for the pipelines of a given
// owner.
// NOTE: This method will soon return the trigger counts of a given requester.
// Returns a timeline of pipline trigger counts for a given requester. The
// response will contain one set of records (datapoints), representing the
// amount of triggers in a time bucket.
rpc ListPipelineTriggerChartRecords(ListPipelineTriggerChartRecordsRequest) returns (ListPipelineTriggerChartRecordsResponse) {
option (google.api.http) = {get: "/v1beta/metrics/vdp/pipeline/charts"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {tags: "Metric"};
Expand Down
Loading
Loading