Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(mgmt): update metrics endpoints to serve new dashboard design #513

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
+260 −244
Open

feat(mgmt): update metrics endpoints to serve new dashboard design #513

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a2a24c7
feat(mgmt): update metrics endpoints to serve new dashboard design
joremysh Nov 6, 2024
3076ab5
fix: resolve comments
joremysh Nov 6, 2024
eca4556
chore: auto-gen by protobufs
droplet-bot Nov 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 57 additions & 146 deletions core/mgmt/v1beta/metric.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,39 +38,6 @@ message TriggerCount {
optional Status status = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
}

/*
// PipelineTriggerChartRecord represents a timeline of pipeline triggers. It
// contains a collection of (timestamp, count) pairs that represent the total
// pipeline triggers in a given time bucket.
// pipeline ID and time frame.
message PipelineTriggerChartRecord {
// This field will be present present when the information is grouped by pipeline.
optional string pipeline_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// 2 is reserved for the pipeline UUID.
reserved 2;
// 3 is reserved for the trigger mode. The server wasn't grouping results by this
// field.
reserved 3;
// 4 is reserved for the trigger status. The server wasn't grouping results
// by this field.
reserved 4;
// Time buckets.
repeated google.protobuf.Timestamp time_buckets = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregated trigger count in each time bucket.
repeated int32 trigger_counts = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// 7 is reserved for the trigger execution duration.
reserved 7;
// 8 is reserved for the pipeline release ID. The server wasn't grouping
// results by this field.
reserved 8;
// 9 is reserved for the pipeline release UUID. The server wasn't grouping
// results by this field.
reserved 9;
// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
}
*/

// GetPipelineTriggerCountRequest represents a request to fetch the trigger
// count of a requester over a time period.
message GetPipelineTriggerCountRequest {
Expand Down Expand Up @@ -111,44 +78,42 @@ message GetModelTriggerCountResponse {
repeated TriggerCount model_trigger_counts = 1;
}

/*
// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger chart records for a given requester, grouped by time buckets.
message ListPipelineTriggerChartRecordsRequest {
// 1 is reserved for the aggregation window in nanoseconds. This is
// deprecated in favour of an aggregation window string that represents a
// duration.
reserved 1;
// 2 is reserved for the filter. For now, this endpoint won't allow filtering
// but in the future we might implement a filter to show the trigger count of
// only certain pipelines and to group by the pipeline ID.
reserved 2;
// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger chart records for a given requester, grouped by time buckets.
message ListPipelineTriggerChartRecordsRequest {
// 1 is reserved for the aggregation window in nanoseconds. This is
// deprecated in favour of an aggregation window string that represents a
// duration.
reserved 1;
// 2 is reserved for the filter. For now, this endpoint won't allow filtering
// but in the future we might implement a filter to show the trigger count of
// only certain pipelines and to group by the pipeline ID.
reserved 2;

// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 3 [(google.api.field_behavior) = REQUIRED];
// Aggregation window. The value is a positive duration string, i.e. a
// sequence of decimal numbers, each with optional fraction and a unit
// suffix, such as "300ms", "1.5h" or "2h45m".
// The minimum (and default) window is 1h.
optional string aggregation_window = 4;
// Beginning of the time range from which the records will be fetched.
// The default value is the beginning of the current day, in UTC.
optional google.protobuf.Timestamp start = 5;
// End of the time range from which the records will be fetched.
// The default value is the current timestamp.
optional google.protobuf.Timestamp stop = 6;
}
// The ID of the namespace that requested the pipeline triggers.
string requester_id = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregation window. The value is a positive duration string, i.e. a
// sequence of decimal numbers, each with optional fraction and a unit
// suffix, such as "300ms", "1.5h" or "2h45m".
// The minimum (and default) window is 1h.
optional string aggregation_window = 4;
// Beginning of the time range from which the records will be fetched.
// The default value is the beginning of the current day, in UTC.
optional google.protobuf.Timestamp start = 5;
// End of the time range from which the records will be fetched.
// The default value is the current timestamp.
optional google.protobuf.Timestamp stop = 6;
}

// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// Pipeline trigger counts. Until we allow filtering or grouping by fields
// like pipeline ID, this list will contain only one element with the
// timeline of trigger counts for a given requester, regardless the pipeline
// ID, trigger mode, final status or other fields.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}
*/
// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// Pipeline trigger counts. Until we allow filtering or grouping by fields
// like pipeline ID, this list will contain only one element with the
// timeline of trigger counts for a given requester, regardless the pipeline
// ID, trigger mode, final status or other fields.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}

// CreditConsumptionChartRecord represents a timeline of Instill Credit
// consumption. It contains a collection of (timestamp, amount) pairs that
Expand Down Expand Up @@ -194,61 +159,6 @@ message ListCreditConsumptionChartRecordsResponse {
reserved 2;
}

// Deprecated messages, to be removed with the new dashboard implementation.

// PipelineTriggerTableRecord contains pipeline trigger metrics, aggregated by
// pipeline ID.
message PipelineTriggerTableRecord {
// Pipeline ID.
string pipeline_id = 1;
// Pipeline UUID.
string pipeline_uid = 2;
// Number of triggers with `STATUS_COMPLETED`.
int32 trigger_count_completed = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Number of triggers with `STATUS_ERRORED`.
int32 trigger_count_errored = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
// Version for the triggered pipeline if it is a release pipeline.
string pipeline_release_id = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Release UUID for the triggered pipeline if it is a release pipeline.
string pipeline_release_uid = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// ListPipelineTriggerTableRecordsRequest represents a request to list the
// pipeline triggers metrics, aggregated by pipeline ID.
message ListPipelineTriggerTableRecordsRequest {
// The maximum number of results to return. If this parameter is unspecified,
// at most 100 pipelines will be returned. The cap value for this parameter
// is 1000 (i.e. any value above that will be coerced to 1000).
optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL];
// Page token.
optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL];
// Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter
// expression.
// - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`.
optional string filter = 3 [(google.api.field_behavior) = OPTIONAL];
}

// ListPipelineTriggerTableRecordsResponse contains the pipeline metrics.
message ListPipelineTriggerTableRecordsResponse {
// A list of pipeline trigger tables.
repeated PipelineTriggerTableRecord pipeline_trigger_table_records = 1;
// Next page token.
string next_page_token = 2;
// Total number of pipeline trigger records
int32 total_size = 3;
}

// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger metrics, aggregated by pipeline ID and time frame.
message ListPipelineTriggerChartRecordsRequest {
// Aggregation window in nanoseconds.
int32 aggregation_window = 1;
// Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter
// expression.
// - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`.
optional string filter = 2 [(google.api.field_behavior) = OPTIONAL];
}

// ListModelTriggerChartRecordsRequest represents a request to list model
// trigger metrics, aggregated by model ID and time frame.
message ListModelTriggerChartRecordsRequest {
Expand All @@ -267,13 +177,6 @@ message ListModelTriggerChartRecordsRequest {
optional google.protobuf.Timestamp stop = 6;
}

// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// A list of pipeline trigger records.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}

// ListModelTriggerChartRecordsResponse contains a list of model trigger
// chart records.
message ListModelTriggerChartRecordsResponse {
Expand All @@ -284,27 +187,35 @@ message ListModelTriggerChartRecordsResponse {
repeated ModelTriggerChartRecord model_trigger_chart_records = 1;
}

// PipelineTriggerChartRecord contains pipeline trigger metrics, aggregated by
// PipelineTriggerChartRecord represents a timeline of pipeline triggers. It
// contains a collection of (timestamp, count) pairs that represent the total
// pipeline triggers in a given time bucket.
// pipeline ID and time frame.
message PipelineTriggerChartRecord {
// Pipeline ID.
string pipeline_id = 1;
// Pipeline UUID.
string pipeline_uid = 2;
// Trigger mode.
Mode trigger_mode = 3;
// Final status.
Status status = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
// This field will be present present when the information is grouped by pipeline.
optional string pipeline_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// 2 is reserved for the pipeline UUID.
reserved 2;
// 3 is reserved for the trigger mode. The server wasn't grouping results by this
// field.
reserved 3;
// 4 is reserved for the trigger status. The server wasn't grouping results
// by this field.
reserved 4;
// Time buckets.
repeated google.protobuf.Timestamp time_buckets = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregated trigger count in each time bucket.
repeated int32 trigger_counts = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// Total computation time duration in each time bucket.
repeated float compute_time_duration = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
// Version for the triggered pipeline if it is a release pipeline.
string pipeline_release_id = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
// Release UUID for the triggered pipeline if it is a release pipeline.
string pipeline_release_uid = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
// 7 is reserved for the trigger execution duration.
reserved 7;
// 8 is reserved for the pipeline release ID. The server wasn't grouping
// results by this field.
reserved 8;
// 9 is reserved for the pipeline release UUID. The server wasn't grouping
// results by this field.
reserved 9;
// The ID of the namespace that requested the model triggers.
string requester_id = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// PipelineTriggerRecord represents a pipeline execution event.
Expand Down
33 changes: 5 additions & 28 deletions core/mgmt/v1beta/mgmt_public_service.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,17 +458,14 @@ service MgmtPublicService {
// Returns the pipeline trigger count of a given requester within a timespan.
// Results are grouped by trigger status.
rpc GetPipelineTriggerCount(GetPipelineTriggerCountRequest) returns (GetPipelineTriggerCountResponse) {
option (google.api.http) = {get: "/v1beta/metrics/vdp/pipeline/trigger-count"};
option (google.api.http) = {get: "/v1beta/pipeline-runs/count"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
tags: "📊 Metrics"
extensions: {
key: "x-stage"
value: {string_value: "beta"}
}
};
// This endpoint will remain hidden until the new dashboard is implemented
// in the frontend. Until then, the server might return empty data.
option (google.api.method_visibility).restriction = "INTERNAL";
}

// Get model trigger count
Expand All @@ -484,35 +481,15 @@ service MgmtPublicService {
value: {string_value: "beta"}
}
};
// This endpoint will remain hidden until the new dashboard is implemented
// in the frontend. Until then, the server might return empty data.
option (google.api.method_visibility).restriction = "INTERNAL";
}

// List pipeline trigger metrics
//
// Returns a paginated list of pipeline executions aggregated by pipeline ID.
// NOTE: This method is deprecated and will be retired soon.
rpc ListPipelineTriggerTableRecords(ListPipelineTriggerTableRecordsRequest) returns (ListPipelineTriggerTableRecordsResponse) {
option (google.api.http) = {get: "/v1beta/metrics/vdp/pipeline/tables"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
tags: "📊 Metrics"
extensions: {
key: "x-stage"
value: {string_value: "beta"}
}
};
option deprecated = true;
option (google.api.method_visibility).restriction = "INTERNAL";
}

// List pipeline trigger time charts
//
// Returns a timeline of pipline trigger counts for the pipelines of a given
// owner.
// NOTE: This method will soon return the trigger counts of a given requester.
// Returns a timeline of pipline trigger counts for a given requester. The
// response will contain one set of records (datapoints), representing the
// amount of triggers in a time bucket.
rpc ListPipelineTriggerChartRecords(ListPipelineTriggerChartRecordsRequest) returns (ListPipelineTriggerChartRecordsResponse) {
option (google.api.http) = {get: "/v1beta/metrics/vdp/pipeline/charts"};
option (google.api.http) = {get: "/v1beta/pipeline-runs/query-charts"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
tags: "📊 Metrics"
extensions: {
Expand Down
16 changes: 9 additions & 7 deletions model/model/v1alpha/model.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1724,8 +1724,9 @@ message UndeployOrganizationModelAdminResponse {}
message ModelRun {
// Model Run UUID.
string uid = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Model UUID.
string model_uid = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
// 2 is reserved for the model UUID. Public resources should be identified by
// their ID and parent resource.
reserved 2;
// Model run status.
common.run.v1alpha.RunStatus status = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Run source.
Expand All @@ -1740,7 +1741,8 @@ message ModelRun {
(google.api.field_behavior) = OUTPUT_ONLY,
(google.api.field_behavior) = OPTIONAL
];
// Runner ID. If current viewing requester does not have enough permission, it will return null.
// Runner ID. The user triggered the run. If current viewing requester does not
// have enough permission, it will return null.
optional string runner_id = 7 [
(google.api.field_behavior) = OUTPUT_ONLY,
(google.api.field_behavior) = OPTIONAL
Expand Down Expand Up @@ -1770,11 +1772,11 @@ message ModelRun {
(google.api.field_behavior) = OUTPUT_ONLY,
(google.api.field_behavior) = OPTIONAL
];
// Requester ID. This field might be empty if the model run belongs to a
// deleted namespace.
// Requester ID. The namespace used to trigger the run. This field might be
// empty if the model run belongs to a deleted namespace.
string requester_id = 16 [(google.api.field_behavior) = OUTPUT_ONLY];
joremysh marked this conversation as resolved.
Show resolved Hide resolved
// Namespace ID.
string namespace_id = 17 [(google.api.field_behavior) = OUTPUT_ONLY];
// ID of the namespace that owns the model.
string model_namespace_id = 21 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// ListModelRunsRequest represents a request to list of model runs.
Expand Down
Loading