Skip to content

Commit

Permalink
feat(mgmt): update metrics endpoints to serve new dashboard design
Browse files Browse the repository at this point in the history
  • Loading branch information
@joremysh
joremysh committed Nov 6, 2024
1 parent 0f9ef80 commit a2a24c7
Show file tree
Hide file tree
Showing 4 changed files with 70 additions and 179 deletions.
203 changes: 57 additions & 146 deletions core/mgmt/v1beta/metric.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,39 +38,6 @@ message TriggerCount {
optional Status status = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
}

/*
// PipelineTriggerChartRecord represents a timeline of pipeline triggers. It
// contains a collection of (timestamp, count) pairs that represent the total
// pipeline triggers in a given time bucket.
// pipeline ID and time frame.
message PipelineTriggerChartRecord {
// This field will be present present when the information is grouped by pipeline.
optional string pipeline_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// 2 is reserved for the pipeline UUID.
reserved 2;
// 3 is reserved for the trigger mode. The server wasn't grouping results by this
// field.
reserved 3;
// 4 is reserved for the trigger status. The server wasn't grouping results
// by this field.
reserved 4;
// Time buckets.
repeated google.protobuf.Timestamp time_buckets = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregated trigger count in each time bucket.
repeated int32 trigger_counts = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// 7 is reserved for the trigger execution duration.
reserved 7;
// 8 is reserved for the pipeline release ID. The server wasn't grouping
// results by this field.
reserved 8;
// 9 is reserved for the pipeline release UUID. The server wasn't grouping
// results by this field.
reserved 9;
// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
}
*/

// GetPipelineTriggerCountRequest represents a request to fetch the trigger
// count of a requester over a time period.
message GetPipelineTriggerCountRequest {
Expand Down Expand Up @@ -111,44 +78,42 @@ message GetModelTriggerCountResponse {
repeated TriggerCount model_trigger_counts = 1;
}

/*
// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger chart records for a given requester, grouped by time buckets.
message ListPipelineTriggerChartRecordsRequest {
// 1 is reserved for the aggregation window in nanoseconds. This is
// deprecated in favour of an aggregation window string that represents a
// duration.
reserved 1;
// 2 is reserved for the filter. For now, this endpoint won't allow filtering
// but in the future we might implement a filter to show the trigger count of
// only certain pipelines and to group by the pipeline ID.
reserved 2;
// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger chart records for a given requester, grouped by time buckets.
message ListPipelineTriggerChartRecordsRequest {
// 1 is reserved for the aggregation window in nanoseconds. This is
// deprecated in favour of an aggregation window string that represents a
// duration.
reserved 1;
// 2 is reserved for the filter. For now, this endpoint won't allow filtering
// but in the future we might implement a filter to show the trigger count of
// only certain pipelines and to group by the pipeline ID.
reserved 2;

// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 3 [(google.api.field_behavior) = REQUIRED];
// Aggregation window. The value is a positive duration string, i.e. a
// sequence of decimal numbers, each with optional fraction and a unit
// suffix, such as "300ms", "1.5h" or "2h45m".
// The minimum (and default) window is 1h.
optional string aggregation_window = 4;
// Beginning of the time range from which the records will be fetched.
// The default value is the beginning of the current day, in UTC.
optional google.protobuf.Timestamp start = 5;
// End of the time range from which the records will be fetched.
// The default value is the current timestamp.
optional google.protobuf.Timestamp stop = 6;
}
// The ID of the namespace that requested the pipeline triggers.
string requester_id = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregation window. The value is a positive duration string, i.e. a
// sequence of decimal numbers, each with optional fraction and a unit
// suffix, such as "300ms", "1.5h" or "2h45m".
// The minimum (and default) window is 1h.
optional string aggregation_window = 4;
// Beginning of the time range from which the records will be fetched.
// The default value is the beginning of the current day, in UTC.
optional google.protobuf.Timestamp start = 5;
// End of the time range from which the records will be fetched.
// The default value is the current timestamp.
optional google.protobuf.Timestamp stop = 6;
}

// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// Pipeline trigger counts. Until we allow filtering or grouping by fields
// like pipeline ID, this list will contain only one element with the
// timeline of trigger counts for a given requester, regardless the pipeline
// ID, trigger mode, final status or other fields.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}
*/
// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// Pipeline trigger counts. Until we allow filtering or grouping by fields
// like pipeline ID, this list will contain only one element with the
// timeline of trigger counts for a given requester, regardless the pipeline
// ID, trigger mode, final status or other fields.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}

// CreditConsumptionChartRecord represents a timeline of Instill Credit
// consumption. It contains a collection of (timestamp, amount) pairs that
Expand Down Expand Up @@ -194,61 +159,6 @@ message ListCreditConsumptionChartRecordsResponse {
reserved 2;
}

// Deprecated messages, to be removed with the new dashboard implementation.

// PipelineTriggerTableRecord contains pipeline trigger metrics, aggregated by
// pipeline ID.
message PipelineTriggerTableRecord {
// Pipeline ID.
string pipeline_id = 1;
// Pipeline UUID.
string pipeline_uid = 2;
// Number of triggers with `STATUS_COMPLETED`.
int32 trigger_count_completed = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Number of triggers with `STATUS_ERRORED`.
int32 trigger_count_errored = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
// Version for the triggered pipeline if it is a release pipeline.
string pipeline_release_id = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Release UUID for the triggered pipeline if it is a release pipeline.
string pipeline_release_uid = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// ListPipelineTriggerTableRecordsRequest represents a request to list the
// pipeline triggers metrics, aggregated by pipeline ID.
message ListPipelineTriggerTableRecordsRequest {
// The maximum number of results to return. If this parameter is unspecified,
// at most 100 pipelines will be returned. The cap value for this parameter
// is 1000 (i.e. any value above that will be coerced to 1000).
optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL];
// Page token.
optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL];
// Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter
// expression.
// - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`.
optional string filter = 3 [(google.api.field_behavior) = OPTIONAL];
}

// ListPipelineTriggerTableRecordsResponse contains the pipeline metrics.
message ListPipelineTriggerTableRecordsResponse {
// A list of pipeline trigger tables.
repeated PipelineTriggerTableRecord pipeline_trigger_table_records = 1;
// Next page token.
string next_page_token = 2;
// Total number of pipeline trigger records
int32 total_size = 3;
}

// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger metrics, aggregated by pipeline ID and time frame.
message ListPipelineTriggerChartRecordsRequest {
// Aggregation window in nanoseconds.
int32 aggregation_window = 1;
// Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter
// expression.
// - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`.
optional string filter = 2 [(google.api.field_behavior) = OPTIONAL];
}

// ListModelTriggerChartRecordsRequest represents a request to list model
// trigger metrics, aggregated by model ID and time frame.
message ListModelTriggerChartRecordsRequest {
Expand All @@ -267,13 +177,6 @@ message ListModelTriggerChartRecordsRequest {
optional google.protobuf.Timestamp stop = 6;
}

// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// A list of pipeline trigger records.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}

// ListModelTriggerChartRecordsResponse contains a list of model trigger
// chart records.
message ListModelTriggerChartRecordsResponse {
Expand All @@ -284,27 +187,35 @@ message ListModelTriggerChartRecordsResponse {
repeated ModelTriggerChartRecord model_trigger_chart_records = 1;
}

// PipelineTriggerChartRecord contains pipeline trigger metrics, aggregated by
// PipelineTriggerChartRecord represents a timeline of pipeline triggers. It
// contains a collection of (timestamp, count) pairs that represent the total
// pipeline triggers in a given time bucket.
// pipeline ID and time frame.
message PipelineTriggerChartRecord {
// Pipeline ID.
string pipeline_id = 1;
// Pipeline UUID.
string pipeline_uid = 2;
// Trigger mode.
Mode trigger_mode = 3;
// Final status.
Status status = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
// This field will be present present when the information is grouped by pipeline.
optional string pipeline_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// 2 is reserved for the pipeline UUID.
reserved 2;
// 3 is reserved for the trigger mode. The server wasn't grouping results by this
// field.
reserved 3;
// 4 is reserved for the trigger status. The server wasn't grouping results
// by this field.
reserved 4;
// Time buckets.
repeated google.protobuf.Timestamp time_buckets = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregated trigger count in each time bucket.
repeated int32 trigger_counts = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// Total computation time duration in each time bucket.
repeated float compute_time_duration = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
// Version for the triggered pipeline if it is a release pipeline.
string pipeline_release_id = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
// Release UUID for the triggered pipeline if it is a release pipeline.
string pipeline_release_uid = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
// 7 is reserved for the trigger execution duration.
reserved 7;
// 8 is reserved for the pipeline release ID. The server wasn't grouping
// results by this field.
reserved 8;
// 9 is reserved for the pipeline release UUID. The server wasn't grouping
// results by this field.
reserved 9;
// The ID of the namespace that requested the model triggers.
string requester_id = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// PipelineTriggerRecord represents a pipeline execution event.
Expand Down
30 changes: 5 additions & 25 deletions core/mgmt/v1beta/mgmt_public_service.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -458,17 +458,14 @@ service MgmtPublicService {
// Returns the pipeline trigger count of a given requester within a timespan.
// Results are grouped by trigger status.
rpc GetPipelineTriggerCount(GetPipelineTriggerCountRequest) returns (GetPipelineTriggerCountResponse) {
option (google.api.http) = {get: "/v1beta/metrics/vdp/pipeline/trigger-count"};
option (google.api.http) = {get: "/v1beta/pipeline-runs/count"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
tags: "📊 Metrics"
extensions: {
key: "x-stage"
value: {string_value: "beta"}
}
};
// This endpoint will remain hidden until the new dashboard is implemented
// in the frontend. Until then, the server might return empty data.
option (google.api.method_visibility).restriction = "INTERNAL";
}

// Get model trigger count
Expand All @@ -489,30 +486,13 @@ service MgmtPublicService {
option (google.api.method_visibility).restriction = "INTERNAL";
}

// List pipeline trigger metrics
//
// Returns a paginated list of pipeline executions aggregated by pipeline ID.
// NOTE: This method is deprecated and will be retired soon.
rpc ListPipelineTriggerTableRecords(ListPipelineTriggerTableRecordsRequest) returns (ListPipelineTriggerTableRecordsResponse) {
option (google.api.http) = {get: "/v1beta/metrics/vdp/pipeline/tables"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
tags: "📊 Metrics"
extensions: {
key: "x-stage"
value: {string_value: "beta"}
}
};
option deprecated = true;
option (google.api.method_visibility).restriction = "INTERNAL";
}

// List pipeline trigger time charts
//
// Returns a timeline of pipline trigger counts for the pipelines of a given
// owner.
// NOTE: This method will soon return the trigger counts of a given requester.
// Returns a timeline of pipline trigger counts for a given requester. The
// response will contain one set of records (datapoints), representing the
// amount of triggers in a time bucket.
rpc ListPipelineTriggerChartRecords(ListPipelineTriggerChartRecordsRequest) returns (ListPipelineTriggerChartRecordsResponse) {
option (google.api.http) = {get: "/v1beta/metrics/vdp/pipeline/charts"};
option (google.api.http) = {get: "/v1beta/pipeline-runs/query-charts"};
option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
tags: "📊 Metrics"
extensions: {
Expand Down
8 changes: 4 additions & 4 deletions model/model/v1alpha/model.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1724,8 +1724,8 @@ message UndeployOrganizationModelAdminResponse {}
message ModelRun {
// Model Run UUID.
string uid = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Model UUID.
string model_uid = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
// Reversed for `model_uid`
reserved 2;
// Model run status.
common.run.v1alpha.RunStatus status = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// Run source.
Expand Down Expand Up @@ -1773,8 +1773,8 @@ message ModelRun {
// Requester ID. This field might be empty if the model run belongs to a
// deleted namespace.
string requester_id = 16 [(google.api.field_behavior) = OUTPUT_ONLY];
// Namespace ID.
string namespace_id = 17 [(google.api.field_behavior) = OUTPUT_ONLY];
// Model Namespace ID. The owner namespace id of the model.
string model_namespace_id = 21 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// ListModelRunsRequest represents a request to list of model runs.
Expand Down
8 changes: 4 additions & 4 deletions vdp/pipeline/v1beta/pipeline.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1996,8 +1996,8 @@ message FileReference {

// PipelineRun represents a single execution of a pipeline.
message PipelineRun {
// Unique identifier for the pipeline.
string pipeline_uid = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Reversed for `pipeline_uid`
reserved 1;

// Unique identifier for each run.
string pipeline_run_uid = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
Expand Down Expand Up @@ -2076,8 +2076,8 @@ message PipelineRun {
// deleted namespace.
string requester_id = 20 [(google.api.field_behavior) = OUTPUT_ONLY];

// Namespace ID
string namespace_id = 21 [(google.api.field_behavior) = OUTPUT_ONLY];
// Pipeline Namespace ID. The owner namespace id of the pipeline.
string pipeline_namespace_id = 21 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// ComponentRun represents the execution details of a single component within a pipeline run.
Expand Down

0 comments on commit a2a24c7

Please sign in to comment.