Skip to content

Commit

Permalink
feat(metric): implement new pipeline dashboard endpoints (#238)
Browse files Browse the repository at this point in the history 
Because

- The new dashboard introduces changes in the metrics endpoints.

This commit

- Applies contracts defined in
instill-ai/protobufs#435, keeping the API
backwards-compatible.
  • Loading branch information
@jvallesm
jvallesm authored Nov 26, 2024
1 parent 07fe104 commit afc27e8
Show file tree
Hide file tree
Showing 9 changed files with 731 additions and 202 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ require (
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1
github.com/iancoleman/strcase v0.2.0
github.com/influxdata/influxdb-client-go/v2 v2.12.3
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241104132014-1bf251739feb
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241120170237-ee31d10bc9c8
github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a
github.com/instill-ai/x v0.4.0-alpha
github.com/knadh/koanf v1.5.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1090,8 +1090,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.12.3 h1:28nRlNMRIV4QbtIUvxhWqaxn0
github.com/influxdata/influxdb-client-go/v2 v2.12.3/go.mod h1:IrrLUbCjjfkmRuaCiGQg4m2GbkaeJDcuWoxiWdQEbA0=
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241104132014-1bf251739feb h1:tJ7/VFdviz4bs3IVkuKwjrs/sltNPb1a3xVx3u3T7hA=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241104132014-1bf251739feb/go.mod h1:rf0UY7VpEgpaLudYEcjx5rnbuwlBaaLyD4FQmWLtgAY=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241120170237-ee31d10bc9c8 h1:Mq87LDBN4fmpN7tD/UMTMahqR/FupEHVIPCPqLZh4XI=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241120170237-ee31d10bc9c8/go.mod h1:rf0UY7VpEgpaLudYEcjx5rnbuwlBaaLyD4FQmWLtgAY=
github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a h1:gmy8BcCFDZQan40c/D3f62DwTYtlCwi0VrSax+pKffw=
github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a/go.mod h1:EpX3Yr661uWULtZf5UnJHfr5rw2PDyX8ku4Kx0UtYFw=
github.com/instill-ai/x v0.4.0-alpha h1:zQV2VLbSHjMv6gyBN/2mwwrvWk0/mJM6ZKS12AzjfQg=
Expand Down
12 changes: 6 additions & 6 deletions integration-test/grpc-public-user.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,15 +293,15 @@ export function CheckPublicMetrics(header) {
});

group(`Management Public API: List Pipeline Trigger Chart Records`, () => {
check(client.invoke('core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords', {}, header), {
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords status': (r) => r && r.status == grpc.StatusOK,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords response has pipelineTriggerChartRecords': (r) => r && r.message.pipelineTriggerChartRecords !== undefined,
check(client.invoke('core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecordsV0', {}, header), {
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecordsV0 status': (r) => r && r.status == grpc.StatusOK,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecordsV0 response has pipelineTriggerChartRecords': (r) => r && r.message.pipelineTriggerChartRecords !== undefined,
});
check(client.invoke('core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords', {
check(client.invoke('core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecordsV0', {
filter: `pipelineId="${pipeline_id}" AND triggerMode=MODE_SYNC`,
}, header), {
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords with filter status': (r) => r && r.status == grpc.StatusOK,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords with filter response pipelineTriggerChartRecords lenght is 0': (r) => r && r.message.pipelineTriggerChartRecords.length === 0,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecordsV0 with filter status': (r) => r && r.status == grpc.StatusOK,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecordsV0 with filter response pipelineTriggerChartRecords lenght is 0': (r) => r && r.message.pipelineTriggerChartRecords.length === 0,
});
});

Expand Down
203 changes: 118 additions & 85 deletions integration-test/proto/core/mgmt/v1beta/metric.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,55 +27,20 @@ enum Status {
STATUS_ERRORED = 2;
}

// ========== Pipeline endpoints

// PipelineTriggerCount represents a pipeline execution count with some
// TriggerCount represents a execution count with some
// aggregation params (e.g. trigger status).
message PipelineTriggerCount {
message TriggerCount {
// Number of triggers.
int32 trigger_count = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// This field will be present when results are grouped by trigger status.
optional Status status = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
}

/*
// PipelineTriggerChartRecord represents a timeline of pipeline triggers. It
// contains a collection of (timestamp, count) pairs that represent the total
// pipeline triggers in a given time bucket.
// pipeline ID and time frame.
message PipelineTriggerChartRecord {
// This field will be present present when the information is grouped by pipeline.
optional string pipeline_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// 2 is reserved for the pipeline UUID.
reserved 2;
// 3 is reserved for the trigger mode. The server wasn't grouping results by this
// field.
reserved 3;
// 4 is reserved for the trigger status. The server wasn't grouping results
// by this field.
reserved 4;
// Time buckets.
repeated google.protobuf.Timestamp time_buckets = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregated trigger count in each time bucket.
repeated int32 trigger_counts = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// 7 is reserved for the trigger execution duration.
reserved 7;
// 8 is reserved for the pipeline release ID. The server wasn't grouping
// results by this field.
reserved 8;
// 9 is reserved for the pipeline release UUID. The server wasn't grouping
// results by this field.
reserved 9;
// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
}
*/

// GetPipelineTriggerCountRequest represents a request to fetch the trigger
// count of a requester over a time period.
message GetPipelineTriggerCountRequest {
// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 1 [(google.api.field_behavior) = REQUIRED];
string requester_id = 1 [(google.api.field_behavior) = REQUIRED];
// Beginning of the time range from which the records will be fetched.
// The default value is the beginning of the current day, in UTC.
optional google.protobuf.Timestamp start = 2;
Expand All @@ -88,47 +53,113 @@ message GetPipelineTriggerCountRequest {
// trigger status.
message GetPipelineTriggerCountResponse {
// The trigger counts, grouped by status.
repeated PipelineTriggerCount pipeline_trigger_counts = 1;
repeated TriggerCount pipeline_trigger_counts = 1;
}

// GetModelTriggerCountRequest represents a request to fetch the trigger
// count of a requester over a time period.
message GetModelTriggerCountRequest {
// The ID of the requester that triggered the model.
string requester_id = 1 [(google.api.field_behavior) = REQUIRED];
// Beginning of the time range from which the records will be fetched.
// The default value is the beginning of the current day, in UTC.
optional google.protobuf.Timestamp start = 2;
// End of the time range from which the records will be fetched.
// The default value is the current timestamp.
optional google.protobuf.Timestamp stop = 3;
}

// GetModelTriggerCountResponse contains the trigger count, grouped by
// trigger status.
message GetModelTriggerCountResponse {
// The trigger counts, grouped by status.
repeated TriggerCount model_trigger_counts = 1;
}

// PipelineTriggerChartRecord represents a timeline of pipeline triggers. It
// contains a collection of (timestamp, count) pairs that represent the total
// pipeline triggers in a given time bucket.
// pipeline ID and time frame.
message PipelineTriggerChartRecord {
// This field will be present present when the information is grouped by pipeline.
optional string pipeline_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Time buckets.
repeated google.protobuf.Timestamp time_buckets = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregated trigger count in each time bucket.
repeated int32 trigger_counts = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// The ID of the namespace that requested the pipeline triggers.
string requester_id = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger chart records for a given requester, grouped by time buckets.
message ListPipelineTriggerChartRecordsRequest {
// The ID of the namespace that requested the pipeline triggers.
string requester_id = 1 [(google.api.field_behavior) = REQUIRED];
// Aggregation window. The value is a positive duration string, i.e. a
// sequence of decimal numbers, each with optional fraction and a unit
// suffix, such as "300ms", "1.5h" or "2h45m".
// The minimum (and default) window is 1h.
optional string aggregation_window = 2;
// Beginning of the time range from which the records will be fetched.
// The default value is the beginning of the current day, in UTC.
optional google.protobuf.Timestamp start = 3;
// End of the time range from which the records will be fetched.
// The default value is the current timestamp.
optional google.protobuf.Timestamp stop = 4;
}

// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// Pipeline trigger counts. Until we allow filtering or grouping by fields
// like pipeline ID, this list will contain only one element with the
// timeline of trigger counts for a given requester, regardless the pipeline
// ID, trigger mode, final status or other fields.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}

/*
// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger chart records for a given requester, grouped by time buckets.
message ListPipelineTriggerChartRecordsRequest {
// 1 is reserved for the aggregation window in nanoseconds. This is
// deprecated in favour of an aggregation window string that represents a
// duration.
reserved 1;
// 2 is reserved for the filter. For now, this endpoint won't allow filtering
// but in the future we might implement a filter to show the trigger count of
// only certain pipelines and to group by the pipeline ID.
reserved 2;
// ModelTriggerChartRecord represents a timeline of model triggers. It
// contains a collection of (timestamp, count) pairs that represent the total
// model triggers in a given time bucket.
message ModelTriggerChartRecord {
// This field will be present present when the information is grouped by model.
optional string model_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// Time buckets.
repeated google.protobuf.Timestamp time_buckets = 2 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregated trigger count in each time bucket.
repeated int32 trigger_counts = 3 [(google.api.field_behavior) = OUTPUT_ONLY];
// The ID of the namespace that requested the model triggers.
string requester_id = 4 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 3 [(google.api.field_behavior) = REQUIRED];
// Aggregation window. The value is a positive duration string, i.e. a
// sequence of decimal numbers, each with optional fraction and a unit
// suffix, such as "300ms", "1.5h" or "2h45m".
// The minimum (and default) window is 1h.
optional string aggregation_window = 4;
// Beginning of the time range from which the records will be fetched.
// The default value is the beginning of the current day, in UTC.
optional google.protobuf.Timestamp start = 5;
// End of the time range from which the records will be fetched.
// The default value is the current timestamp.
optional google.protobuf.Timestamp stop = 6;
}
// ListModelTriggerChartRecordsRequest represents a request to list model
// trigger metrics, aggregated by model ID and time frame.
message ListModelTriggerChartRecordsRequest {
// The ID of the namespace that requested the model triggers.
string requester_id = 1 [(google.api.field_behavior) = REQUIRED];
// Aggregation window. The value is a positive duration string, i.e. a
// sequence of decimal numbers, each with optional fraction and a unit
// suffix, such as "300ms", "1.5h" or "2h45m".
// The minimum (and default) window is 1h.
optional string aggregation_window = 2;
// Beginning of the time range from which the records will be fetched.
// The default value is the beginning of the current day, in UTC.
optional google.protobuf.Timestamp start = 3;
// End of the time range from which the records will be fetched.
// The default value is the current timestamp.
optional google.protobuf.Timestamp stop = 4;
}

// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// Pipeline trigger counts. Until we allow filtering or grouping by fields
// like pipeline ID, this list will contain only one element with the
// timeline of trigger counts for a given requester, regardless the pipeline
// ID, trigger mode, final status or other fields.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}
*/
// ListModelTriggerChartRecordsResponse contains a list of model trigger
// chart records.
message ListModelTriggerChartRecordsResponse {
// Model trigger counts. Until we allow filtering or grouping by fields
// like model ID, this list will contain only one element with the
// timeline of trigger counts for a given requester, regardless the model
// ID, trigger mode, final status or other fields.
repeated ModelTriggerChartRecord model_trigger_chart_records = 1;
}

// CreditConsumptionChartRecord represents a timeline of Instill Credit
// consumption. It contains a collection of (timestamp, amount) pairs that
Expand Down Expand Up @@ -174,7 +205,9 @@ message ListCreditConsumptionChartRecordsResponse {
reserved 2;
}

// =============================================================================
// Deprecated messages, to be removed with the new dashboard implementation.
// =============================================================================

// PipelineTriggerTableRecord contains pipeline trigger metrics, aggregated by
// pipeline ID.
Expand Down Expand Up @@ -218,9 +251,9 @@ message ListPipelineTriggerTableRecordsResponse {
int32 total_size = 3;
}

// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger metrics, aggregated by pipeline ID and time frame.
message ListPipelineTriggerChartRecordsRequest {
// ListPipelineTriggerChartRecordsV0Request represents a request to list
// pipeline trigger metrics, aggregated by pipeline ID and time frame.
message ListPipelineTriggerChartRecordsV0Request {
// Aggregation window in nanoseconds.
int32 aggregation_window = 1;
// Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter
Expand All @@ -229,16 +262,16 @@ message ListPipelineTriggerChartRecordsRequest {
optional string filter = 2 [(google.api.field_behavior) = OPTIONAL];
}

// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// ListPipelineTriggerChartRecordsV0Response contains a list of pipeline
// trigger chart records.
message ListPipelineTriggerChartRecordsV0Response {
// A list of pipeline trigger records.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
repeated PipelineTriggerChartRecordV0 pipeline_trigger_chart_records = 1;
}

// PipelineTriggerChartRecord contains pipeline trigger metrics, aggregated by
// pipeline ID and time frame.
message PipelineTriggerChartRecord {
// PipelineTriggerChartRecordV0 contains pipeline trigger metrics, aggregated
// by pipeline ID and time frame.
message PipelineTriggerChartRecordV0 {
// Pipeline ID.
string pipeline_id = 1;
// Pipeline UUID.
Expand Down
Loading

0 comments on commit afc27e8

Please sign in to comment.