diff --git a/pkg/api/server/cel/view/records.go b/pkg/api/server/cel/view/records.go new file mode 100644 index 000000000..3719173c9 --- /dev/null +++ b/pkg/api/server/cel/view/records.go @@ -0,0 +1,40 @@ +package view + +import ( + "github.com/google/cel-go/cel" + "github.com/tektoncd/results/pkg/api/server/cel2sql" +) + +// NewRecordsView return the set of variables and constants available for CEL +// filters +func NewRecordsView() (*cel2sql.View, error) { + view := &cel2sql.View{ + Constants: map[string]cel2sql.Constant{}, + Fields: map[string]cel2sql.Field{ + "parent": { + CELType: cel.StringType, + SQL: `parent`, + }, + "result_name": { + CELType: cel.StringType, + SQL: `result_name`, + }, + "name": { + CELType: cel.StringType, + SQL: `name`, + }, + "data_type": { + CELType: cel.StringType, + SQL: `type`, + }, + "data": { + CELType: cel.AnyType, + SQL: `data`, + }, + }, + } + for typeName, value := range typeConstants { + view.Constants[typeName] = value + } + return view, nil +} diff --git a/pkg/api/server/cel/view/records_test.go b/pkg/api/server/cel/view/records_test.go new file mode 100644 index 000000000..070149daf --- /dev/null +++ b/pkg/api/server/cel/view/records_test.go @@ -0,0 +1,24 @@ +package view + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/tektoncd/results/pkg/api/server/cel2sql" + "k8s.io/utils/pointer" +) + +func TestNewRecordsViewConstants(t *testing.T) { + view, err := NewRecordsView() + if err != nil { + t.Fatal(err) + } + + expectedConstants := map[string]cel2sql.Constant{ + "PIPELINE_RUN": {StringVal: pointer.String("tekton.dev/v1beta1.PipelineRun")}, + "TASK_RUN": {StringVal: pointer.String("tekton.dev/v1beta1.TaskRun")}, + } + if diff := cmp.Diff(expectedConstants, view.Constants); diff != "" { + t.Errorf("Invalid constants (-want, +got):\n%s", diff) + } +} diff --git a/pkg/api/server/cel/view/results.go b/pkg/api/server/cel/view/results.go new file mode 100644 index 000000000..082af7e4c --- /dev/null +++ b/pkg/api/server/cel/view/results.go @@ -0,0 +1,66 @@ +package view + +import ( + "github.com/google/cel-go/cel" + "github.com/tektoncd/results/pkg/api/server/cel2sql" + resultspb "github.com/tektoncd/results/proto/v1alpha2/results_go_proto" +) + +var ( + typePipelineRun = "tekton.dev/v1beta1.PipelineRun" + typeTaskRun = "tekton.dev/v1beta1.TaskRun" + + typeConstants = map[string]cel2sql.Constant{ + "PIPELINE_RUN": { + StringVal: &typePipelineRun, + }, + "TASK_RUN": { + StringVal: &typeTaskRun, + }, + } +) + +// NewResultsView return the set of variables and constants available for CEL +// filters +func NewResultsView() (*cel2sql.View, error) { + view := &cel2sql.View{ + Constants: map[string]cel2sql.Constant{}, + Fields: map[string]cel2sql.Field{ + "parent": { + CELType: cel.StringType, + SQL: `parent`, + }, + "uid": { + CELType: cel.StringType, + SQL: `id`, + }, + "create_time": { + CELType: cel2sql.CELTypeTimestamp, + SQL: `created_time`, + }, + "update_time": { + CELType: cel2sql.CELTypeTimestamp, + SQL: `updated_time`, + }, + "annotations": { + CELType: cel.MapType(cel.StringType, cel.StringType), + SQL: `annotations`, + }, + "summary": { + CELType: cel.ObjectType("tekton.results.v1alpha2.RecordSummary"), + ObjectType: &resultspb.RecordSummary{}, + SQL: `recordsummary_{{.Field}}`, + }, + }, + } + for typeName, value := range typeConstants { + view.Constants[typeName] = value + } + for name, value := range resultspb.RecordSummary_Status_value { + v := value + view.Constants[name] = cel2sql.Constant{ + Int32Val: &v, + } + } + return view, nil +} diff --git a/pkg/api/server/cel/view/results_test.go b/pkg/api/server/cel/view/results_test.go new file mode 100644 index 000000000..ef6458871 --- /dev/null +++ b/pkg/api/server/cel/view/results_test.go @@ -0,0 +1,29 @@ +package view + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/tektoncd/results/pkg/api/server/cel2sql" + "k8s.io/utils/pointer" +) + +func TestNewResultsViewConstants(t *testing.T) { + view, err := NewResultsView() + if err != nil { + t.Fatal(err) + } + + expectedConstants := map[string]cel2sql.Constant{ + "PIPELINE_RUN": {StringVal: pointer.String("tekton.dev/v1beta1.PipelineRun")}, + "TASK_RUN": {StringVal: pointer.String("tekton.dev/v1beta1.TaskRun")}, + "UNKNOWN": {Int32Val: pointer.Int32(0)}, + "SUCCESS": {Int32Val: pointer.Int32(1)}, + "FAILURE": {Int32Val: pointer.Int32(2)}, + "TIMEOUT": {Int32Val: pointer.Int32(3)}, + "CANCELLED": {Int32Val: pointer.Int32(4)}, + } + if diff := cmp.Diff(expectedConstants, view.Constants); diff != "" { + t.Errorf("Invalid constants (-want, +got):\n%s", diff) + } +} diff --git a/pkg/api/server/cel2sql/convert.go b/pkg/api/server/cel2sql/convert.go index 2af41b42a..8b0571f14 100644 --- a/pkg/api/server/cel2sql/convert.go +++ b/pkg/api/server/cel2sql/convert.go @@ -20,9 +20,14 @@ import ( "github.com/google/cel-go/cel" ) -// Convert takes CEL expressions and attempt to convert them into Postgres SQL -// filters. -func Convert(env *cel.Env, filters string) (string, error) { +// Convert takes a View and CEL expressions and attempt to convert them into +// Postgres SQL filters. +func Convert(view *View, filters string) (string, error) { + env, err := view.GetEnv() + if err != nil { + return "", fmt.Errorf("invalid view: %w", err) + } + ast, issues := env.Compile(filters) if issues != nil && issues.Err() != nil { return "", fmt.Errorf("error compiling CEL filters: %w", issues.Err()) @@ -32,10 +37,15 @@ func Convert(env *cel.Env, filters string) (string, error) { return "", fmt.Errorf("expected boolean expression, but got %s", outputType.String()) } - interpreter, err := newInterpreter(ast) + interpreter, err := newInterpreter(ast, view) if err != nil { return "", fmt.Errorf("error creating cel2sql interpreter: %w", err) } - return interpreter.interpret() + sql, err := interpreter.interpret() + if err != nil { + return "", err + } + + return sql, nil } diff --git a/pkg/api/server/cel2sql/convert_test.go b/pkg/api/server/cel2sql/convert_test.go index cdc21638f..04d0a3717 100644 --- a/pkg/api/server/cel2sql/convert_test.go +++ b/pkg/api/server/cel2sql/convert_test.go @@ -18,234 +18,237 @@ import ( "errors" "testing" - "github.com/tektoncd/results/pkg/api/server/cel" - + "github.com/google/cel-go/cel" "github.com/google/go-cmp/cmp" + resultspb "github.com/tektoncd/results/proto/v1alpha2/results_go_proto" ) -func TestConvertRecordExpressions(t *testing.T) { +func newTestView() *View { + typePipelineRun := "tekton.dev/v1beta1.PipelineRun" + return &View{ + Constants: map[string]Constant{ + "PIPELINE_RUN": { + StringVal: &typePipelineRun, + }, + }, + Fields: map[string]Field{ + "parent": { + CELType: cel.StringType, + SQL: `parent`, + }, + "create_time": { + CELType: CELTypeTimestamp, + SQL: `created_time`, + }, + "annotations": { + CELType: cel.MapType(cel.StringType, cel.StringType), + SQL: `annotations`, + }, + "summary": { + CELType: cel.ObjectType("tekton.results.v1alpha2.RecordSummary"), + ObjectType: &resultspb.RecordSummary{}, + SQL: `recordsummary_{{.Field}}`, + }, + "data": { + CELType: cel.AnyType, + SQL: `data`, + }, + }, + } +} +func TestConversionErrors(t *testing.T) { tests := []struct { name string in string - want string + want error }{ { - name: "simple expression", - in: `name == "foo"`, - want: "name = 'foo'", - }, - { - name: "select expression", - in: `data.metadata.namespace == "default"`, - want: "(data->'metadata'->>'namespace') = 'default'", - }, - { - name: "type coercion with a dyn expression in the left hand side", - in: `data.status.completionTime > timestamp("2022/10/30T21:45:00.000Z")`, - want: "(data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE > '2022/10/30T21:45:00.000Z'::TIMESTAMP WITH TIME ZONE", - }, - { - name: "type coercion with a dyn expression in the right hand side", - in: `timestamp("2022/10/30T21:45:00.000Z") < data.status.completionTime`, - want: "'2022/10/30T21:45:00.000Z'::TIMESTAMP WITH TIME ZONE < (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE", + name: "compile error missing field", + in: "parnt", + want: errors.New("error compiling CEL filters: ERROR: :1:1: undeclared reference to 'parnt' (in container '')\n | parnt\n | ^"), }, { - name: "in operator", - in: `data.metadata.namespace in ["foo", "bar"]`, - want: "(data->'metadata'->>'namespace') IN ('foo', 'bar')", - }, - { - name: "index operator", - in: `data.metadata.labels["foo"] == "bar"`, - want: "(data->'metadata'->'labels'->>'foo') = 'bar'", - }, - { - name: "concatenate strings", - in: `name + "bar" == "foobar"`, - want: "CONCAT(name, 'bar') = 'foobar'", - }, - { - name: "multiple concatenate strings", - in: `name + "bar" + "baz" == "foobarbaz"`, - want: "CONCAT(name, 'bar', 'baz') = 'foobarbaz'", + name: "non-boolean expression", + in: "parent", + want: errors.New("expected boolean expression, but got string"), }, + } + + view := newTestView() + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := Convert(view, test.in) + if err == nil { + t.Fatalf("Want the %q error, but the interpreter returned the following result instead: %q", test.want.Error(), got) + } + + if diff := cmp.Diff(test.want.Error(), err.Error()); diff != "" { + t.Fatalf("Mismatch in the error returned by the Convert function (-want +got):\n%s", diff) + } + }) + } +} + +func TestConvert(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + // About operations { - name: "contains string function", - in: `data.metadata.name.contains("foo")`, - want: "POSITION('foo' IN (data->'metadata'->>'name')) <> 0", + name: "able to match strings exactly", + in: `parent == "foo"`, + want: "parent = 'foo'", }, { - name: "endsWith string function", - in: `data.metadata.name.endsWith("bar")`, - want: "(data->'metadata'->>'name') LIKE '%' || 'bar'", + name: "able to use endsWith function", + in: `parent.endsWith("bar")`, + want: "parent LIKE '%' || 'bar'", }, { - name: "getDate function", - in: `data.status.completionTime.getDate() == 2`, - want: "EXTRACT(DAY FROM (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE) = 2", + name: "able to use in operator", + in: `parent in ["foo", "bar"]`, + want: "parent IN ('foo', 'bar')", }, { - name: "getDayOfMonth function", - in: `data.status.completionTime.getDayOfMonth() == 2`, - want: "(EXTRACT(DAY FROM (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE) - 1) = 2", + name: "able to select values from any type", + in: `data.metadata.namespace == "default"`, + want: "(data->'metadata'->>'namespace') = 'default'", }, { - name: "getDayOfWeek function", - in: `data.status.completionTime.getDayOfWeek() > 0`, - want: "EXTRACT(DOW FROM (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE) > 0", + name: "able to use index operator to navigate maps", + in: `data.metadata.labels["foo"] == "bar"`, + want: "(data->'metadata'->'labels'->>'foo') = 'bar'", }, { - name: "getDayOfYear function", - in: `data.status.completionTime.getDayOfYear() > 15`, - want: "(EXTRACT(DOY FROM (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE) - 1) > 15", + name: "able to use index operator to navigate arrays", + in: `data.metadata.ownerReferences[0].name == "bar"`, + want: "(data->'metadata'->'ownerReferences'->0->>'name') = 'bar'", }, { - name: "getFullYear function", - in: `data.status.completionTime.getFullYear() >= 2022`, - want: "EXTRACT(YEAR FROM (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE) >= 2022", + name: "able to use index operator in first selection in any type", + in: `data["status"].conditions[0].status == "True"`, + want: "(data->'status'->'conditions'->0->>'status') = 'True'", }, { - name: "matches function", - in: `data.metadata.name.matches("^foo.*$")`, - want: "(data->'metadata'->>'name') ~ '^foo.*$'", + name: "able to chain index operators", + in: `data.status["conditions"][0].status == "True"`, + want: "(data->'status'->'conditions'->0->>'status') = 'True'", }, { - name: "startsWith string function", - in: `data.metadata.name.startsWith("bar")`, - want: "(data->'metadata'->>'name') LIKE 'bar' || '%'", + name: "able to concatenate strings", + in: `parent + "bar" == "foobar"`, + want: "CONCAT(parent, 'bar') = 'foobar'", }, { - name: "data_type field", - in: `data_type == PIPELINE_RUN`, - want: "type = 'tekton.dev/v1beta1.PipelineRun'", + name: "able to concatenate multiple concatenate strings", + in: `parent + "bar" + "baz" == "foobarbaz"`, + want: "CONCAT(parent, 'bar', 'baz') = 'foobarbaz'", }, { - name: "index operator with numeric argument in JSON arrays", - in: `data_type == "tekton.dev/v1beta1.TaskRun" && data.status.conditions[0].status == "True"`, - want: "type = 'tekton.dev/v1beta1.TaskRun' AND (data->'status'->'conditions'->0->>'status') = 'True'", + name: "able to use contains string function", + in: `parent.contains("foo")`, + want: "POSITION('foo' IN parent) <> 0", }, { - name: "index operator as first operation in JSON object", - in: `data_type == "tekton.dev/v1beta1.TaskRun" && data["status"].conditions[0].status == "True"`, - want: "type = 'tekton.dev/v1beta1.TaskRun' AND (data->'status'->'conditions'->0->>'status') = 'True'", + name: "able to use matches function", + in: `parent.matches("^foo.*$")`, + want: "parent ~ '^foo.*$'", }, + // About maps { - name: "index operator with string argument in JSON object", - in: `data_type == "tekton.dev/v1beta1.TaskRun" && data.status["conditions"][0].status == "True"`, - want: "type = 'tekton.dev/v1beta1.TaskRun' AND (data->'status'->'conditions'->0->>'status') = 'True'", - }, - } - - env, err := cel.NewRecordsEnv() - if err != nil { - t.Fatal(err) - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - got, err := Convert(env, test.in) - if err != nil { - t.Fatal(err) - } - t.Logf("want: %+v\n", test.want) - t.Logf("got: %+v\n", got) - if diff := cmp.Diff(test.want, got); diff != "" { - t.Errorf("Mismatch (-want +got):\n%s", diff) - } - }) - } -} - -func TestConvertResultExpressions(t *testing.T) { - tests := []struct { - name string - in string - want string - }{{ - name: "Result.Parent field", - in: `parent.endsWith("bar")`, - want: "parent LIKE '%' || 'bar'", - }, - { - name: "Result.Uid field", - in: `uid == "foo"`, - want: "id = 'foo'", - }, - { - name: "Result.Annotations field", + name: "able to match map of strings in left hand side", in: `annotations["repo"] == "tektoncd/results"`, want: `annotations @> '{"repo":"tektoncd/results"}'::jsonb`, }, { - name: "Result.Annotations field", + name: "able to match map of strings in right hand side", in: `"tektoncd/results" == annotations["repo"]`, want: `annotations @> '{"repo":"tektoncd/results"}'::jsonb`, }, { - name: "other operators involving the Result.Annotations field", + name: "able to use functions after accessing map values", in: `annotations["repo"].startsWith("tektoncd")`, want: "annotations->>'repo' LIKE 'tektoncd' || '%'", }, + // About timestamp { - name: "Result.CreateTime field", + name: "able to filter timestamp", in: `create_time > timestamp("2022/10/30T21:45:00.000Z")`, want: "created_time > '2022/10/30T21:45:00.000Z'::TIMESTAMP WITH TIME ZONE", }, { - name: "Result.UpdateTime field", - in: `update_time > timestamp("2022/10/30T21:45:00.000Z")`, - want: "updated_time > '2022/10/30T21:45:00.000Z'::TIMESTAMP WITH TIME ZONE", + name: "able to perform type coercion with a dyn expression in the left hand side", + in: `data.status.completionTime > timestamp("2022/10/30T21:45:00.000Z")`, + want: "(data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE > '2022/10/30T21:45:00.000Z'::TIMESTAMP WITH TIME ZONE", + }, + { + name: "able to perform type coercion with a dyn expression in the right hand side", + in: `timestamp("2022/10/30T21:45:00.000Z") < data.status.completionTime`, + want: "'2022/10/30T21:45:00.000Z'::TIMESTAMP WITH TIME ZONE < (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE", }, { - name: "Result.Summary.Record field", - in: `summary.record == "foo/results/bar/records/baz"`, - want: "recordsummary_record = 'foo/results/bar/records/baz'", + name: "able to use getDate function", + in: `data.status.completionTime.getDate() == 2`, + want: "EXTRACT(DAY FROM (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE) = 2", }, { - name: "Result.Summary.StartTime field", - in: `summary.start_time > timestamp("2022/10/30T21:45:00.000Z")`, - want: "recordsummary_start_time > '2022/10/30T21:45:00.000Z'::TIMESTAMP WITH TIME ZONE", + name: "able to use getDayOfMonth function", + in: `data.status.completionTime.getDayOfMonth() == 2`, + want: "(EXTRACT(DAY FROM (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE) - 1) = 2", }, { - name: "comparison with the PIPELINE_RUN const value", - in: `summary.type == PIPELINE_RUN`, - want: "recordsummary_type = 'tekton.dev/v1beta1.PipelineRun'", + name: "able to use getDayOfWeek function", + in: `data.status.completionTime.getDayOfWeek() > 0`, + want: "EXTRACT(DOW FROM (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE) > 0", }, { - name: "comparison with the TASK_RUN const value", - in: `summary.type == TASK_RUN`, - want: "recordsummary_type = 'tekton.dev/v1beta1.TaskRun'", + name: "able to use getDayOfYear function", + in: `data.status.completionTime.getDayOfYear() > 15`, + want: "(EXTRACT(DOY FROM (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE) - 1) > 15", }, { - name: "RecordSummary_Status constants", - in: `summary.status == CANCELLED || summary.status == TIMEOUT`, - want: "recordsummary_status = 4 OR recordsummary_status = 3", + name: "able to use getFullYear function", + in: `data.status.completionTime.getFullYear() >= 2022`, + want: "EXTRACT(YEAR FROM (data->'status'->>'completionTime')::TIMESTAMP WITH TIME ZONE) >= 2022", }, + // About objects { - name: "Result.Summary.Annotations", + name: "able to map object fields to columns", + in: `summary.record == "foo/results/bar/baz"`, + want: "recordsummary_record = 'foo/results/bar/baz'", + }, + // About constants + { + name: "able to compare with const value", + in: `summary.type == PIPELINE_RUN`, + want: "recordsummary_type = 'tekton.dev/v1beta1.PipelineRun'", + }, + // About compatibility of features + { + name: "able to use map of strings inside objects in the left hand side", in: `summary.annotations["branch"] == "main"`, want: `recordsummary_annotations @> '{"branch":"main"}'::jsonb`, }, { - name: "Result.Summary.Annotations", + name: "able to use map of strings inside objects in the right hand side", in: `"main" == summary.annotations["branch"]`, want: `recordsummary_annotations @> '{"branch":"main"}'::jsonb`, }, { name: "more complex expression", - in: `summary.annotations["actor"] == "john-doe" && summary.annotations["branch"] == "feat/amazing" && summary.status == SUCCESS`, - want: `recordsummary_annotations @> '{"actor":"john-doe"}'::jsonb AND recordsummary_annotations @> '{"branch":"feat/amazing"}'::jsonb AND recordsummary_status = 1`, + in: `summary.annotations["actor"] == "john-doe" && summary.annotations["branch"] == "feat/amazing" && summary.type == PIPELINE_RUN`, + want: `recordsummary_annotations @> '{"actor":"john-doe"}'::jsonb AND recordsummary_annotations @> '{"branch":"feat/amazing"}'::jsonb AND recordsummary_type = 'tekton.dev/v1beta1.PipelineRun'`, }, } - env, err := cel.NewResultsEnv() - if err != nil { - t.Fatal(err) - } + view := newTestView() for _, test := range tests { t.Run(test.name, func(t *testing.T) { - got, err := Convert(env, test.in) + got, err := Convert(view, test.in) if err != nil { t.Fatal(err) } @@ -256,34 +259,3 @@ func TestConvertResultExpressions(t *testing.T) { }) } } - -func TestConversionErrors(t *testing.T) { - tests := []struct { - name string - in string - want error - }{{ - name: "non-boolean expression", - in: "parent", - want: errors.New("expected boolean expression, but got string"), - }, - } - - env, err := cel.NewResultsEnv() - if err != nil { - t.Fatal(err) - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - got, err := Convert(env, test.in) - if err == nil { - t.Fatalf("Want the %q error, but the interpreter returned the following result instead: %q", test.want.Error(), got) - } - - if diff := cmp.Diff(test.want.Error(), err.Error()); diff != "" { - t.Fatalf("Mismatch in the error returned by the Convert function (-want +got):\n%s", diff) - } - }) - } -} diff --git a/pkg/api/server/cel2sql/doc.go b/pkg/api/server/cel2sql/doc.go new file mode 100644 index 000000000..20b6ba00b --- /dev/null +++ b/pkg/api/server/cel2sql/doc.go @@ -0,0 +1,3 @@ +// package cel2sql provides a generic interface to transform CEL expression into +// SQL where clauses. +package cel2sql diff --git a/pkg/api/server/cel2sql/interpreter.go b/pkg/api/server/cel2sql/interpreter.go index ccb5d06a2..b452dcb5a 100644 --- a/pkg/api/server/cel2sql/interpreter.go +++ b/pkg/api/server/cel2sql/interpreter.go @@ -36,19 +36,21 @@ var ErrUnsupportedExpression = errors.New("unsupported CEL") // filters in the Postgres dialect. type interpreter struct { checkedExpr *exprpb.CheckedExpr + view *View query strings.Builder } // newInterpreter takes an abstract syntax tree and returns an Interpreter object capable // of converting it to a set of SQL filters. -func newInterpreter(ast *cel.Ast) (*interpreter, error) { +func newInterpreter(ast *cel.Ast, view *View) (*interpreter, error) { checkedExpr, err := cel.AstToCheckedExpr(ast) if err != nil { return nil, err } return &interpreter{ checkedExpr: checkedExpr, + view: view, }, nil } @@ -142,22 +144,18 @@ func (i *interpreter) interpretConstExpr(id int64, expr *exprpb.Constant) error return nil } -var identToColumn = map[string]string{ - "uid": "id", - "create_time": "created_time", - "update_time": "updated_time", - "data_type": "type", -} - func (i *interpreter) interpretIdentExpr(id int64, expr *exprpb.Expr_IdentExpr) error { if reference, found := i.checkedExpr.ReferenceMap[id]; found && reference.GetValue() != nil { return i.interpretConstExpr(id, reference.GetValue()) } name := expr.IdentExpr.GetName() - if column, found := identToColumn[name]; found { - name = column + + field, found := i.view.Fields[name] + if !found { + return i.unsupportedExprError(id, fmt.Sprintf("unexpected field %q", name)) } - i.query.WriteString(name) + + i.query.WriteString(field.SQL) return nil } @@ -194,8 +192,9 @@ func (i *interpreter) getIndexKey(expr *exprpb.Expr) (fmt.Stringer, error) { } } -func (i *interpreter) getSelectFields(expr *exprpb.Expr) ([]fmt.Stringer, error) { +func (i *interpreter) getSelectFields(expr *exprpb.Expr) ([]fmt.Stringer, *Field, error) { var target *exprpb.Expr + var identField *Field fields := []fmt.Stringer{} switch node := expr.ExprKind.(type) { case *exprpb.Expr_SelectExpr: @@ -205,38 +204,45 @@ func (i *interpreter) getSelectFields(expr *exprpb.Expr) ([]fmt.Stringer, error) case *exprpb.Expr_CallExpr: if !isIndexExpr(expr) { // TODO: return which function is not supported - return nil, i.unsupportedExprError(expr.Id, "function") + return nil, identField, i.unsupportedExprError(expr.Id, "function") } // Sanity check, index function should always have two arguments if len(node.CallExpr.Args) != 2 { - return nil, ErrUnsupportedExpression + return nil, identField, ErrUnsupportedExpression } target = node.CallExpr.Args[0] index, err := i.getIndexKey(expr) if err != nil { - return nil, err + return nil, identField, err } fields = append(fields, index) case *exprpb.Expr_IdentExpr: - fields = append(fields, &Unquoted{node.IdentExpr.GetName()}) + name := node.IdentExpr.GetName() + field, found := i.view.Fields[name] + if !found { + return fields, identField, fmt.Errorf("unexpected field: %q", name) + } + fields = append(fields, &Unquoted{field.SQL}) + identField = &field target = nil default: - return nil, ErrUnsupportedExpression + return nil, identField, ErrUnsupportedExpression } if target != nil { - newFields, err := i.getSelectFields(target) + newFields, field, err := i.getSelectFields(target) if err != nil { - return nil, err + return nil, nil, err } + identField = field fields = append(fields, newFields...) } - return fields, nil + return fields, identField, nil } func (i *interpreter) interpretSelectExpr(id int64, expr *exprpb.Expr_SelectExpr, additionalExprs ...*exprpb.Expr) error { - fields, err := i.getSelectFields(&exprpb.Expr{Id: id, ExprKind: expr}) + fields, identField, err := i.getSelectFields(&exprpb.Expr{Id: id, ExprKind: expr}) if err != nil { return err } @@ -256,13 +262,12 @@ func (i *interpreter) interpretSelectExpr(id int64, expr *exprpb.Expr_SelectExpr } } - if i.isDyn(expr.SelectExpr.GetOperand()) { - i.translateToJSONAccessors(reversedFields) - return nil + if identField.ObjectType != nil { + return i.translateIntoStruct(reversedFields) } - if i.isRecordSummary(expr.SelectExpr.GetOperand()) { - i.translateToRecordSummaryColumn(reversedFields) + if i.isDyn(expr.SelectExpr.GetOperand()) { + i.translateToJSONAccessors(reversedFields) return nil } diff --git a/pkg/api/server/cel2sql/select.go b/pkg/api/server/cel2sql/select.go index 571b49468..8efb2381a 100644 --- a/pkg/api/server/cel2sql/select.go +++ b/pkg/api/server/cel2sql/select.go @@ -15,7 +15,9 @@ package cel2sql import ( + "bytes" "fmt" + "text/template" "gorm.io/gorm/schema" ) @@ -35,13 +37,32 @@ func (i *interpreter) translateToJSONAccessors(fieldPath []fmt.Stringer) { fmt.Fprintf(&i.query, ")") } -// translateToRecordSummaryColumn -func (i *interpreter) translateToRecordSummaryColumn(fieldPath []fmt.Stringer) { - namer := &schema.NamingStrategy{} - switch f := fieldPath[1].(type) { +func getRawString(s fmt.Stringer) string { + switch f := s.(type) { case *Unquoted: - fmt.Fprintf(&i.query, "recordsummary_%s", namer.ColumnName("", f.s)) + return f.s case *SingleQuoted: - fmt.Fprintf(&i.query, "recordsummary_%s", namer.ColumnName("", f.s)) + return f.s + } + return s.String() +} + +// translateIntoStruct +func (i *interpreter) translateIntoStruct(fieldPath []fmt.Stringer) error { + namer := &schema.NamingStrategy{} + rawSql := getRawString(fieldPath[0]) + rawField := getRawString(fieldPath[1]) + sqlTemplate, err := template.New("").Parse(rawSql) + if err != nil { + return err + } + var sql bytes.Buffer + err = sqlTemplate.Execute(&sql, map[string]string{ + "Field": namer.ColumnName("", rawField), + }) + if err != nil { + return err } + _, err = fmt.Fprintf(&i.query, "%s", sql.String()) + return err } diff --git a/pkg/api/server/cel2sql/type_coercion.go b/pkg/api/server/cel2sql/type_coercion.go index eb387a16b..a0e46b231 100644 --- a/pkg/api/server/cel2sql/type_coercion.go +++ b/pkg/api/server/cel2sql/type_coercion.go @@ -40,15 +40,6 @@ func (i *interpreter) isString(expr *exprpb.Expr) bool { return false } -func (i *interpreter) isRecordSummary(expr *exprpb.Expr) bool { - if theType, found := i.checkedExpr.TypeMap[expr.GetId()]; found { - if messageType := theType.GetMessageType(); messageType == "tekton.results.v1alpha2.RecordSummary" { - return true - } - } - return false -} - // coerceToTypeOf writes a Postgres cast directive to the current position of // the SQL statement in the buffer, in order to cast the current SQL expression // to the SQL type of the provided CEL expression. This feature provides diff --git a/pkg/api/server/cel2sql/view_types.go b/pkg/api/server/cel2sql/view_types.go new file mode 100644 index 000000000..186f3b85f --- /dev/null +++ b/pkg/api/server/cel2sql/view_types.go @@ -0,0 +1,109 @@ +package cel2sql + +import ( + "github.com/google/cel-go/cel" + "github.com/google/cel-go/checker/decls" + exprpb "google.golang.org/genproto/googleapis/api/expr/v1alpha1" + "google.golang.org/protobuf/types/known/timestamppb" +) + +var ( + CELTypeTimestamp = cel.ObjectType("google.protobuf.Timestamp") +) + +// Field is the configuration that allows mapping between CEL variables and +// SQL queries +type Field struct { + // CELType is an enum to express how the field is going to be used in CEL + CELType *cel.Type + // SQL is a fragment that can be used to select this field in a given database + SQL string + // ObjectType is a special kind of type that allows CEL field selection map + // to different columns + ObjectType any +} + +// Constant respresent a value +type Constant struct { + StringVal *string + Int32Val *int32 +} + +// View represents a set of variables accessible by the CEL expression +type View struct { + // Fields is a map of variable names and Field configuration + Fields map[string]Field + // Constants is a map of constant names and its values + Constants map[string]Constant +} + +// protoType return the protobuf type equivalent of the Constant +func (c *Constant) protoType() *exprpb.Type { + if c.StringVal != nil { + return decls.String + } + if c.Int32Val != nil { + return decls.Int + } + return decls.Dyn +} + +// protoConstant adapts the Constant to protobuf types +func (c *Constant) protoConstant() *exprpb.Constant { + if c.StringVal != nil { + return &exprpb.Constant{ + ConstantKind: &exprpb.Constant_StringValue{StringValue: *c.StringVal}, + } + } + + if c.Int32Val != nil { + return &exprpb.Constant{ + ConstantKind: &exprpb.Constant_Int64Value{Int64Value: int64(*c.Int32Val)}, + } + } + + return nil +} + +// GetEnv generates a new CEL environment with all variables, constants and +// types available +func (v *View) GetEnv() (*cel.Env, error) { + return cel.NewEnv( + cel.Declarations(v.celConstants()...), + cel.Types(v.celTypes()...), + cel.Declarations(v.celVariables()...), + ) +} + +// celConstants gets all protobuf declarations of constants for a given View +func (v *View) celConstants() []*exprpb.Decl { + constants := make([]*exprpb.Decl, 0, len(v.Constants)) + for name, value := range v.Constants { + constants = append(constants, decls.NewConst(name, value.protoType(), value.protoConstant())) + } + return constants +} + +// celTypes returns all custom types used in the View +func (v *View) celTypes() []any { + types := []any{×tamppb.Timestamp{}} + for _, field := range v.Fields { + if field.ObjectType != nil { + types = append(types, field.ObjectType) + } + } + return types +} + +// celVariables returns all variables protobuf declarations +func (v *View) celVariables() []*exprpb.Decl { + vars := []*exprpb.Decl{} + for name, field := range v.Fields { + exprType, err := cel.TypeToExprType(field.CELType) + if err != nil { + panic("unexpected field type in view") + } + vars = append(vars, decls.NewVar(name, exprType)) + } + return vars +} diff --git a/pkg/api/server/v1alpha2/lister/filter.go b/pkg/api/server/v1alpha2/lister/filter.go index 7bf77fd4c..17cc5be33 100644 --- a/pkg/api/server/v1alpha2/lister/filter.go +++ b/pkg/api/server/v1alpha2/lister/filter.go @@ -20,13 +20,12 @@ import ( pagetokenpb "github.com/tektoncd/results/pkg/api/server/v1alpha2/lister/proto/pagetoken_go_proto" - "github.com/google/cel-go/cel" "github.com/tektoncd/results/pkg/api/server/cel2sql" "gorm.io/gorm" ) type filter struct { - env *cel.Env + view *cel2sql.View expr string equalityClauses []equalityClause } @@ -57,7 +56,7 @@ func (f *filter) build(db *gorm.DB) (*gorm.DB, error) { } if expr := strings.TrimSpace(f.expr); expr != "" { - sql, err := cel2sql.Convert(f.env, expr) + sql, err := cel2sql.Convert(f.view, expr) if err != nil { return nil, err } diff --git a/pkg/api/server/v1alpha2/lister/filter_test.go b/pkg/api/server/v1alpha2/lister/filter_test.go index 4aae9b96b..ca0a6a3f0 100644 --- a/pkg/api/server/v1alpha2/lister/filter_test.go +++ b/pkg/api/server/v1alpha2/lister/filter_test.go @@ -21,7 +21,7 @@ import ( pagetokenpb "github.com/tektoncd/results/pkg/api/server/v1alpha2/lister/proto/pagetoken_go_proto" - "github.com/tektoncd/results/pkg/api/server/cel" + celview "github.com/tektoncd/results/pkg/api/server/cel/view" "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -45,15 +45,15 @@ func TestFilterValidateToken(t *testing.T) { } func TestFilterBuild(t *testing.T) { - env, err := cel.NewResultsEnv() + db, _ := gorm.Open(tests.DummyDialector{}) + statement := &gorm.Statement{DB: db, Table: "testtable", Clauses: map[string]clause.Clause{}} + db.Statement = statement + + view, err := celview.NewResultsView() if err != nil { t.Fatal(err) } - db, _ := gorm.Open(tests.DummyDialector{}) - statement := &gorm.Statement{DB: db, Clauses: map[string]clause.Clause{}} - db.Statement = statement - t.Run("no where clause", func(t *testing.T) { filter := &filter{} testDB, err := filter.build(db) @@ -86,7 +86,7 @@ func TestFilterBuild(t *testing.T) { t.Run("where clause with parent and id", func(t *testing.T) { filter := &filter{ - env: env, + view: view, equalityClauses: []equalityClause{ {columnName: "parent", value: "foo"}, {columnName: "id", value: "bar"}, @@ -108,7 +108,7 @@ func TestFilterBuild(t *testing.T) { t.Run("where clause with cel2sql filters", func(t *testing.T) { filter := &filter{ - env: env, + view: view, expr: `summary.status == SUCCESS`, } @@ -127,7 +127,7 @@ func TestFilterBuild(t *testing.T) { t.Run("more complex filter", func(t *testing.T) { filter := &filter{ - env: env, + view: view, equalityClauses: []equalityClause{ {columnName: "parent", value: "foo"}, {columnName: "id", value: "bar"}, diff --git a/pkg/api/server/v1alpha2/lister/lister.go b/pkg/api/server/v1alpha2/lister/lister.go index 309286623..203d3b522 100644 --- a/pkg/api/server/v1alpha2/lister/lister.go +++ b/pkg/api/server/v1alpha2/lister/lister.go @@ -20,7 +20,7 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" - "github.com/google/cel-go/cel" + "github.com/tektoncd/results/pkg/api/server/cel2sql" "github.com/tektoncd/results/pkg/api/server/db" "github.com/tektoncd/results/pkg/api/server/db/errors" pagetokenpb "github.com/tektoncd/results/pkg/api/server/v1alpha2/lister/proto/pagetoken_go_proto" @@ -131,14 +131,14 @@ func (l *Lister[M, W]) List(ctx context.Context, db *gorm.DB) ([]W, string, erro } // OfResults creates a Lister for Result objects. -func OfResults(env *cel.Env, request *resultspb.ListResultsRequest) (*Lister[*db.Result, *resultspb.Result], error) { - return newLister(env, resultFieldsToColumns, request, result.ToAPI, equalityClause{ +func OfResults(view *cel2sql.View, request *resultspb.ListResultsRequest) (*Lister[*db.Result, *resultspb.Result], error) { + return newLister(view, resultFieldsToColumns, request, result.ToAPI, equalityClause{ columnName: "parent", value: strings.TrimSpace(request.GetParent()), }) } -func newLister[M any, W wireObject](env *cel.Env, fieldsToColumns map[string]string, listObjectsRequest request, convert Converter[M, W], clauses ...equalityClause) (*Lister[M, W], error) { +func newLister[M any, W wireObject](view *cel2sql.View, fieldsToColumns map[string]string, listObjectsRequest request, convert Converter[M, W], clauses ...equalityClause) (*Lister[M, W], error) { pageToken, err := decodePageToken(strings.TrimSpace(listObjectsRequest.GetPageToken())) if err != nil { return nil, err @@ -155,7 +155,7 @@ func newLister[M any, W wireObject](env *cel.Env, fieldsToColumns map[string]str } filter := &filter{ - env: env, + view: view, expr: strings.TrimSpace(listObjectsRequest.GetFilter()), equalityClauses: clauses, } @@ -227,8 +227,8 @@ func getTimestamp(in wireObject, fieldName string) (timestamp *timestamppb.Times } // OfRecords creates a Lister for Record objects. -func OfRecords(env *cel.Env, resultParent, resultName string, request *resultspb.ListRecordsRequest) (*Lister[*db.Record, *resultspb.Record], error) { - return newLister(env, recordFieldsToColumns, request, record.ToAPI, equalityClause{ +func OfRecords(view *cel2sql.View, resultParent, resultName string, request *resultspb.ListRecordsRequest) (*Lister[*db.Record, *resultspb.Record], error) { + return newLister(view, recordFieldsToColumns, request, record.ToAPI, equalityClause{ columnName: "parent", value: resultParent, }, diff --git a/pkg/api/server/v1alpha2/lister/lister_test.go b/pkg/api/server/v1alpha2/lister/lister_test.go index f16987f35..7bbedd670 100644 --- a/pkg/api/server/v1alpha2/lister/lister_test.go +++ b/pkg/api/server/v1alpha2/lister/lister_test.go @@ -23,7 +23,7 @@ import ( resultspb "github.com/tektoncd/results/proto/v1alpha2/results_go_proto" "github.com/google/go-cmp/cmp" - "github.com/tektoncd/results/pkg/api/server/cel" + celview "github.com/tektoncd/results/pkg/api/server/cel/view" pagetokenpb "github.com/tektoncd/results/pkg/api/server/v1alpha2/lister/proto/pagetoken_go_proto" "google.golang.org/protobuf/types/known/timestamppb" "gorm.io/gorm" @@ -32,15 +32,15 @@ import ( ) func TestBuildQuery(t *testing.T) { - env, err := cel.NewResultsEnv() + db, _ := gorm.Open(tests.DummyDialector{}) + statement := &gorm.Statement{DB: db, Table: "testtable", Clauses: map[string]clause.Clause{}} + db.Statement = statement + + view, err := celview.NewResultsView() if err != nil { t.Fatal(err) } - db, _ := gorm.Open(tests.DummyDialector{}) - statement := &gorm.Statement{DB: db, Clauses: map[string]clause.Clause{}} - db.Statement = statement - now := time.Now() order := &order{ @@ -67,7 +67,7 @@ func TestBuildQuery(t *testing.T) { pageToken: token, }, &filter{ - env: env, + view: view, equalityClauses: []equalityClause{{ columnName: "parent", value: "foo", diff --git a/pkg/api/server/v1alpha2/records.go b/pkg/api/server/v1alpha2/records.go index 1218488c0..a2d2c710e 100644 --- a/pkg/api/server/v1alpha2/records.go +++ b/pkg/api/server/v1alpha2/records.go @@ -154,7 +154,7 @@ func (s *Server) ListRecords(ctx context.Context, req *pb.ListRecordsRequest) (* return nil, err } - recordsLister, err := lister.OfRecords(s.recordsEnv, parent, resultName, req) + recordsLister, err := lister.OfRecords(s.recordsView, parent, resultName, req) if err != nil { return nil, err } diff --git a/pkg/api/server/v1alpha2/results.go b/pkg/api/server/v1alpha2/results.go index 54f48f21a..aa724de18 100644 --- a/pkg/api/server/v1alpha2/results.go +++ b/pkg/api/server/v1alpha2/results.go @@ -185,7 +185,7 @@ func (s *Server) ListResults(ctx context.Context, req *pb.ListResultsRequest) (* return nil, err } - resultsLister, err := lister.OfResults(s.resultsEnv, req) + resultsLister, err := lister.OfResults(s.resultsView, req) if err != nil { return nil, err } diff --git a/pkg/api/server/v1alpha2/server.go b/pkg/api/server/v1alpha2/server.go index aaff2ae4e..8518b14a0 100644 --- a/pkg/api/server/v1alpha2/server.go +++ b/pkg/api/server/v1alpha2/server.go @@ -19,6 +19,8 @@ import ( "fmt" "github.com/google/cel-go/cel" + celview "github.com/tektoncd/results/pkg/api/server/cel/view" + "github.com/tektoncd/results/pkg/api/server/cel2sql" "github.com/tektoncd/results/pkg/api/server/config" "go.uber.org/zap" @@ -44,13 +46,13 @@ type getResultID func(ctx context.Context, parent, result string) (string, error type Server struct { pb.UnimplementedResultsServer pb.UnimplementedLogsServer - config *config.Config - logger *zap.SugaredLogger - env *cel.Env - resultsEnv *cel.Env - recordsEnv *cel.Env - db *gorm.DB - auth auth.Checker + config *config.Config + logger *zap.SugaredLogger + env *cel.Env + resultsView *cel2sql.View + recordsView *cel2sql.View + db *gorm.DB + auth auth.Checker // testing. getResultID getResultID @@ -63,22 +65,22 @@ func New(config *config.Config, logger *zap.SugaredLogger, db *gorm.DB, opts ... return nil, fmt.Errorf("failed to create CEL environment: %w", err) } // TODO: turn the func into a MustX that should panic on error. - resultsEnv, err := resultscel.NewResultsEnv() + resultsView, err := celview.NewResultsView() if err != nil { return nil, err } - recordsEnv, err := resultscel.NewRecordsEnv() + recordsView, err := celview.NewRecordsView() if err != nil { return nil, err } srv := &Server{ - db: db, - env: env, - resultsEnv: resultsEnv, - recordsEnv: recordsEnv, - config: config, - logger: logger, + db: db, + env: env, + resultsView: resultsView, + recordsView: recordsView, + config: config, + logger: logger, // Default open auth for easier testing. auth: auth.AllowAll{}, }