From e0083034dc032ddaa502a7962162f8f268fecc4c Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 3 Sep 2024 12:34:51 +0300 Subject: [PATCH 01/46] Source versioning initial implementation --- pkg/materialize/table_from_source.go | 156 +++++++++++++ pkg/provider/provider.go | 1 + pkg/resources/resource_source_postgres.go | 4 + pkg/resources/resource_table_from_source.go | 242 ++++++++++++++++++++ 4 files changed, 403 insertions(+) create mode 100644 pkg/materialize/table_from_source.go create mode 100644 pkg/resources/resource_table_from_source.go diff --git a/pkg/materialize/table_from_source.go b/pkg/materialize/table_from_source.go new file mode 100644 index 00000000..92f2146d --- /dev/null +++ b/pkg/materialize/table_from_source.go @@ -0,0 +1,156 @@ +package materialize + +import ( + "database/sql" + "fmt" + "strings" + + "github.com/jmoiron/sqlx" + "github.com/lib/pq" +) + +type TableFromSourceParams struct { + TableId sql.NullString `db:"id"` + TableName sql.NullString `db:"name"` + SchemaName sql.NullString `db:"schema_name"` + DatabaseName sql.NullString `db:"database_name"` + SourceName sql.NullString `db:"source_name"` + SourceSchemaName sql.NullString `db:"source_schema_name"` + SourceDatabaseName sql.NullString `db:"source_database_name"` + UpstreamName sql.NullString `db:"upstream_name"` + UpstreamSchemaName sql.NullString `db:"upstream_schema_name"` + TextColumns pq.StringArray `db:"text_columns"` + Comment sql.NullString `db:"comment"` + OwnerName sql.NullString `db:"owner_name"` + Privileges pq.StringArray `db:"privileges"` +} + +// TODO: Extend this query to include the upstream table name and schema name and the source +var tableFromSourceQuery = NewBaseQuery(` + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN ( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + ) comments + ON mz_tables.id = comments.id +`) + +func TableFromSourceId(conn *sqlx.DB, obj MaterializeObject) (string, error) { + p := map[string]string{ + "mz_tables.name": obj.Name, + "mz_schemas.name": obj.SchemaName, + "mz_databases.name": obj.DatabaseName, + } + q := tableFromSourceQuery.QueryPredicate(p) + + var t TableFromSourceParams + if err := conn.Get(&t, q); err != nil { + return "", err + } + + return t.TableId.String, nil +} + +func ScanTableFromSource(conn *sqlx.DB, id string) (TableFromSourceParams, error) { + q := tableFromSourceQuery.QueryPredicate(map[string]string{"mz_tables.id": id}) + + var t TableFromSourceParams + if err := conn.Get(&t, q); err != nil { + return t, err + } + + return t, nil +} + +type TableFromSourceBuilder struct { + ddl Builder + tableName string + schemaName string + databaseName string + source IdentifierSchemaStruct + upstreamName string + upstreamSchemaName string + textColumns []string +} + +func NewTableFromSourceBuilder(conn *sqlx.DB, obj MaterializeObject) *TableFromSourceBuilder { + return &TableFromSourceBuilder{ + ddl: Builder{conn, Table}, + tableName: obj.Name, + schemaName: obj.SchemaName, + databaseName: obj.DatabaseName, + } +} + +func (b *TableFromSourceBuilder) QualifiedName() string { + return QualifiedName(b.databaseName, b.schemaName, b.tableName) +} + +func (b *TableFromSourceBuilder) Source(s IdentifierSchemaStruct) *TableFromSourceBuilder { + b.source = s + return b +} + +func (b *TableFromSourceBuilder) UpstreamName(n string) *TableFromSourceBuilder { + b.upstreamName = n + return b +} + +func (b *TableFromSourceBuilder) UpstreamSchemaName(n string) *TableFromSourceBuilder { + b.upstreamSchemaName = n + return b +} + +func (b *TableFromSourceBuilder) TextColumns(c []string) *TableFromSourceBuilder { + b.textColumns = c + return b +} + +func (b *TableFromSourceBuilder) Create() error { + q := strings.Builder{} + q.WriteString(fmt.Sprintf(`CREATE TABLE %s`, b.QualifiedName())) + q.WriteString(fmt.Sprintf(` FROM SOURCE %s`, b.source.QualifiedName())) + q.WriteString(` (REFERENCE `) + + if b.upstreamSchemaName != "" { + q.WriteString(fmt.Sprintf(`%s.`, QuoteIdentifier(b.upstreamSchemaName))) + } + q.WriteString(QuoteIdentifier(b.upstreamName)) + + q.WriteString(")") + + if len(b.textColumns) > 0 { + q.WriteString(fmt.Sprintf(` WITH (TEXT COLUMNS (%s))`, strings.Join(b.textColumns, ", "))) + } + + q.WriteString(`;`) + return b.ddl.exec(q.String()) +} + +func (b *TableFromSourceBuilder) Rename(newName string) error { + oldName := b.QualifiedName() + b.tableName = newName + newName = b.QualifiedName() + return b.ddl.rename(oldName, newName) +} + +func (b *TableFromSourceBuilder) Drop() error { + qn := b.QualifiedName() + return b.ddl.drop(qn) +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index ce9eb5ac..7749a6fb 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -133,6 +133,7 @@ func Provider(version string) *schema.Provider { "materialize_source_grant": resources.GrantSource(), "materialize_system_parameter": resources.SystemParameter(), "materialize_table": resources.Table(), + "materialize_table_from_source": resources.TableFromSource(), "materialize_table_grant": resources.GrantTable(), "materialize_table_grant_default_privilege": resources.GrantTableDefaultPrivilege(), "materialize_type": resources.Type(), diff --git a/pkg/resources/resource_source_postgres.go b/pkg/resources/resource_source_postgres.go index c95e9887..e07b6455 100644 --- a/pkg/resources/resource_source_postgres.go +++ b/pkg/resources/resource_source_postgres.go @@ -34,12 +34,14 @@ var sourcePostgresSchema = map[string]*schema.Schema{ }, "text_columns": { Description: "Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute.", + Deprecated: "Use the new materialize_table_from_source resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "table": { Description: "Creates subsources for specific tables in the Postgres connection.", + Deprecated: "Use the new materialize_table_from_source resource instead.", Type: schema.TypeSet, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ @@ -203,6 +205,7 @@ func sourcePostgresCreate(ctx context.Context, d *schema.ResourceData, meta any) } if v, ok := d.GetOk("table"); ok { + log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_table_from_source resource instead.") tables := v.(*schema.Set).List() t := materialize.GetTableStruct(tables) b.Table(t) @@ -289,6 +292,7 @@ func sourcePostgresUpdate(ctx context.Context, d *schema.ResourceData, meta any) } if d.HasChange("table") { + log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_table_from_source resource instead.") ot, nt := d.GetChange("table") addTables := materialize.DiffTableStructs(nt.(*schema.Set).List(), ot.(*schema.Set).List()) dropTables := materialize.DiffTableStructs(ot.(*schema.Set).List(), nt.(*schema.Set).List()) diff --git a/pkg/resources/resource_table_from_source.go b/pkg/resources/resource_table_from_source.go new file mode 100644 index 00000000..a806e29f --- /dev/null +++ b/pkg/resources/resource_table_from_source.go @@ -0,0 +1,242 @@ +package resources + +import ( + "context" + "database/sql" + "log" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +var tableFromSourceSchema = map[string]*schema.Schema{ + "name": ObjectNameSchema("table", true, false), + "schema_name": SchemaNameSchema("table", false), + "database_name": DatabaseNameSchema("table", false), + "qualified_sql_name": QualifiedNameSchema("table"), + "source": IdentifierSchema(IdentifierSchemaParams{ + Elem: "source", + Description: "The source this table is created from.", + Required: true, + ForceNew: true, + }), + "upstream_name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "The name of the table in the upstream database.", + }, + "upstream_schema_name": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: "The schema of the table in the upstream database.", + }, + "text_columns": { + Description: "Columns to be decoded as text.", + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + ForceNew: true, + }, + "comment": CommentSchema(false), + "ownership_role": OwnershipRoleSchema(), + "region": RegionSchema(), +} + +func TableFromSource() *schema.Resource { + return &schema.Resource{ + CreateContext: tableFromSourceCreate, + ReadContext: tableFromSourceRead, + UpdateContext: tableFromSourceUpdate, + DeleteContext: tableFromSourceDelete, + + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + + Schema: tableFromSourceSchema, + } +} + +func tableFromSourceCreate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + tableName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewTableFromSourceBuilder(metaDb, o) + + source := materialize.GetIdentifierSchemaStruct(d.Get("source")) + b.Source(source) + + b.UpstreamName(d.Get("upstream_name").(string)) + + if v, ok := d.GetOk("upstream_schema_name"); ok { + b.UpstreamSchemaName(v.(string)) + } + + if v, ok := d.GetOk("text_columns"); ok { + textColumns, err := materialize.GetSliceValueString("text_columns", v.([]interface{})) + if err != nil { + return diag.FromErr(err) + } + b.TextColumns(textColumns) + } + + if err := b.Create(); err != nil { + return diag.FromErr(err) + } + + // Handle ownership + if v, ok := d.GetOk("ownership_role"); ok { + ownership := materialize.NewOwnershipBuilder(metaDb, o) + if err := ownership.Alter(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed ownership, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + // Handle comments + if v, ok := d.GetOk("comment"); ok { + comment := materialize.NewCommentBuilder(metaDb, o) + if err := comment.Object(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed comment, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + i, err := materialize.TableFromSourceId(metaDb, o) + if err != nil { + return diag.FromErr(err) + } + d.SetId(utils.TransformIdWithRegion(string(region), i)) + + return tableFromSourceRead(ctx, d, meta) +} + +func tableFromSourceRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + i := d.Id() + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + t, err := materialize.ScanTableFromSource(metaDb, utils.ExtractId(i)) + if err == sql.ErrNoRows { + d.SetId("") + return nil + } else if err != nil { + return diag.FromErr(err) + } + + d.SetId(utils.TransformIdWithRegion(string(region), i)) + + if err := d.Set("name", t.TableName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("schema_name", t.SchemaName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("database_name", t.DatabaseName.String); err != nil { + return diag.FromErr(err) + } + + // TODO: Set source once the source_id is available in the mz_tables table + + // if err := d.Set("upstream_name", t.UpstreamName.String); err != nil { + // return diag.FromErr(err) + // } + + // if err := d.Set("upstream_schema_name", t.UpstreamSchemaName.String); err != nil { + // return diag.FromErr(err) + // } + + if err := d.Set("ownership_role", t.OwnerName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("comment", t.Comment.String); err != nil { + return diag.FromErr(err) + } + + return nil +} + +func tableFromSourceUpdate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + tableName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, _, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} + + if d.HasChange("name") { + oldName, newName := d.GetChange("name") + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: oldName.(string), SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewTableFromSourceBuilder(metaDb, o) + if err := b.Rename(newName.(string)); err != nil { + return diag.FromErr(err) + } + } + + // TODO: Handle source changes + // TODO: Handle text_columns changes + + if d.HasChange("ownership_role") { + _, newRole := d.GetChange("ownership_role") + b := materialize.NewOwnershipBuilder(metaDb, o) + + if err := b.Alter(newRole.(string)); err != nil { + return diag.FromErr(err) + } + } + + if d.HasChange("comment") { + _, newComment := d.GetChange("comment") + b := materialize.NewCommentBuilder(metaDb, o) + + if err := b.Object(newComment.(string)); err != nil { + return diag.FromErr(err) + } + } + + return tableFromSourceRead(ctx, d, meta) +} + +func tableFromSourceDelete(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + tableName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, _, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewTableFromSourceBuilder(metaDb, o) + + if err := b.Drop(); err != nil { + return diag.FromErr(err) + } + + return nil +} From 9bb3a55943b608130fc5ac438426b8f0a99e0224 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 3 Sep 2024 12:39:06 +0300 Subject: [PATCH 02/46] Use source table instead of table from source --- docs/resources/source_postgres.md | 4 +- docs/resources/source_table.md | 49 +++++++++++++++++++ .../{table_from_source.go => source_table.go} | 38 +++++++------- pkg/provider/provider.go | 2 +- pkg/resources/resource_source_postgres.go | 8 +-- ...rom_source.go => resource_source_table.go} | 36 +++++++------- 6 files changed, 93 insertions(+), 44 deletions(-) create mode 100644 docs/resources/source_table.md rename pkg/materialize/{table_from_source.go => source_table.go} (72%) rename pkg/resources/{resource_table_from_source.go => resource_source_table.go} (84%) diff --git a/docs/resources/source_postgres.md b/docs/resources/source_postgres.md index cda942fb..958f6785 100644 --- a/docs/resources/source_postgres.md +++ b/docs/resources/source_postgres.md @@ -52,7 +52,7 @@ resource "materialize_source_postgres" "example_source_postgres" { - `name` (String) The identifier for the source. - `postgres_connection` (Block List, Min: 1, Max: 1) The PostgreSQL connection to use in the source. (see [below for nested schema](#nestedblock--postgres_connection)) - `publication` (String) The PostgreSQL publication (the replication data set containing the tables to be streamed to Materialize). -- `table` (Block Set, Min: 1) Creates subsources for specific tables in the Postgres connection. (see [below for nested schema](#nestedblock--table)) +- `table` (Block Set, Min: 1, Deprecated) Creates subsources for specific tables in the Postgres connection. (see [below for nested schema](#nestedblock--table)) ### Optional @@ -63,7 +63,7 @@ resource "materialize_source_postgres" "example_source_postgres" { - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `text_columns` (List of String) Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. +- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. ### Read-Only diff --git a/docs/resources/source_table.md b/docs/resources/source_table.md new file mode 100644 index 00000000..b93470b4 --- /dev/null +++ b/docs/resources/source_table.md @@ -0,0 +1,49 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "materialize_source_table Resource - terraform-provider-materialize" +subcategory: "" +description: |- + +--- + +# materialize_source_table (Resource) + + + + + + +## Schema + +### Required + +- `name` (String) The identifier for the table. +- `source` (Block List, Min: 1, Max: 1) The source this table is created from. (see [below for nested schema](#nestedblock--source)) +- `upstream_name` (String) The name of the table in the upstream database. + +### Optional + +- `comment` (String) **Public Preview** Comment on an object in the database. +- `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `ownership_role` (String) The owernship role of the object. +- `region` (String) The region to use for the resource connection. If not set, the default region is used. +- `schema_name` (String) The identifier for the table schema in Materialize. Defaults to `public`. +- `text_columns` (List of String) Columns to be decoded as text. +- `upstream_schema_name` (String) The schema of the table in the upstream database. + +### Read-Only + +- `id` (String) The ID of this resource. +- `qualified_sql_name` (String) The fully qualified name of the table. + + +### Nested Schema for `source` + +Required: + +- `name` (String) The source name. + +Optional: + +- `database_name` (String) The source database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The source schema name. Defaults to `public`. diff --git a/pkg/materialize/table_from_source.go b/pkg/materialize/source_table.go similarity index 72% rename from pkg/materialize/table_from_source.go rename to pkg/materialize/source_table.go index 92f2146d..3520b859 100644 --- a/pkg/materialize/table_from_source.go +++ b/pkg/materialize/source_table.go @@ -9,7 +9,7 @@ import ( "github.com/lib/pq" ) -type TableFromSourceParams struct { +type SourceTableParams struct { TableId sql.NullString `db:"id"` TableName sql.NullString `db:"name"` SchemaName sql.NullString `db:"schema_name"` @@ -26,7 +26,7 @@ type TableFromSourceParams struct { } // TODO: Extend this query to include the upstream table name and schema name and the source -var tableFromSourceQuery = NewBaseQuery(` +var sourceTableQuery = NewBaseQuery(` SELECT mz_tables.id, mz_tables.name, @@ -51,15 +51,15 @@ var tableFromSourceQuery = NewBaseQuery(` ON mz_tables.id = comments.id `) -func TableFromSourceId(conn *sqlx.DB, obj MaterializeObject) (string, error) { +func SourceTableId(conn *sqlx.DB, obj MaterializeObject) (string, error) { p := map[string]string{ "mz_tables.name": obj.Name, "mz_schemas.name": obj.SchemaName, "mz_databases.name": obj.DatabaseName, } - q := tableFromSourceQuery.QueryPredicate(p) + q := sourceTableQuery.QueryPredicate(p) - var t TableFromSourceParams + var t SourceTableParams if err := conn.Get(&t, q); err != nil { return "", err } @@ -67,10 +67,10 @@ func TableFromSourceId(conn *sqlx.DB, obj MaterializeObject) (string, error) { return t.TableId.String, nil } -func ScanTableFromSource(conn *sqlx.DB, id string) (TableFromSourceParams, error) { - q := tableFromSourceQuery.QueryPredicate(map[string]string{"mz_tables.id": id}) +func ScanSourceTable(conn *sqlx.DB, id string) (SourceTableParams, error) { + q := sourceTableQuery.QueryPredicate(map[string]string{"mz_tables.id": id}) - var t TableFromSourceParams + var t SourceTableParams if err := conn.Get(&t, q); err != nil { return t, err } @@ -78,7 +78,7 @@ func ScanTableFromSource(conn *sqlx.DB, id string) (TableFromSourceParams, error return t, nil } -type TableFromSourceBuilder struct { +type SourceTableBuilder struct { ddl Builder tableName string schemaName string @@ -89,8 +89,8 @@ type TableFromSourceBuilder struct { textColumns []string } -func NewTableFromSourceBuilder(conn *sqlx.DB, obj MaterializeObject) *TableFromSourceBuilder { - return &TableFromSourceBuilder{ +func NewSourceTableBuilder(conn *sqlx.DB, obj MaterializeObject) *SourceTableBuilder { + return &SourceTableBuilder{ ddl: Builder{conn, Table}, tableName: obj.Name, schemaName: obj.SchemaName, @@ -98,31 +98,31 @@ func NewTableFromSourceBuilder(conn *sqlx.DB, obj MaterializeObject) *TableFromS } } -func (b *TableFromSourceBuilder) QualifiedName() string { +func (b *SourceTableBuilder) QualifiedName() string { return QualifiedName(b.databaseName, b.schemaName, b.tableName) } -func (b *TableFromSourceBuilder) Source(s IdentifierSchemaStruct) *TableFromSourceBuilder { +func (b *SourceTableBuilder) Source(s IdentifierSchemaStruct) *SourceTableBuilder { b.source = s return b } -func (b *TableFromSourceBuilder) UpstreamName(n string) *TableFromSourceBuilder { +func (b *SourceTableBuilder) UpstreamName(n string) *SourceTableBuilder { b.upstreamName = n return b } -func (b *TableFromSourceBuilder) UpstreamSchemaName(n string) *TableFromSourceBuilder { +func (b *SourceTableBuilder) UpstreamSchemaName(n string) *SourceTableBuilder { b.upstreamSchemaName = n return b } -func (b *TableFromSourceBuilder) TextColumns(c []string) *TableFromSourceBuilder { +func (b *SourceTableBuilder) TextColumns(c []string) *SourceTableBuilder { b.textColumns = c return b } -func (b *TableFromSourceBuilder) Create() error { +func (b *SourceTableBuilder) Create() error { q := strings.Builder{} q.WriteString(fmt.Sprintf(`CREATE TABLE %s`, b.QualifiedName())) q.WriteString(fmt.Sprintf(` FROM SOURCE %s`, b.source.QualifiedName())) @@ -143,14 +143,14 @@ func (b *TableFromSourceBuilder) Create() error { return b.ddl.exec(q.String()) } -func (b *TableFromSourceBuilder) Rename(newName string) error { +func (b *SourceTableBuilder) Rename(newName string) error { oldName := b.QualifiedName() b.tableName = newName newName = b.QualifiedName() return b.ddl.rename(oldName, newName) } -func (b *TableFromSourceBuilder) Drop() error { +func (b *SourceTableBuilder) Drop() error { qn := b.QualifiedName() return b.ddl.drop(qn) } diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 7749a6fb..c5f0886e 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -133,7 +133,7 @@ func Provider(version string) *schema.Provider { "materialize_source_grant": resources.GrantSource(), "materialize_system_parameter": resources.SystemParameter(), "materialize_table": resources.Table(), - "materialize_table_from_source": resources.TableFromSource(), + "materialize_source_table": resources.SourceTable(), "materialize_table_grant": resources.GrantTable(), "materialize_table_grant_default_privilege": resources.GrantTableDefaultPrivilege(), "materialize_type": resources.Type(), diff --git a/pkg/resources/resource_source_postgres.go b/pkg/resources/resource_source_postgres.go index e07b6455..b9185db4 100644 --- a/pkg/resources/resource_source_postgres.go +++ b/pkg/resources/resource_source_postgres.go @@ -34,14 +34,14 @@ var sourcePostgresSchema = map[string]*schema.Schema{ }, "text_columns": { Description: "Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute.", - Deprecated: "Use the new materialize_table_from_source resource instead.", + Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "table": { Description: "Creates subsources for specific tables in the Postgres connection.", - Deprecated: "Use the new materialize_table_from_source resource instead.", + Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeSet, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ @@ -205,7 +205,7 @@ func sourcePostgresCreate(ctx context.Context, d *schema.ResourceData, meta any) } if v, ok := d.GetOk("table"); ok { - log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_table_from_source resource instead.") + log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_source_table resource instead.") tables := v.(*schema.Set).List() t := materialize.GetTableStruct(tables) b.Table(t) @@ -292,7 +292,7 @@ func sourcePostgresUpdate(ctx context.Context, d *schema.ResourceData, meta any) } if d.HasChange("table") { - log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_table_from_source resource instead.") + log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_source_table resource instead.") ot, nt := d.GetChange("table") addTables := materialize.DiffTableStructs(nt.(*schema.Set).List(), ot.(*schema.Set).List()) dropTables := materialize.DiffTableStructs(ot.(*schema.Set).List(), nt.(*schema.Set).List()) diff --git a/pkg/resources/resource_table_from_source.go b/pkg/resources/resource_source_table.go similarity index 84% rename from pkg/resources/resource_table_from_source.go rename to pkg/resources/resource_source_table.go index a806e29f..412bf7f6 100644 --- a/pkg/resources/resource_table_from_source.go +++ b/pkg/resources/resource_source_table.go @@ -12,7 +12,7 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) -var tableFromSourceSchema = map[string]*schema.Schema{ +var sourceTableSchema = map[string]*schema.Schema{ "name": ObjectNameSchema("table", true, false), "schema_name": SchemaNameSchema("table", false), "database_name": DatabaseNameSchema("table", false), @@ -47,22 +47,22 @@ var tableFromSourceSchema = map[string]*schema.Schema{ "region": RegionSchema(), } -func TableFromSource() *schema.Resource { +func SourceTable() *schema.Resource { return &schema.Resource{ - CreateContext: tableFromSourceCreate, - ReadContext: tableFromSourceRead, - UpdateContext: tableFromSourceUpdate, - DeleteContext: tableFromSourceDelete, + CreateContext: sourceTableCreate, + ReadContext: sourceTableRead, + UpdateContext: sourceTableUpdate, + DeleteContext: sourceTableDelete, Importer: &schema.ResourceImporter{ StateContext: schema.ImportStatePassthroughContext, }, - Schema: tableFromSourceSchema, + Schema: sourceTableSchema, } } -func tableFromSourceCreate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { +func sourceTableCreate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { tableName := d.Get("name").(string) schemaName := d.Get("schema_name").(string) databaseName := d.Get("database_name").(string) @@ -73,7 +73,7 @@ func tableFromSourceCreate(ctx context.Context, d *schema.ResourceData, meta any } o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} - b := materialize.NewTableFromSourceBuilder(metaDb, o) + b := materialize.NewSourceTableBuilder(metaDb, o) source := materialize.GetIdentifierSchemaStruct(d.Get("source")) b.Source(source) @@ -116,16 +116,16 @@ func tableFromSourceCreate(ctx context.Context, d *schema.ResourceData, meta any } } - i, err := materialize.TableFromSourceId(metaDb, o) + i, err := materialize.SourceTableId(metaDb, o) if err != nil { return diag.FromErr(err) } d.SetId(utils.TransformIdWithRegion(string(region), i)) - return tableFromSourceRead(ctx, d, meta) + return sourceTableRead(ctx, d, meta) } -func tableFromSourceRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { +func sourceTableRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { i := d.Id() metaDb, region, err := utils.GetDBClientFromMeta(meta, d) @@ -133,7 +133,7 @@ func tableFromSourceRead(ctx context.Context, d *schema.ResourceData, meta inter return diag.FromErr(err) } - t, err := materialize.ScanTableFromSource(metaDb, utils.ExtractId(i)) + t, err := materialize.ScanSourceTable(metaDb, utils.ExtractId(i)) if err == sql.ErrNoRows { d.SetId("") return nil @@ -176,7 +176,7 @@ func tableFromSourceRead(ctx context.Context, d *schema.ResourceData, meta inter return nil } -func tableFromSourceUpdate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { +func sourceTableUpdate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { tableName := d.Get("name").(string) schemaName := d.Get("schema_name").(string) databaseName := d.Get("database_name").(string) @@ -191,7 +191,7 @@ func tableFromSourceUpdate(ctx context.Context, d *schema.ResourceData, meta any if d.HasChange("name") { oldName, newName := d.GetChange("name") o := materialize.MaterializeObject{ObjectType: "TABLE", Name: oldName.(string), SchemaName: schemaName, DatabaseName: databaseName} - b := materialize.NewTableFromSourceBuilder(metaDb, o) + b := materialize.NewSourceTableBuilder(metaDb, o) if err := b.Rename(newName.(string)); err != nil { return diag.FromErr(err) } @@ -218,10 +218,10 @@ func tableFromSourceUpdate(ctx context.Context, d *schema.ResourceData, meta any } } - return tableFromSourceRead(ctx, d, meta) + return sourceTableRead(ctx, d, meta) } -func tableFromSourceDelete(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { +func sourceTableDelete(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { tableName := d.Get("name").(string) schemaName := d.Get("schema_name").(string) databaseName := d.Get("database_name").(string) @@ -232,7 +232,7 @@ func tableFromSourceDelete(ctx context.Context, d *schema.ResourceData, meta any } o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} - b := materialize.NewTableFromSourceBuilder(metaDb, o) + b := materialize.NewSourceTableBuilder(metaDb, o) if err := b.Drop(); err != nil { return diag.FromErr(err) From 467b05ea61d1f4dadd102e4c165f6078efaecc5b Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 3 Sep 2024 22:18:46 +0300 Subject: [PATCH 03/46] MySQL source: separate for tables and all tables --- docs/resources/source_mysql.md | 5 +++-- docs/resources/source_postgres.md | 26 +++++++++++------------ pkg/materialize/source_mysql.go | 10 ++++++++- pkg/materialize/source_mysql_test.go | 1 + pkg/resources/resource_source_mysql.go | 13 ++++++++++++ pkg/resources/resource_source_postgres.go | 3 +-- 6 files changed, 40 insertions(+), 18 deletions(-) diff --git a/docs/resources/source_mysql.md b/docs/resources/source_mysql.md index 99f3563e..0e3e3c1f 100644 --- a/docs/resources/source_mysql.md +++ b/docs/resources/source_mysql.md @@ -52,6 +52,7 @@ resource "materialize_source_mysql" "test" { ### Optional +- `all_tables` (Boolean, Deprecated) Include all tables in the source. If `table` is specified, this will be ignored. - `cluster_name` (String) The cluster to maintain this source. - `comment` (String) Comment on an object in the database. - `database_name` (String) The identifier for the source database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. @@ -60,8 +61,8 @@ resource "materialize_source_mysql" "test" { - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `table` (Block Set) Specify the tables to be included in the source. If not specified, all tables are included. (see [below for nested schema](#nestedblock--table)) -- `text_columns` (List of String) Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. +- `table` (Block Set, Deprecated) Specify the tables to be included in the source. If not specified, all tables are included. (see [below for nested schema](#nestedblock--table)) +- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. ### Read-Only diff --git a/docs/resources/source_postgres.md b/docs/resources/source_postgres.md index 958f6785..78314094 100644 --- a/docs/resources/source_postgres.md +++ b/docs/resources/source_postgres.md @@ -52,7 +52,6 @@ resource "materialize_source_postgres" "example_source_postgres" { - `name` (String) The identifier for the source. - `postgres_connection` (Block List, Min: 1, Max: 1) The PostgreSQL connection to use in the source. (see [below for nested schema](#nestedblock--postgres_connection)) - `publication` (String) The PostgreSQL publication (the replication data set containing the tables to be streamed to Materialize). -- `table` (Block Set, Min: 1, Deprecated) Creates subsources for specific tables in the Postgres connection. (see [below for nested schema](#nestedblock--table)) ### Optional @@ -63,6 +62,7 @@ resource "materialize_source_postgres" "example_source_postgres" { - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. +- `table` (Block Set, Deprecated) Creates subsources for specific tables in the Postgres connection. (see [below for nested schema](#nestedblock--table)) - `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. ### Read-Only @@ -84,32 +84,32 @@ Optional: - `schema_name` (String) The postgres_connection schema name. Defaults to `public`. - -### Nested Schema for `table` + +### Nested Schema for `expose_progress` Required: -- `upstream_name` (String) The name of the table in the upstream Postgres database. +- `name` (String) The expose_progress name. Optional: -- `database_name` (String) The database of the table in Materialize. -- `name` (String) The name of the table in Materialize. -- `schema_name` (String) The schema of the table in Materialize. -- `upstream_schema_name` (String) The schema of the table in the upstream Postgres database. +- `database_name` (String) The expose_progress database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The expose_progress schema name. Defaults to `public`. - -### Nested Schema for `expose_progress` + +### Nested Schema for `table` Required: -- `name` (String) The expose_progress name. +- `upstream_name` (String) The name of the table in the upstream Postgres database. Optional: -- `database_name` (String) The expose_progress database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. -- `schema_name` (String) The expose_progress schema name. Defaults to `public`. +- `database_name` (String) The database of the table in Materialize. +- `name` (String) The name of the table in Materialize. +- `schema_name` (String) The schema of the table in Materialize. +- `upstream_schema_name` (String) The schema of the table in the upstream Postgres database. ## Import diff --git a/pkg/materialize/source_mysql.go b/pkg/materialize/source_mysql.go index f90d79b3..97db730a 100644 --- a/pkg/materialize/source_mysql.go +++ b/pkg/materialize/source_mysql.go @@ -15,6 +15,7 @@ type SourceMySQLBuilder struct { ignoreColumns []string textColumns []string tables []TableStruct + allTables bool exposeProgress IdentifierSchemaStruct } @@ -55,6 +56,11 @@ func (b *SourceMySQLBuilder) Tables(tables []TableStruct) *SourceMySQLBuilder { return b } +func (b *SourceMySQLBuilder) AllTables() *SourceMySQLBuilder { + b.allTables = true + return b +} + func (b *SourceMySQLBuilder) ExposeProgress(e IdentifierSchemaStruct) *SourceMySQLBuilder { b.exposeProgress = e return b @@ -111,7 +117,9 @@ func (b *SourceMySQLBuilder) Create() error { } q.WriteString(`)`) } else { - q.WriteString(` FOR ALL TABLES`) + if b.allTables { + q.WriteString(` FOR ALL TABLES`) + } } if b.exposeProgress.Name != "" { diff --git a/pkg/materialize/source_mysql_test.go b/pkg/materialize/source_mysql_test.go index c9735126..e4c4f3d6 100644 --- a/pkg/materialize/source_mysql_test.go +++ b/pkg/materialize/source_mysql_test.go @@ -22,6 +22,7 @@ func TestSourceMySQLAllTablesCreate(t *testing.T) { b := NewSourceMySQLBuilder(db, sourceMySQL) b.MySQLConnection(IdentifierSchemaStruct{Name: "mysql_connection", SchemaName: "schema", DatabaseName: "database"}) + b.AllTables() if err := b.Create(); err != nil { t.Fatal(err) diff --git a/pkg/resources/resource_source_mysql.go b/pkg/resources/resource_source_mysql.go index 6421d76a..275520eb 100644 --- a/pkg/resources/resource_source_mysql.go +++ b/pkg/resources/resource_source_mysql.go @@ -34,12 +34,14 @@ var sourceMySQLSchema = map[string]*schema.Schema{ }, "text_columns": { Description: "Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute.", + Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "table": { Description: "Specify the tables to be included in the source. If not specified, all tables are included.", + Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeSet, Optional: true, Elem: &schema.Resource{ @@ -76,6 +78,13 @@ var sourceMySQLSchema = map[string]*schema.Schema{ }, }, }, + "all_tables": { + Description: "Include all tables in the source. If `table` is specified, this will be ignored.", + Deprecated: "Use the new materialize_source_table resource instead.", + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + }, "expose_progress": IdentifierSchema(IdentifierSchemaParams{ Elem: "expose_progress", Description: "The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`.", @@ -131,6 +140,10 @@ func sourceMySQLCreate(ctx context.Context, d *schema.ResourceData, meta any) di b.Tables(t) } + if v, ok := d.GetOk("all_tables"); ok && v.(bool) { + b.AllTables() + } + if v, ok := d.GetOk("ignore_columns"); ok && len(v.([]interface{})) > 0 { columns, err := materialize.GetSliceValueString("ignore_columns", v.([]interface{})) if err != nil { diff --git a/pkg/resources/resource_source_postgres.go b/pkg/resources/resource_source_postgres.go index b9185db4..9b057a22 100644 --- a/pkg/resources/resource_source_postgres.go +++ b/pkg/resources/resource_source_postgres.go @@ -43,6 +43,7 @@ var sourcePostgresSchema = map[string]*schema.Schema{ Description: "Creates subsources for specific tables in the Postgres connection.", Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeSet, + Optional: true, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "upstream_name": { @@ -76,8 +77,6 @@ var sourcePostgresSchema = map[string]*schema.Schema{ }, }, }, - Required: true, - MinItems: 1, }, "expose_progress": IdentifierSchema(IdentifierSchemaParams{ Elem: "expose_progress", From 2fcb9a16a6e59b26db24f957223b65afe80c8ba9 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 3 Sep 2024 22:43:59 +0300 Subject: [PATCH 04/46] Loadgen source: add all tables bool attr --- docs/resources/source_load_generator.md | 1 + pkg/materialize/source_load_generator.go | 10 +++++++++- pkg/materialize/source_load_generator_test.go | 3 +++ pkg/resources/resource_source_load_generator.go | 13 +++++++++++++ .../resource_source_load_generator_test.go | 1 + 5 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/resources/source_load_generator.md b/docs/resources/source_load_generator.md index a5d18c33..99db6947 100644 --- a/docs/resources/source_load_generator.md +++ b/docs/resources/source_load_generator.md @@ -40,6 +40,7 @@ resource "materialize_source_load_generator" "example_source_load_generator" { ### Optional +- `all_tables` (Boolean) Whether to include all tables in the source. Compatible with `auction_options`, `marketing_options`, and `tpch_options`. IF not specified, use the `materialize_source_table` resource to specify tables to include. - `auction_options` (Block List, Max: 1) Auction Options. (see [below for nested schema](#nestedblock--auction_options)) - `cluster_name` (String) The cluster to maintain this source. - `comment` (String) Comment on an object in the database. diff --git a/pkg/materialize/source_load_generator.go b/pkg/materialize/source_load_generator.go index 24c5d52c..2aec6865 100644 --- a/pkg/materialize/source_load_generator.go +++ b/pkg/materialize/source_load_generator.go @@ -127,6 +127,7 @@ type SourceLoadgenBuilder struct { clusterName string size string loadGeneratorType string + allTables bool counterOptions CounterOptions auctionOptions AuctionOptions marketingOptions MarketingOptions @@ -157,6 +158,11 @@ func (b *SourceLoadgenBuilder) LoadGeneratorType(l string) *SourceLoadgenBuilder return b } +func (b *SourceLoadgenBuilder) AllTables() *SourceLoadgenBuilder { + b.allTables = true + return b +} + func (b *SourceLoadgenBuilder) ExposeProgress(e IdentifierSchemaStruct) *SourceLoadgenBuilder { b.exposeProgress = e return b @@ -251,7 +257,9 @@ func (b *SourceLoadgenBuilder) Create() error { // Include for multi-output sources if b.loadGeneratorType == "AUCTION" || b.loadGeneratorType == "MARKETING" || b.loadGeneratorType == "TPCH" { - q.WriteString(` FOR ALL TABLES`) + if b.allTables { + q.WriteString(` FOR ALL TABLES`) + } } if b.exposeProgress.Name != "" { diff --git a/pkg/materialize/source_load_generator_test.go b/pkg/materialize/source_load_generator_test.go index d70ed3c5..781771f2 100644 --- a/pkg/materialize/source_load_generator_test.go +++ b/pkg/materialize/source_load_generator_test.go @@ -44,6 +44,7 @@ func TestSourceLoadgenAuctionCreate(t *testing.T) { b := NewSourceLoadgenBuilder(db, sourceLoadgen) b.LoadGeneratorType("AUCTION") + b.AllTables() b.AuctionOptions(AuctionOptions{ TickInterval: "1s", }) @@ -65,6 +66,7 @@ func TestSourceLoadgenMarketingCreate(t *testing.T) { b := NewSourceLoadgenBuilder(db, sourceLoadgen) b.LoadGeneratorType("MARKETING") + b.AllTables() b.MarketingOptions(MarketingOptions{ TickInterval: "1s", }) @@ -86,6 +88,7 @@ func TestSourceLoadgenTPCHParamsCreate(t *testing.T) { b := NewSourceLoadgenBuilder(db, sourceLoadgen) b.LoadGeneratorType("TPCH") + b.AllTables() b.TPCHOptions(TPCHOptions{ TickInterval: "1s", ScaleFactor: 0.01, diff --git a/pkg/resources/resource_source_load_generator.go b/pkg/resources/resource_source_load_generator.go index 30ce5d81..58febe32 100644 --- a/pkg/resources/resource_source_load_generator.go +++ b/pkg/resources/resource_source_load_generator.go @@ -174,6 +174,14 @@ var sourceLoadgenSchema = map[string]*schema.Schema{ ForceNew: true, ConflictsWith: []string{"counter_options", "auction_options", "marketing_options", "tpch_options"}, }, + "all_tables": { + Description: "Whether to include all tables in the source. Compatible with `auction_options`, `marketing_options`, and `tpch_options`. IF not specified, use the `materialize_source_table` resource to specify tables to include.", + Type: schema.TypeBool, + Optional: true, + Default: false, + ConflictsWith: []string{"counter_options", "key_value_options"}, + ForceNew: true, + }, "expose_progress": IdentifierSchema(IdentifierSchemaParams{ Elem: "expose_progress", Description: "The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`.", @@ -251,6 +259,11 @@ func sourceLoadgenCreate(ctx context.Context, d *schema.ResourceData, meta any) b.KeyValueOptions(o) } + // all_tables + if v, ok := d.GetOk("all_tables"); ok && v.(bool) { + b.AllTables() + } + // create resource if err := b.Create(); err != nil { return diag.FromErr(err) diff --git a/pkg/resources/resource_source_load_generator_test.go b/pkg/resources/resource_source_load_generator_test.go index 8ba0ac9d..414addd0 100644 --- a/pkg/resources/resource_source_load_generator_test.go +++ b/pkg/resources/resource_source_load_generator_test.go @@ -19,6 +19,7 @@ var inSourceLoadgen = map[string]interface{}{ "cluster_name": "cluster", "expose_progress": []interface{}{map[string]interface{}{"name": "progress"}}, "load_generator_type": "TPCH", + "all_tables": true, "tpch_options": []interface{}{map[string]interface{}{ "tick_interval": "1s", "scale_factor": 0.5, From 95e7eebe6621564ae2c456c8fdf05f0978c3d9ac Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Wed, 4 Sep 2024 20:47:28 +0300 Subject: [PATCH 05/46] Add tests --- integration/postgres/postgres_bootstrap.sql | 5 +- pkg/materialize/source_table.go | 3 +- pkg/materialize/source_table_test.go | 58 ++++ pkg/provider/acceptance_source_table_test.go | 270 +++++++++++++++++++ pkg/resources/resource_source_table.go | 3 +- pkg/resources/resource_source_table_test.go | 113 ++++++++ 6 files changed, 447 insertions(+), 5 deletions(-) create mode 100644 pkg/materialize/source_table_test.go create mode 100644 pkg/provider/acceptance_source_table_test.go create mode 100644 pkg/resources/resource_source_table_test.go diff --git a/integration/postgres/postgres_bootstrap.sql b/integration/postgres/postgres_bootstrap.sql index 0fc7ca01..39163abb 100644 --- a/integration/postgres/postgres_bootstrap.sql +++ b/integration/postgres/postgres_bootstrap.sql @@ -11,7 +11,8 @@ CREATE TABLE table2 ( ); CREATE TABLE table3 ( - id INT GENERATED ALWAYS AS IDENTITY + id INT GENERATED ALWAYS AS IDENTITY, + updated_at timestamp NOT NULL ); -- Enable REPLICA for both tables @@ -24,4 +25,4 @@ CREATE PUBLICATION mz_source FOR TABLE table1, table2, table3; INSERT INTO table1 VALUES (1), (2), (3), (4), (5); INSERT INTO table2 VALUES (1, NOW()), (2, NOW()), (3, NOW()), (4, NOW()), (5, NOW()); -INSERT INTO table3 VALUES (1), (2), (3), (4), (5); +INSERT INTO table3 VALUES (1, NOW()), (2, NOW()), (3, NOW()), (4, NOW()), (5, NOW()); diff --git a/pkg/materialize/source_table.go b/pkg/materialize/source_table.go index 3520b859..520df928 100644 --- a/pkg/materialize/source_table.go +++ b/pkg/materialize/source_table.go @@ -136,7 +136,8 @@ func (b *SourceTableBuilder) Create() error { q.WriteString(")") if len(b.textColumns) > 0 { - q.WriteString(fmt.Sprintf(` WITH (TEXT COLUMNS (%s))`, strings.Join(b.textColumns, ", "))) + c := strings.Join(b.textColumns, ", ") + q.WriteString(fmt.Sprintf(` WITH (TEXT COLUMNS [%s])`, c)) } q.WriteString(`;`) diff --git a/pkg/materialize/source_table_test.go b/pkg/materialize/source_table_test.go new file mode 100644 index 00000000..384fe815 --- /dev/null +++ b/pkg/materialize/source_table_test.go @@ -0,0 +1,58 @@ +package materialize + +import ( + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/jmoiron/sqlx" +) + +var sourceTable = MaterializeObject{Name: "table", SchemaName: "schema", DatabaseName: "database"} + +func TestSourceTableCreate(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."source" + \(REFERENCE "upstream_schema"."upstream_table"\) + WITH \(TEXT COLUMNS \(column1, column2\)\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableBuilder(db, sourceTable) + b.Source(IdentifierSchemaStruct{Name: "source", SchemaName: "public", DatabaseName: "materialize"}) + b.UpstreamName("upstream_table") + b.UpstreamSchemaName("upstream_schema") + b.TextColumns([]string{"column1", "column2"}) + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableRename(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `ALTER TABLE "database"."schema"."table" RENAME TO "database"."schema"."new_table";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableBuilder(db, sourceTable) + if err := b.Rename("new_table"); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableDrop(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `DROP TABLE "database"."schema"."table";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableBuilder(db, sourceTable) + if err := b.Drop(); err != nil { + t.Fatal(err) + } + }) +} diff --git a/pkg/provider/acceptance_source_table_test.go b/pkg/provider/acceptance_source_table_test.go new file mode 100644 index 00000000..26037255 --- /dev/null +++ b/pkg/provider/acceptance_source_table_test.go @@ -0,0 +1,270 @@ +package provider + +import ( + "database/sql" + "fmt" + "testing" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + "github.com/hashicorp/terraform-plugin-testing/helper/acctest" + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-plugin-testing/terraform" +) + +func TestAccSourceTable_basic(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: nil, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableBasicResource(nameSpace), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table.test"), + resource.TestMatchResourceAttr("materialize_source_table.test", "id", terraformObjectIdRegex), + resource.TestCheckResourceAttr("materialize_source_table.test", "name", nameSpace+"_table"), + resource.TestCheckResourceAttr("materialize_source_table.test", "database_name", "materialize"), + resource.TestCheckResourceAttr("materialize_source_table.test", "schema_name", "public"), + // resource.TestCheckResourceAttr("materialize_source_table.test", "qualified_sql_name", fmt.Sprintf(`"materialize"."public"."%s_table"`, nameSpace)), + resource.TestCheckResourceAttr("materialize_source_table.test", "text_columns.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table.test", "text_columns.0", "updated_at"), + resource.TestCheckResourceAttr("materialize_source_table.test", "upstream_name", "table2"), + resource.TestCheckResourceAttr("materialize_source_table.test", "upstream_schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table.test", "ownership_role", "mz_system"), + resource.TestCheckResourceAttr("materialize_source_table.test", "comment", ""), + ), + }, + { + ResourceName: "materialize_source_table.test", + ImportState: true, + ImportStateVerify: false, + }, + }, + }) +} + +func TestAccSourceTable_update(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: nil, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableResource(nameSpace, "table2", "mz_system", ""), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table.test"), + resource.TestCheckResourceAttr("materialize_source_table.test", "name", nameSpace+"_table"), + resource.TestCheckResourceAttr("materialize_source_table.test", "upstream_name", "table2"), + resource.TestCheckResourceAttr("materialize_source_table.test", "text_columns.#", "2"), + resource.TestCheckResourceAttr("materialize_source_table.test", "ownership_role", "mz_system"), + resource.TestCheckResourceAttr("materialize_source_table.test", "comment", ""), + ), + }, + { + Config: testAccSourceTableResource(nameSpace, "table3", nameSpace+"_role", "Updated comment"), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table.test"), + resource.TestCheckResourceAttr("materialize_source_table.test", "name", nameSpace+"_table"), + resource.TestCheckResourceAttr("materialize_source_table.test", "upstream_name", "table3"), + resource.TestCheckResourceAttr("materialize_source_table.test", "text_columns.#", "2"), + resource.TestCheckResourceAttr("materialize_source_table.test", "ownership_role", nameSpace+"_role"), + resource.TestCheckResourceAttr("materialize_source_table.test", "comment", "Updated comment"), + ), + }, + }, + }) +} + +func TestAccSourceTable_disappears(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: testAccCheckAllSourceTableDestroyed, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableResource(nameSpace, "table2", "mz_system", ""), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table.test"), + testAccCheckObjectDisappears( + materialize.MaterializeObject{ + ObjectType: "TABLE", + Name: nameSpace + "_table", + }, + ), + ), + PlanOnly: true, + ExpectNonEmptyPlan: true, + }, + }, + }) +} + +func testAccSourceTableBasicResource(nameSpace string) string { + return fmt.Sprintf(` + resource "materialize_secret" "postgres_password" { + name = "%[1]s_secret" + value = "c2VjcmV0Cg==" + } + + resource "materialize_connection_postgres" "postgres_connection" { + name = "%[1]s_connection" + host = "localhost" + port = 5432 + user { + text = "postgres" + } + password { + name = materialize_secret.postgres_password.name + database_name = materialize_secret.postgres_password.database_name + schema_name = materialize_secret.postgres_password.schema_name + } + database = "postgres" + } + + resource "materialize_source_postgres" "test_source" { + name = "%[1]s_source" + cluster_name = "quickstart" + + postgres_connection { + name = materialize_connection_postgres.postgres_connection.name + schema_name = materialize_connection_postgres.postgres_connection.schema_name + database_name = materialize_connection_postgres.postgres_connection.database_name + } + publication = "mz_source" + table { + upstream_name = "table2" + upstream_schema_name = "public" + } + } + + resource "materialize_source_table" "test" { + name = "%[1]s_table" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_postgres.test_source.name + schema_name = "public" + database_name = "materialize" + } + + upstream_name = "table2" + upstream_schema_name = "public" + + text_columns = [ + "updated_at" + ] + } + `, nameSpace) +} + +func testAccSourceTableResource(nameSpace, upstreamName, ownershipRole, comment string) string { + return fmt.Sprintf(` + resource "materialize_secret" "postgres_password" { + name = "%[1]s_secret" + value = "c2VjcmV0Cg==" + } + + resource "materialize_connection_postgres" "postgres_connection" { + name = "%[1]s_connection" + host = "localhost" + port = 5432 + user { + text = "postgres" + } + password { + name = materialize_secret.postgres_password.name + database_name = materialize_secret.postgres_password.database_name + schema_name = materialize_secret.postgres_password.schema_name + } + database = "postgres" + } + + resource "materialize_source_postgres" "test_source" { + name = "%[1]s_source" + cluster_name = "quickstart" + + postgres_connection { + name = materialize_connection_postgres.postgres_connection.name + schema_name = materialize_connection_postgres.postgres_connection.schema_name + database_name = materialize_connection_postgres.postgres_connection.database_name + } + publication = "mz_source" + table { + upstream_name = "%[2]s" + upstream_schema_name = "public" + } + } + + resource "materialize_role" "test_role" { + name = "%[1]s_role" + } + + resource "materialize_source_table" "test" { + name = "%[1]s_table" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_postgres.test_source.name + schema_name = "public" + database_name = "materialize" + } + + upstream_name = "%[2]s" + upstream_schema_name = "public" + + text_columns = [ + "updated_at", + "id" + ] + + ownership_role = "%[3]s" + comment = "%[4]s" + + depends_on = [materialize_role.test_role] + } + `, nameSpace, upstreamName, ownershipRole, comment) +} + +func testAccCheckSourceTableExists(name string) resource.TestCheckFunc { + return func(s *terraform.State) error { + meta := testAccProvider.Meta() + db, _, err := utils.GetDBClientFromMeta(meta, nil) + if err != nil { + return fmt.Errorf("error getting DB client: %s", err) + } + r, ok := s.RootModule().Resources[name] + if !ok { + return fmt.Errorf("source table not found: %s", name) + } + _, err = materialize.ScanSourceTable(db, utils.ExtractId(r.Primary.ID)) + return err + } +} + +func testAccCheckAllSourceTableDestroyed(s *terraform.State) error { + meta := testAccProvider.Meta() + db, _, err := utils.GetDBClientFromMeta(meta, nil) + if err != nil { + return fmt.Errorf("error getting DB client: %s", err) + } + + for _, r := range s.RootModule().Resources { + if r.Type != "materialize_source_table" { + continue + } + + _, err := materialize.ScanSourceTable(db, utils.ExtractId(r.Primary.ID)) + if err == nil { + return fmt.Errorf("source table %v still exists", utils.ExtractId(r.Primary.ID)) + } else if err != sql.ErrNoRows { + return err + } + } + return nil +} diff --git a/pkg/resources/resource_source_table.go b/pkg/resources/resource_source_table.go index 412bf7f6..115bcf6b 100644 --- a/pkg/resources/resource_source_table.go +++ b/pkg/resources/resource_source_table.go @@ -197,8 +197,7 @@ func sourceTableUpdate(ctx context.Context, d *schema.ResourceData, meta any) di } } - // TODO: Handle source changes - // TODO: Handle text_columns changes + // TODO: Handle source and text_columns changes once supported on the Materialize side if d.HasChange("ownership_role") { _, newRole := d.GetChange("ownership_role") diff --git a/pkg/resources/resource_source_table_test.go b/pkg/resources/resource_source_table_test.go new file mode 100644 index 00000000..5876a197 --- /dev/null +++ b/pkg/resources/resource_source_table_test.go @@ -0,0 +1,113 @@ +package resources + +import ( + "context" + "testing" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/stretchr/testify/require" +) + +var inSourceTable = map[string]interface{}{ + "name": "table", + "schema_name": "schema", + "database_name": "database", + "source": []interface{}{ + map[string]interface{}{ + "name": "source", + "schema_name": "public", + "database_name": "materialize", + }, + }, + "upstream_name": "upstream_table", + "upstream_schema_name": "upstream_schema", + "text_columns": []interface{}{"column1", "column2"}, +} + +func TestResourceSourceTableCreate(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTable().Schema, inSourceTable) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."source" + \(REFERENCE "upstream_schema"."upstream_table"\) + WITH \(TEXT COLUMNS \(column1, column2\)\)`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` + testhelpers.MockTableScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockTableScan(mock, pp) + + if err := sourceTableCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableRead(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTable().Schema, inSourceTable) + d.SetId("u1") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockTableScan(mock, pp) + + if err := sourceTableRead(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + + r.Equal("table", d.Get("name").(string)) + r.Equal("schema", d.Get("schema_name").(string)) + r.Equal("database", d.Get("database_name").(string)) + }) +} + +func TestResourceSourceTableUpdate(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTable().Schema, inSourceTable) + d.SetId("u1") + d.Set("name", "old_table") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + mock.ExpectExec(`ALTER TABLE "database"."schema"."" RENAME TO "database"."schema"."table"`).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockTableScan(mock, pp) + + if err := sourceTableUpdate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableDelete(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTable().Schema, inSourceTable) + d.SetId("u1") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + mock.ExpectExec(`DROP TABLE "database"."schema"."table"`).WillReturnResult(sqlmock.NewResult(1, 1)) + + if err := sourceTableDelete(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} From 876caf9f38b04de22542dc6533e16e75536d3c03 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Wed, 4 Sep 2024 20:51:29 +0300 Subject: [PATCH 06/46] Add more tests for mysql and loadgen --- pkg/provider/acceptance_source_table_test.go | 202 +++++++++++++++++-- 1 file changed, 185 insertions(+), 17 deletions(-) diff --git a/pkg/provider/acceptance_source_table_test.go b/pkg/provider/acceptance_source_table_test.go index 26037255..d110b6df 100644 --- a/pkg/provider/acceptance_source_table_test.go +++ b/pkg/provider/acceptance_source_table_test.go @@ -12,7 +12,7 @@ import ( "github.com/hashicorp/terraform-plugin-testing/terraform" ) -func TestAccSourceTable_basic(t *testing.T) { +func TestAccSourceTablePostgres_basic(t *testing.T) { nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, @@ -20,26 +20,63 @@ func TestAccSourceTable_basic(t *testing.T) { CheckDestroy: nil, Steps: []resource.TestStep{ { - Config: testAccSourceTableBasicResource(nameSpace), + Config: testAccSourceTablePostgresBasicResource(nameSpace), Check: resource.ComposeTestCheckFunc( - testAccCheckSourceTableExists("materialize_source_table.test"), - resource.TestMatchResourceAttr("materialize_source_table.test", "id", terraformObjectIdRegex), - resource.TestCheckResourceAttr("materialize_source_table.test", "name", nameSpace+"_table"), - resource.TestCheckResourceAttr("materialize_source_table.test", "database_name", "materialize"), - resource.TestCheckResourceAttr("materialize_source_table.test", "schema_name", "public"), - // resource.TestCheckResourceAttr("materialize_source_table.test", "qualified_sql_name", fmt.Sprintf(`"materialize"."public"."%s_table"`, nameSpace)), - resource.TestCheckResourceAttr("materialize_source_table.test", "text_columns.#", "1"), - resource.TestCheckResourceAttr("materialize_source_table.test", "text_columns.0", "updated_at"), - resource.TestCheckResourceAttr("materialize_source_table.test", "upstream_name", "table2"), - resource.TestCheckResourceAttr("materialize_source_table.test", "upstream_schema_name", "public"), - resource.TestCheckResourceAttr("materialize_source_table.test", "ownership_role", "mz_system"), - resource.TestCheckResourceAttr("materialize_source_table.test", "comment", ""), + testAccCheckSourceTableExists("materialize_source_table.test_postgres"), + resource.TestMatchResourceAttr("materialize_source_table.test_postgres", "id", terraformObjectIdRegex), + resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "name", nameSpace+"_table_postgres"), + resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "database_name", "materialize"), + resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "text_columns.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "text_columns.0", "updated_at"), + resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "upstream_name", "table2"), + resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "upstream_schema_name", "public"), ), }, + }, + }) +} + +func TestAccSourceTableMySQL_basic(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: nil, + Steps: []resource.TestStep{ { - ResourceName: "materialize_source_table.test", - ImportState: true, - ImportStateVerify: false, + Config: testAccSourceTableMySQLBasicResource(nameSpace), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table.test_mysql"), + resource.TestMatchResourceAttr("materialize_source_table.test_mysql", "id", terraformObjectIdRegex), + resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "name", nameSpace+"_table_mysql"), + resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "database_name", "materialize"), + resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "upstream_name", "mysql_table1"), + resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "upstream_schema_name", "shop"), + ), + }, + }, + }) +} + +func TestAccSourceTableLoadGen_basic(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: nil, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableLoadGenBasicResource(nameSpace), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table.test_loadgen"), + resource.TestMatchResourceAttr("materialize_source_table.test_loadgen", "id", terraformObjectIdRegex), + resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "name", nameSpace+"_table_loadgen"), + resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "database_name", "materialize"), + resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "upstream_name", "bids"), + ), }, }, }) @@ -103,6 +140,135 @@ func TestAccSourceTable_disappears(t *testing.T) { }) } +func testAccSourceTablePostgresBasicResource(nameSpace string) string { + return fmt.Sprintf(` + resource "materialize_secret" "postgres_password" { + name = "%[1]s_secret_postgres" + value = "c2VjcmV0Cg==" + } + + resource "materialize_connection_postgres" "postgres_connection" { + name = "%[1]s_connection_postgres" + // TODO: Change with container name once new image is available + host = "localhost" + port = 5432 + user { + text = "postgres" + } + password { + name = materialize_secret.postgres_password.name + } + database = "postgres" + } + + resource "materialize_source_postgres" "test_source_postgres" { + name = "%[1]s_source_postgres" + cluster_name = "quickstart" + + postgres_connection { + name = materialize_connection_postgres.postgres_connection.name + } + publication = "mz_source" + table { + upstream_name = "table2" + upstream_schema_name = "public" + } + } + + resource "materialize_source_table" "test_postgres" { + name = "%[1]s_table_postgres" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_postgres.test_source_postgres.name + } + + upstream_name = "table2" + upstream_schema_name = "public" + + text_columns = [ + "updated_at" + ] + } + `, nameSpace) +} + +func testAccSourceTableMySQLBasicResource(nameSpace string) string { + return fmt.Sprintf(` + resource "materialize_secret" "mysql_password" { + name = "%[1]s_secret_mysql" + value = "c2VjcmV0Cg==" + } + + resource "materialize_connection_mysql" "mysql_connection" { + name = "%[1]s_connection_mysql" + // TODO: Change with container name once new image is available + host = "localhost" + port = 3306 + user { + text = "repluser" + } + password { + name = materialize_secret.mysql_password.name + } + } + + resource "materialize_source_mysql" "test_source_mysql" { + name = "%[1]s_source_mysql" + cluster_name = "quickstart" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } + + table { + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + name = "mysql_table1_local" + } + } + + resource "materialize_source_table" "test_mysql" { + name = "%[1]s_table_mysql" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.test_source_mysql.name + } + + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + } + `, nameSpace) +} + +func testAccSourceTableLoadGenBasicResource(nameSpace string) string { + return fmt.Sprintf(` + resource "materialize_source_load_generator" "test_loadgen" { + name = "%[1]s_loadgen" + load_generator_type = "AUCTION" + + auction_options { + tick_interval = "500ms" + } + } + + resource "materialize_source_table" "test_loadgen" { + name = "%[1]s_table_loadgen" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_load_generator.test_loadgen.name + } + + upstream_name = "bids" + } + `, nameSpace) +} + func testAccSourceTableBasicResource(nameSpace string) string { return fmt.Sprintf(` resource "materialize_secret" "postgres_password" { @@ -112,6 +278,7 @@ func testAccSourceTableBasicResource(nameSpace string) string { resource "materialize_connection_postgres" "postgres_connection" { name = "%[1]s_connection" + // TODO: Change with container name once new image is available host = "localhost" port = 5432 user { @@ -171,6 +338,7 @@ func testAccSourceTableResource(nameSpace, upstreamName, ownershipRole, comment resource "materialize_connection_postgres" "postgres_connection" { name = "%[1]s_connection" + // TODO: Change with container name once new image is available host = "localhost" port = 5432 user { From 3acaa249c9356719eaa0defa2488a7be2c77437b Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Wed, 4 Sep 2024 23:38:14 +0300 Subject: [PATCH 07/46] Add ignore columns for MySQL --- docs/resources/source_load_generator.md | 2 +- docs/resources/source_mysql.md | 2 +- docs/resources/source_table.md | 1 + pkg/materialize/source_table.go | 24 +++++++++++++++++-- pkg/provider/acceptance_source_table_test.go | 7 +++++- .../resource_source_load_generator.go | 2 +- pkg/resources/resource_source_mysql.go | 1 + pkg/resources/resource_source_table.go | 14 +++++++++++ pkg/resources/resource_source_table_test.go | 3 ++- 9 files changed, 49 insertions(+), 7 deletions(-) diff --git a/docs/resources/source_load_generator.md b/docs/resources/source_load_generator.md index 99db6947..4dddf435 100644 --- a/docs/resources/source_load_generator.md +++ b/docs/resources/source_load_generator.md @@ -40,7 +40,7 @@ resource "materialize_source_load_generator" "example_source_load_generator" { ### Optional -- `all_tables` (Boolean) Whether to include all tables in the source. Compatible with `auction_options`, `marketing_options`, and `tpch_options`. IF not specified, use the `materialize_source_table` resource to specify tables to include. +- `all_tables` (Boolean) Whether to include all tables in the source. Compatible with `auction_options`, `marketing_options`, and `tpch_options`. If not specified, use the `materialize_source_table` resource to specify tables to include. - `auction_options` (Block List, Max: 1) Auction Options. (see [below for nested schema](#nestedblock--auction_options)) - `cluster_name` (String) The cluster to maintain this source. - `comment` (String) Comment on an object in the database. diff --git a/docs/resources/source_mysql.md b/docs/resources/source_mysql.md index 0e3e3c1f..0e279236 100644 --- a/docs/resources/source_mysql.md +++ b/docs/resources/source_mysql.md @@ -57,7 +57,7 @@ resource "materialize_source_mysql" "test" { - `comment` (String) Comment on an object in the database. - `database_name` (String) The identifier for the source database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `expose_progress` (Block List, Max: 1) The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`. (see [below for nested schema](#nestedblock--expose_progress)) -- `ignore_columns` (List of String) Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. +- `ignore_columns` (List of String, Deprecated) Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. diff --git a/docs/resources/source_table.md b/docs/resources/source_table.md index b93470b4..3ddcadce 100644 --- a/docs/resources/source_table.md +++ b/docs/resources/source_table.md @@ -25,6 +25,7 @@ description: |- - `comment` (String) **Public Preview** Comment on an object in the database. - `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `ignore_columns` (List of String) Ignore specific columns when reading data from MySQL. Only compatible with MySQL sources, if the source is not MySQL, the attribute will be ignored. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the table schema in Materialize. Defaults to `public`. diff --git a/pkg/materialize/source_table.go b/pkg/materialize/source_table.go index 520df928..3268d348 100644 --- a/pkg/materialize/source_table.go +++ b/pkg/materialize/source_table.go @@ -87,6 +87,7 @@ type SourceTableBuilder struct { upstreamName string upstreamSchemaName string textColumns []string + ignoreColumns []string } func NewSourceTableBuilder(conn *sqlx.DB, obj MaterializeObject) *SourceTableBuilder { @@ -122,6 +123,11 @@ func (b *SourceTableBuilder) TextColumns(c []string) *SourceTableBuilder { return b } +func (b *SourceTableBuilder) IgnoreColumns(c []string) *SourceTableBuilder { + b.ignoreColumns = c + return b +} + func (b *SourceTableBuilder) Create() error { q := strings.Builder{} q.WriteString(fmt.Sprintf(`CREATE TABLE %s`, b.QualifiedName())) @@ -135,9 +141,23 @@ func (b *SourceTableBuilder) Create() error { q.WriteString(")") + var options []string + if len(b.textColumns) > 0 { - c := strings.Join(b.textColumns, ", ") - q.WriteString(fmt.Sprintf(` WITH (TEXT COLUMNS [%s])`, c)) + s := strings.Join(b.textColumns, ", ") + options = append(options, fmt.Sprintf(`TEXT COLUMNS (%s)`, s)) + } + + // TODO: Implement logic to only use IGNORE COLUMNS if the source is a MySQL source + if len(b.ignoreColumns) > 0 { + s := strings.Join(b.ignoreColumns, ", ") + options = append(options, fmt.Sprintf(`IGNORE COLUMNS (%s)`, s)) + } + + if len(options) > 0 { + q.WriteString(" WITH (") + q.WriteString(strings.Join(options, ", ")) + q.WriteString(")") } q.WriteString(`;`) diff --git a/pkg/provider/acceptance_source_table_test.go b/pkg/provider/acceptance_source_table_test.go index d110b6df..a557adc7 100644 --- a/pkg/provider/acceptance_source_table_test.go +++ b/pkg/provider/acceptance_source_table_test.go @@ -54,6 +54,10 @@ func TestAccSourceTableMySQL_basic(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "schema_name", "public"), resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "upstream_name", "mysql_table1"), resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "upstream_schema_name", "shop"), + resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "ignore_columns.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "ignore_columns.0", "banned"), + // resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "text_columns.#", "1"), + // resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "text_columns.0", "about"), ), }, }, @@ -221,7 +225,7 @@ func testAccSourceTableMySQLBasicResource(nameSpace string) string { mysql_connection { name = materialize_connection_mysql.mysql_connection.name } - + table { upstream_name = "mysql_table1" upstream_schema_name = "shop" @@ -240,6 +244,7 @@ func testAccSourceTableMySQLBasicResource(nameSpace string) string { upstream_name = "mysql_table1" upstream_schema_name = "shop" + ignore_columns = ["banned"] } `, nameSpace) } diff --git a/pkg/resources/resource_source_load_generator.go b/pkg/resources/resource_source_load_generator.go index 58febe32..0cd4ef8a 100644 --- a/pkg/resources/resource_source_load_generator.go +++ b/pkg/resources/resource_source_load_generator.go @@ -175,7 +175,7 @@ var sourceLoadgenSchema = map[string]*schema.Schema{ ConflictsWith: []string{"counter_options", "auction_options", "marketing_options", "tpch_options"}, }, "all_tables": { - Description: "Whether to include all tables in the source. Compatible with `auction_options`, `marketing_options`, and `tpch_options`. IF not specified, use the `materialize_source_table` resource to specify tables to include.", + Description: "Whether to include all tables in the source. Compatible with `auction_options`, `marketing_options`, and `tpch_options`. If not specified, use the `materialize_source_table` resource to specify tables to include.", Type: schema.TypeBool, Optional: true, Default: false, diff --git a/pkg/resources/resource_source_mysql.go b/pkg/resources/resource_source_mysql.go index 275520eb..64266c76 100644 --- a/pkg/resources/resource_source_mysql.go +++ b/pkg/resources/resource_source_mysql.go @@ -28,6 +28,7 @@ var sourceMySQLSchema = map[string]*schema.Schema{ }), "ignore_columns": { Description: "Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute.", + Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, diff --git a/pkg/resources/resource_source_table.go b/pkg/resources/resource_source_table.go index 115bcf6b..063fb103 100644 --- a/pkg/resources/resource_source_table.go +++ b/pkg/resources/resource_source_table.go @@ -42,6 +42,12 @@ var sourceTableSchema = map[string]*schema.Schema{ Optional: true, ForceNew: true, }, + "ignore_columns": { + Description: "Ignore specific columns when reading data from MySQL. Only compatible with MySQL sources, if the source is not MySQL, the attribute will be ignored.", + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + }, "comment": CommentSchema(false), "ownership_role": OwnershipRoleSchema(), "region": RegionSchema(), @@ -92,6 +98,14 @@ func sourceTableCreate(ctx context.Context, d *schema.ResourceData, meta any) di b.TextColumns(textColumns) } + if v, ok := d.GetOk("ignore_columns"); ok && len(v.([]interface{})) > 0 { + columns, err := materialize.GetSliceValueString("ignore_columns", v.([]interface{})) + if err != nil { + return diag.FromErr(err) + } + b.IgnoreColumns(columns) + } + if err := b.Create(); err != nil { return diag.FromErr(err) } diff --git a/pkg/resources/resource_source_table_test.go b/pkg/resources/resource_source_table_test.go index 5876a197..b912b9e9 100644 --- a/pkg/resources/resource_source_table_test.go +++ b/pkg/resources/resource_source_table_test.go @@ -26,6 +26,7 @@ var inSourceTable = map[string]interface{}{ "upstream_name": "upstream_table", "upstream_schema_name": "upstream_schema", "text_columns": []interface{}{"column1", "column2"}, + "ignore_columns": []interface{}{"column3", "column4"}, } func TestResourceSourceTableCreate(t *testing.T) { @@ -39,7 +40,7 @@ func TestResourceSourceTableCreate(t *testing.T) { `CREATE TABLE "database"."schema"."table" FROM SOURCE "materialize"."public"."source" \(REFERENCE "upstream_schema"."upstream_table"\) - WITH \(TEXT COLUMNS \(column1, column2\)\)`, + WITH \(TEXT COLUMNS \(column1, column2\), IGNORE COLUMNS \(column3, column4\)\);`, ).WillReturnResult(sqlmock.NewResult(1, 1)) // Query Id From 9e512e1895d2fbcd8c505d653a1978e9deed060d Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 9 Sep 2024 14:41:49 +0300 Subject: [PATCH 08/46] Add source_id logic --- pkg/materialize/source_table.go | 43 ++++++++++- pkg/materialize/source_table_test.go | 47 ++++++++++++ pkg/provider/acceptance_source_table_test.go | 31 ++++++-- pkg/resources/resource_source_table.go | 13 +++- pkg/resources/resource_source_table_test.go | 49 ++++++++++-- pkg/testhelpers/mock_scans.go | 80 ++++++++++++++++++++ 6 files changed, 245 insertions(+), 18 deletions(-) diff --git a/pkg/materialize/source_table.go b/pkg/materialize/source_table.go index 3268d348..287660b5 100644 --- a/pkg/materialize/source_table.go +++ b/pkg/materialize/source_table.go @@ -17,6 +17,7 @@ type SourceTableParams struct { SourceName sql.NullString `db:"source_name"` SourceSchemaName sql.NullString `db:"source_schema_name"` SourceDatabaseName sql.NullString `db:"source_database_name"` + SourceType sql.NullString `db:"source_type"` UpstreamName sql.NullString `db:"upstream_name"` UpstreamSchemaName sql.NullString `db:"upstream_schema_name"` TextColumns pq.StringArray `db:"text_columns"` @@ -32,6 +33,10 @@ var sourceTableQuery = NewBaseQuery(` mz_tables.name, mz_schemas.name AS schema_name, mz_databases.name AS database_name, + mz_sources.name AS source_name, + source_schemas.name AS source_schema_name, + source_databases.name AS source_database_name, + mz_sources.type AS source_type, comments.comment AS comment, mz_roles.name AS owner_name, mz_tables.privileges @@ -40,6 +45,12 @@ var sourceTableQuery = NewBaseQuery(` ON mz_tables.schema_id = mz_schemas.id JOIN mz_databases ON mz_schemas.database_id = mz_databases.id + JOIN mz_sources + ON mz_tables.source_id = mz_sources.id + JOIN mz_schemas AS source_schemas + ON mz_sources.schema_id = source_schemas.id + JOIN mz_databases AS source_databases + ON source_schemas.database_id = source_databases.id JOIN mz_roles ON mz_tables.owner_id = mz_roles.id LEFT JOIN ( @@ -88,6 +99,8 @@ type SourceTableBuilder struct { upstreamSchemaName string textColumns []string ignoreColumns []string + sourceType string + conn *sqlx.DB } func NewSourceTableBuilder(conn *sqlx.DB, obj MaterializeObject) *SourceTableBuilder { @@ -96,9 +109,31 @@ func NewSourceTableBuilder(conn *sqlx.DB, obj MaterializeObject) *SourceTableBui tableName: obj.Name, schemaName: obj.SchemaName, databaseName: obj.DatabaseName, + conn: conn, } } +func (b *SourceTableBuilder) GetSourceType() (string, error) { + if b.sourceType != "" { + return b.sourceType, nil + } + + q := sourceQuery.QueryPredicate(map[string]string{ + "mz_sources.name": b.source.Name, + "mz_schemas.name": b.source.SchemaName, + "mz_databases.name": b.source.DatabaseName, + }) + + var s SourceParams + if err := b.conn.Get(&s, q); err != nil { + return "", err + } + + b.sourceType = s.SourceType.String + + return b.sourceType, nil +} + func (b *SourceTableBuilder) QualifiedName() string { return QualifiedName(b.databaseName, b.schemaName, b.tableName) } @@ -148,8 +183,12 @@ func (b *SourceTableBuilder) Create() error { options = append(options, fmt.Sprintf(`TEXT COLUMNS (%s)`, s)) } - // TODO: Implement logic to only use IGNORE COLUMNS if the source is a MySQL source - if len(b.ignoreColumns) > 0 { + sourceType, err := b.GetSourceType() + if err != nil { + return err + } + + if strings.EqualFold(sourceType, "mysql") && len(b.ignoreColumns) > 0 { s := strings.Join(b.ignoreColumns, ", ") options = append(options, fmt.Sprintf(`IGNORE COLUMNS (%s)`, s)) } diff --git a/pkg/materialize/source_table_test.go b/pkg/materialize/source_table_test.go index 384fe815..79360f07 100644 --- a/pkg/materialize/source_table_test.go +++ b/pkg/materialize/source_table_test.go @@ -12,6 +12,9 @@ var sourceTable = MaterializeObject{Name: "table", SchemaName: "schema", Databas func TestSourceTableCreate(t *testing.T) { testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` + testhelpers.MockSourceScanWithType(mock, sourceTypeQuery, "kafka") + mock.ExpectExec( `CREATE TABLE "database"."schema"."table" FROM SOURCE "materialize"."public"."source" @@ -31,6 +34,50 @@ func TestSourceTableCreate(t *testing.T) { }) } +func TestSourceTableCreateWithMySQLSource(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` + testhelpers.MockSourceScanWithType(mock, sourceTypeQuery, "mysql") + + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."source" + \(REFERENCE "upstream_schema"."upstream_table"\) + WITH \(TEXT COLUMNS \(column1, column2\), IGNORE COLUMNS \(ignore1, ignore2\)\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableBuilder(db, sourceTable) + b.Source(IdentifierSchemaStruct{Name: "source", SchemaName: "public", DatabaseName: "materialize"}) + b.UpstreamName("upstream_table") + b.UpstreamSchemaName("upstream_schema") + b.TextColumns([]string{"column1", "column2"}) + b.IgnoreColumns([]string{"ignore1", "ignore2"}) + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestGetSourceType(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` + testhelpers.MockSourceScanWithType(mock, sourceTypeQuery, "KAFKA") + + b := NewSourceTableBuilder(db, sourceTable) + b.Source(IdentifierSchemaStruct{Name: "source", SchemaName: "public", DatabaseName: "materialize"}) + + sourceType, err := b.GetSourceType() + if err != nil { + t.Fatal(err) + } + + if sourceType != "KAFKA" { + t.Fatalf("Expected source type 'KAFKA', got '%s'", sourceType) + } + }) +} + func TestSourceTableRename(t *testing.T) { testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { mock.ExpectExec( diff --git a/pkg/provider/acceptance_source_table_test.go b/pkg/provider/acceptance_source_table_test.go index a557adc7..de133107 100644 --- a/pkg/provider/acceptance_source_table_test.go +++ b/pkg/provider/acceptance_source_table_test.go @@ -31,6 +31,10 @@ func TestAccSourceTablePostgres_basic(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "text_columns.0", "updated_at"), resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "upstream_name", "table2"), resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "upstream_schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "source.0.name", nameSpace+"_source_postgres"), + resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "source.0.schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "source.0.database_name", "materialize"), ), }, }, @@ -56,6 +60,10 @@ func TestAccSourceTableMySQL_basic(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "upstream_schema_name", "shop"), resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "ignore_columns.#", "1"), resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "ignore_columns.0", "banned"), + resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "source.0.name", nameSpace+"_source_mysql"), + resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "source.0.schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "source.0.database_name", "materialize"), // resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "text_columns.#", "1"), // resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "text_columns.0", "about"), ), @@ -80,6 +88,10 @@ func TestAccSourceTableLoadGen_basic(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "database_name", "materialize"), resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "schema_name", "public"), resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "upstream_name", "bids"), + resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "source.0.name", nameSpace+"_loadgen"), + resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "source.0.schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "source.0.database_name", "materialize"), ), }, }, @@ -102,6 +114,10 @@ func TestAccSourceTable_update(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table.test", "text_columns.#", "2"), resource.TestCheckResourceAttr("materialize_source_table.test", "ownership_role", "mz_system"), resource.TestCheckResourceAttr("materialize_source_table.test", "comment", ""), + resource.TestCheckResourceAttr("materialize_source_table.test", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table.test", "source.0.name", nameSpace+"_source"), + resource.TestCheckResourceAttr("materialize_source_table.test", "source.0.schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table.test", "source.0.database_name", "materialize"), ), }, { @@ -113,6 +129,9 @@ func TestAccSourceTable_update(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table.test", "text_columns.#", "2"), resource.TestCheckResourceAttr("materialize_source_table.test", "ownership_role", nameSpace+"_role"), resource.TestCheckResourceAttr("materialize_source_table.test", "comment", "Updated comment"), + resource.TestCheckResourceAttr("materialize_source_table.test", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table.test", "source.0.name", nameSpace+"_source"), + resource.TestCheckResourceAttr("materialize_source_table.test", "source.0.schema_name", "public"), ), }, }, @@ -153,8 +172,7 @@ func testAccSourceTablePostgresBasicResource(nameSpace string) string { resource "materialize_connection_postgres" "postgres_connection" { name = "%[1]s_connection_postgres" - // TODO: Change with container name once new image is available - host = "localhost" + host = "postgres" port = 5432 user { text = "postgres" @@ -207,8 +225,7 @@ func testAccSourceTableMySQLBasicResource(nameSpace string) string { resource "materialize_connection_mysql" "mysql_connection" { name = "%[1]s_connection_mysql" - // TODO: Change with container name once new image is available - host = "localhost" + host = "mysql" port = 3306 user { text = "repluser" @@ -283,8 +300,7 @@ func testAccSourceTableBasicResource(nameSpace string) string { resource "materialize_connection_postgres" "postgres_connection" { name = "%[1]s_connection" - // TODO: Change with container name once new image is available - host = "localhost" + host = "postgres" port = 5432 user { text = "postgres" @@ -343,8 +359,7 @@ func testAccSourceTableResource(nameSpace, upstreamName, ownershipRole, comment resource "materialize_connection_postgres" "postgres_connection" { name = "%[1]s_connection" - // TODO: Change with container name once new image is available - host = "localhost" + host = "postgres" port = 5432 user { text = "postgres" diff --git a/pkg/resources/resource_source_table.go b/pkg/resources/resource_source_table.go index 063fb103..93291025 100644 --- a/pkg/resources/resource_source_table.go +++ b/pkg/resources/resource_source_table.go @@ -47,6 +47,7 @@ var sourceTableSchema = map[string]*schema.Schema{ Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, + ForceNew: true, }, "comment": CommentSchema(false), "ownership_role": OwnershipRoleSchema(), @@ -169,8 +170,18 @@ func sourceTableRead(ctx context.Context, d *schema.ResourceData, meta interface return diag.FromErr(err) } - // TODO: Set source once the source_id is available in the mz_tables table + source := []interface{}{ + map[string]interface{}{ + "name": t.SourceName.String, + "schema_name": t.SourceSchemaName.String, + "database_name": t.SourceDatabaseName.String, + }, + } + if err := d.Set("source", source); err != nil { + return diag.FromErr(err) + } + // TODO: Set the upstream_name and upstream_schema_name once supported on the Materialize side // if err := d.Set("upstream_name", t.UpstreamName.String); err != nil { // return diag.FromErr(err) // } diff --git a/pkg/resources/resource_source_table_test.go b/pkg/resources/resource_source_table_test.go index b912b9e9..2262bc24 100644 --- a/pkg/resources/resource_source_table_test.go +++ b/pkg/resources/resource_source_table_test.go @@ -35,21 +35,56 @@ func TestResourceSourceTableCreate(t *testing.T) { r.NotNil(d) testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Expect source type query + sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` + testhelpers.MockSourceScanWithType(mock, sourceTypeQuery, "mysql") + // Create mock.ExpectExec( `CREATE TABLE "database"."schema"."table" - FROM SOURCE "materialize"."public"."source" - \(REFERENCE "upstream_schema"."upstream_table"\) - WITH \(TEXT COLUMNS \(column1, column2\), IGNORE COLUMNS \(column3, column4\)\);`, + FROM SOURCE "materialize"."public"."source" + \(REFERENCE "upstream_schema"."upstream_table"\) + WITH \(TEXT COLUMNS \(column1, column2\), IGNORE COLUMNS \(column3, column4\)\);`, ).WillReturnResult(sqlmock.NewResult(1, 1)) // Query Id ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` - testhelpers.MockTableScan(mock, ip) + testhelpers.MockSourceTableScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableCreateNonMySQL(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTable().Schema, inSourceTable) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Expect source type query + sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` + testhelpers.MockSourceScan(mock, sourceTypeQuery) + + // Create (without IGNORE COLUMNS) + mock.ExpectExec(`CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."source" + \(REFERENCE "upstream_schema"."upstream_table"\) + WITH \(TEXT COLUMNS \(column1, column2\)\);`). + WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` + testhelpers.MockSourceTableScan(mock, ip) // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockTableScan(mock, pp) + testhelpers.MockSourceTableScan(mock, pp) if err := sourceTableCreate(context.TODO(), d, db); err != nil { t.Fatal(err) @@ -66,7 +101,7 @@ func TestResourceSourceTableRead(t *testing.T) { testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockTableScan(mock, pp) + testhelpers.MockSourceTableScan(mock, pp) if err := sourceTableRead(context.TODO(), d, db); err != nil { t.Fatal(err) @@ -90,7 +125,7 @@ func TestResourceSourceTableUpdate(t *testing.T) { // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockTableScan(mock, pp) + testhelpers.MockSourceTableScan(mock, pp) if err := sourceTableUpdate(context.TODO(), d, db); err != nil { t.Fatal(err) diff --git a/pkg/testhelpers/mock_scans.go b/pkg/testhelpers/mock_scans.go index 9107bd91..4abed015 100644 --- a/pkg/testhelpers/mock_scans.go +++ b/pkg/testhelpers/mock_scans.go @@ -569,6 +569,45 @@ func MockSourceScan(mock sqlmock.Sqlmock, predicate string) { mock.ExpectQuery(q).WillReturnRows(ir) } +func MockSourceScanWithType(mock sqlmock.Sqlmock, predicate string, sourceType string) { + b := ` + SELECT + mz_sources.id, + mz_sources.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.type AS source_type, + COALESCE\(mz_sources.size, mz_clusters.size\) AS size, + mz_sources.envelope_type, + mz_connections.name as connection_name, + mz_clusters.name as cluster_name, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_sources.privileges + FROM mz_sources + JOIN mz_schemas + ON mz_sources.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + LEFT JOIN mz_connections + ON mz_sources.connection_id = mz_connections.id + LEFT JOIN mz_clusters + ON mz_sources.cluster_id = mz_clusters.id + JOIN mz_roles + ON mz_sources.owner_id = mz_roles.id + LEFT JOIN \( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'source' + \) comments + ON mz_sources.id = comments.id` + + q := mockQueryBuilder(b, predicate, "") + ir := mock.NewRows([]string{"id", "name", "schema_name", "database_name", "source_type", "size", "envelope_type", "connection_name", "cluster_name", "owner_name", "privileges"}). + AddRow("u1", "source", "schema", "database", sourceType, "small", "BYTES", "conn", "cluster", "joe", defaultPrivilege) + mock.ExpectQuery(q).WillReturnRows(ir) +} + func MockSubsourceScan(mock sqlmock.Sqlmock, predicate string) { b := ` WITH dependencies AS \( @@ -736,6 +775,47 @@ func MockTableScan(mock sqlmock.Sqlmock, predicate string) { mock.ExpectQuery(q).WillReturnRows(ir) } +func MockSourceTableScan(mock sqlmock.Sqlmock, predicate string) { + b := ` + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.name AS source_name, + source_schemas.name AS source_schema_name, + source_databases.name AS source_database_name, + mz_sources.type AS source_type, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_sources + ON mz_tables.source_id = mz_sources.id + JOIN mz_schemas AS source_schemas + ON mz_sources.schema_id = source_schemas.id + JOIN mz_databases AS source_databases + ON source_schemas.database_id = source_databases.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN \( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + \) comments + ON mz_tables.id = comments.id` + + q := mockQueryBuilder(b, predicate, "") + ir := mock.NewRows([]string{"id", "name", "schema_name", "database_name", "source_name", "source_schema_name", "source_database_name", "source_type", "comment", "owner_name", "privileges"}). + AddRow("u1", "table", "schema", "database", "source", "public", "materialize", "KAFKA", "comment", "materialize", defaultPrivilege) + mock.ExpectQuery(q).WillReturnRows(ir) +} + func MockTypeScan(mock sqlmock.Sqlmock, predicate string) { b := ` SELECT From 37b51a7dd8d20e82898e8760ad649202fc56d642 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 9 Sep 2024 16:26:32 +0300 Subject: [PATCH 09/46] Add source table migration guide --- docs/guides/materialize_source_table.md | 181 ++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 docs/guides/materialize_source_table.md diff --git a/docs/guides/materialize_source_table.md b/docs/guides/materialize_source_table.md new file mode 100644 index 00000000..eaf224f5 --- /dev/null +++ b/docs/guides/materialize_source_table.md @@ -0,0 +1,181 @@ +--- +page_title: "Source versioning: migrating to `materialize_source_table` Resource" +subcategory: "" +description: |- + +--- + +# Source versioning: migrating to `materialize_source_table` Resource + +In previous versions of the Materialize Terraform provider, source tables were defined within the source resource itself and were considered subsources of the source rather than separate entities. + +This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table` resource. + +## Old Approach + +Previously, source tables were defined directly within the source resource: + +### Example: MySQL Source + +```hcl +resource "materialize_source_mysql" "mysql_source" { + name = "mysql_source" + cluster_name = "cluster_name" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } + + table { + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + name = "mysql_table1_local" + } +} +``` + +The same approach was used for other source types such as Postgres and the load generator sources. + +## New Approach + +The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table` resource. + +## Manual Migration Process + +This manual migration process requires users to create new source tables using the new `materialize_source_table` resource and then remove the old ones. + +### Step 1: Define `materialize_source_table` Resources + +Before making any changes to your existing source resources, create new `materialize_source_table` resources for each table that is currently defined within your sources. This ensures that the tables are preserved during the migration: + +```hcl +resource "materialize_source_table" "mysql_table_from_source" { + name = "mysql_table1_from_source" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.mysql_source.name + // Define the schema and database for the source if needed + } + + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + + ignore_columns = ["about"] +} +``` + +### Step 2: Apply the Changes + +Run `terraform plan` and `terraform apply` to create the new `materialize_source_table` resources. This step ensures that the tables are defined separately from the source and are not removed from Materialize. + +> **Note:** This will start an ingestion process for the newly created source tables. + +### Step 3: Remove Table Blocks from Source Resources + +Once the new `materialize_source_table` resources are successfully created, you can safely remove the `table` blocks from your existing source resources: + +```hcl +resource "materialize_source_mysql" "mysql_source" { + name = "mysql_source" + cluster_name = "cluster_name" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } +} +``` + +This will drop the old tables from the source resources. + +### Step 4: Update Terraform State + +After removing the `table` blocks from your source resources, run `terraform plan` and `terraform apply` again to update the Terraform state and apply the changes. + +### Step 5: Verify the Migration + +After applying the changes, verify that your tables are still correctly set up in Materialize by checking the table definitions using Materialize’s SQL commands. + +During the migration, you can use both the old `table` blocks and the new `materialize_source_table` resources simultaneously. This allows for a gradual transition until the old method is fully deprecated. + +## Automated Migration Process (TBD) + +> **Note:** This will still not work as the previous source tables are considered subsources of the source and are missing from the `mz_tables` table in Materialize so we can't import them directly without recreating them. + +Once the migration on the Materialize side has been implemented, a more automated migration process will be available. The steps will include: + +### Step 1: Define `materialize_source_table` Resources + +First, define the new `materialize_source_table` resources for each table: + +```hcl +resource "materialize_source_table" "mysql_table_from_source" { + name = "mysql_table1_from_source" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.mysql_source.name + // Define the schema and database for the source if needed + } + + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + + ignore_columns = ["about"] +} +``` + +### Step 2: Modify the Existing Source Resource + +Next, modify the existing source resource by removing the `table` blocks and adding an `ignore_changes` directive for the `table` attribute. This prevents Terraform from trying to delete the tables: + +```hcl +resource "materialize_source_mysql" "mysql_source" { + name = "mysql_source" + cluster_name = "cluster_name" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } + + lifecycle { + ignore_changes = [table] + } +} +``` + +- **`lifecycle { ignore_changes = [table] }`**: This directive tells Terraform to ignore changes to the `table` attribute, preventing it from trying to delete tables that were previously defined in the source resource. + +### Step 3: Import the Existing Tables + +You can then import the existing tables into the new `materialize_source_table` resources without disrupting your existing setup: + +```bash +terraform import materialize_source_table.mysql_table_from_source : +``` + +Replace `` with the actual region and `` with the table ID. You can find the table ID by querying the `mz_tables` table. + +### Step 4: Run Terraform Plan and Apply + +Finally, run `terraform plan` and `terraform apply` to ensure that everything is correctly set up without triggering any unwanted deletions. + +This approach allows you to migrate your tables safely without disrupting your existing setup. + +## Importing Existing Tables + +To import existing tables into your Terraform state using the manual migration process, use the following command: + +```bash +terraform import materialize_source_table.table_name : +``` + +Ensure you replace `` with the region where the table is located and `` with the ID of the table. + +> **Note:** The `upstream_name` and `upstream_schema_name` attributes are not yet implemented on the Materialize side, so the import process will not work until these changes are made. + +## Future Improvements + +The Kafka and Webhooks sources are currently being implemented. Once these changes, the migration process will be updated to include them. From bc0db2dbb3df6c6fce3ebfc8eed88e575898c0fa Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 9 Sep 2024 16:32:51 +0300 Subject: [PATCH 10/46] Add deprecated message --- docs/resources/source_mysql.md | 6 +- docs/resources/source_postgres.md | 4 +- pkg/resources/resource_source_mysql.go | 6 +- pkg/resources/resource_source_postgres.go | 4 +- .../guides/materialize_source_table.md.tmpl | 181 ++++++++++++++++++ 5 files changed, 191 insertions(+), 10 deletions(-) create mode 100644 templates/guides/materialize_source_table.md.tmpl diff --git a/docs/resources/source_mysql.md b/docs/resources/source_mysql.md index 0e279236..67559839 100644 --- a/docs/resources/source_mysql.md +++ b/docs/resources/source_mysql.md @@ -57,12 +57,12 @@ resource "materialize_source_mysql" "test" { - `comment` (String) Comment on an object in the database. - `database_name` (String) The identifier for the source database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `expose_progress` (Block List, Max: 1) The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`. (see [below for nested schema](#nestedblock--expose_progress)) -- `ignore_columns` (List of String, Deprecated) Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. +- `ignore_columns` (List of String, Deprecated) Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `table` (Block Set, Deprecated) Specify the tables to be included in the source. If not specified, all tables are included. (see [below for nested schema](#nestedblock--table)) -- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. +- `table` (Block Set, Deprecated) Specify the tables to be included in the source. Deprecated: Use the new materialize_source_table resource instead. (see [below for nested schema](#nestedblock--table)) +- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead. ### Read-Only diff --git a/docs/resources/source_postgres.md b/docs/resources/source_postgres.md index 78314094..c98e17d8 100644 --- a/docs/resources/source_postgres.md +++ b/docs/resources/source_postgres.md @@ -62,8 +62,8 @@ resource "materialize_source_postgres" "example_source_postgres" { - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `table` (Block Set, Deprecated) Creates subsources for specific tables in the Postgres connection. (see [below for nested schema](#nestedblock--table)) -- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. +- `table` (Block Set, Deprecated) Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new materialize_source_table resource instead. (see [below for nested schema](#nestedblock--table)) +- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead. ### Read-Only diff --git a/pkg/resources/resource_source_mysql.go b/pkg/resources/resource_source_mysql.go index 64266c76..01dc5dd1 100644 --- a/pkg/resources/resource_source_mysql.go +++ b/pkg/resources/resource_source_mysql.go @@ -27,21 +27,21 @@ var sourceMySQLSchema = map[string]*schema.Schema{ ForceNew: true, }), "ignore_columns": { - Description: "Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute.", + Description: "Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead.", Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "text_columns": { - Description: "Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute.", + Description: "Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead.", Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "table": { - Description: "Specify the tables to be included in the source. If not specified, all tables are included.", + Description: "Specify the tables to be included in the source. Deprecated: Use the new materialize_source_table resource instead.", Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeSet, Optional: true, diff --git a/pkg/resources/resource_source_postgres.go b/pkg/resources/resource_source_postgres.go index 9b057a22..bea0cad5 100644 --- a/pkg/resources/resource_source_postgres.go +++ b/pkg/resources/resource_source_postgres.go @@ -33,14 +33,14 @@ var sourcePostgresSchema = map[string]*schema.Schema{ ForceNew: true, }, "text_columns": { - Description: "Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute.", + Description: "Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead.", Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "table": { - Description: "Creates subsources for specific tables in the Postgres connection.", + Description: "Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new materialize_source_table resource instead.", Deprecated: "Use the new materialize_source_table resource instead.", Type: schema.TypeSet, Optional: true, diff --git a/templates/guides/materialize_source_table.md.tmpl b/templates/guides/materialize_source_table.md.tmpl new file mode 100644 index 00000000..eaf224f5 --- /dev/null +++ b/templates/guides/materialize_source_table.md.tmpl @@ -0,0 +1,181 @@ +--- +page_title: "Source versioning: migrating to `materialize_source_table` Resource" +subcategory: "" +description: |- + +--- + +# Source versioning: migrating to `materialize_source_table` Resource + +In previous versions of the Materialize Terraform provider, source tables were defined within the source resource itself and were considered subsources of the source rather than separate entities. + +This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table` resource. + +## Old Approach + +Previously, source tables were defined directly within the source resource: + +### Example: MySQL Source + +```hcl +resource "materialize_source_mysql" "mysql_source" { + name = "mysql_source" + cluster_name = "cluster_name" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } + + table { + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + name = "mysql_table1_local" + } +} +``` + +The same approach was used for other source types such as Postgres and the load generator sources. + +## New Approach + +The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table` resource. + +## Manual Migration Process + +This manual migration process requires users to create new source tables using the new `materialize_source_table` resource and then remove the old ones. + +### Step 1: Define `materialize_source_table` Resources + +Before making any changes to your existing source resources, create new `materialize_source_table` resources for each table that is currently defined within your sources. This ensures that the tables are preserved during the migration: + +```hcl +resource "materialize_source_table" "mysql_table_from_source" { + name = "mysql_table1_from_source" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.mysql_source.name + // Define the schema and database for the source if needed + } + + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + + ignore_columns = ["about"] +} +``` + +### Step 2: Apply the Changes + +Run `terraform plan` and `terraform apply` to create the new `materialize_source_table` resources. This step ensures that the tables are defined separately from the source and are not removed from Materialize. + +> **Note:** This will start an ingestion process for the newly created source tables. + +### Step 3: Remove Table Blocks from Source Resources + +Once the new `materialize_source_table` resources are successfully created, you can safely remove the `table` blocks from your existing source resources: + +```hcl +resource "materialize_source_mysql" "mysql_source" { + name = "mysql_source" + cluster_name = "cluster_name" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } +} +``` + +This will drop the old tables from the source resources. + +### Step 4: Update Terraform State + +After removing the `table` blocks from your source resources, run `terraform plan` and `terraform apply` again to update the Terraform state and apply the changes. + +### Step 5: Verify the Migration + +After applying the changes, verify that your tables are still correctly set up in Materialize by checking the table definitions using Materialize’s SQL commands. + +During the migration, you can use both the old `table` blocks and the new `materialize_source_table` resources simultaneously. This allows for a gradual transition until the old method is fully deprecated. + +## Automated Migration Process (TBD) + +> **Note:** This will still not work as the previous source tables are considered subsources of the source and are missing from the `mz_tables` table in Materialize so we can't import them directly without recreating them. + +Once the migration on the Materialize side has been implemented, a more automated migration process will be available. The steps will include: + +### Step 1: Define `materialize_source_table` Resources + +First, define the new `materialize_source_table` resources for each table: + +```hcl +resource "materialize_source_table" "mysql_table_from_source" { + name = "mysql_table1_from_source" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.mysql_source.name + // Define the schema and database for the source if needed + } + + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + + ignore_columns = ["about"] +} +``` + +### Step 2: Modify the Existing Source Resource + +Next, modify the existing source resource by removing the `table` blocks and adding an `ignore_changes` directive for the `table` attribute. This prevents Terraform from trying to delete the tables: + +```hcl +resource "materialize_source_mysql" "mysql_source" { + name = "mysql_source" + cluster_name = "cluster_name" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } + + lifecycle { + ignore_changes = [table] + } +} +``` + +- **`lifecycle { ignore_changes = [table] }`**: This directive tells Terraform to ignore changes to the `table` attribute, preventing it from trying to delete tables that were previously defined in the source resource. + +### Step 3: Import the Existing Tables + +You can then import the existing tables into the new `materialize_source_table` resources without disrupting your existing setup: + +```bash +terraform import materialize_source_table.mysql_table_from_source : +``` + +Replace `` with the actual region and `` with the table ID. You can find the table ID by querying the `mz_tables` table. + +### Step 4: Run Terraform Plan and Apply + +Finally, run `terraform plan` and `terraform apply` to ensure that everything is correctly set up without triggering any unwanted deletions. + +This approach allows you to migrate your tables safely without disrupting your existing setup. + +## Importing Existing Tables + +To import existing tables into your Terraform state using the manual migration process, use the following command: + +```bash +terraform import materialize_source_table.table_name : +``` + +Ensure you replace `` with the region where the table is located and `` with the ID of the table. + +> **Note:** The `upstream_name` and `upstream_schema_name` attributes are not yet implemented on the Materialize side, so the import process will not work until these changes are made. + +## Future Improvements + +The Kafka and Webhooks sources are currently being implemented. Once these changes, the migration process will be updated to include them. From 9b97e58ab5adc12c53d3d00e09001fe9eecc1dd8 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Wed, 11 Sep 2024 18:09:53 +0300 Subject: [PATCH 11/46] Ignore text columns for load gen source tables --- docs/resources/source_table.md | 2 +- pkg/materialize/source_table.go | 12 +++++++----- pkg/materialize/source_table_test.go | 24 ++++++++++++++++++++++++ pkg/resources/resource_source_table.go | 2 +- 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/docs/resources/source_table.md b/docs/resources/source_table.md index 3ddcadce..e3760607 100644 --- a/docs/resources/source_table.md +++ b/docs/resources/source_table.md @@ -29,7 +29,7 @@ description: |- - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the table schema in Materialize. Defaults to `public`. -- `text_columns` (List of String) Columns to be decoded as text. +- `text_columns` (List of String) Columns to be decoded as text. Not supported for the load generator sources, if the source is a load generator, the attribute will be ignored. - `upstream_schema_name` (String) The schema of the table in the upstream database. ### Read-Only diff --git a/pkg/materialize/source_table.go b/pkg/materialize/source_table.go index 287660b5..4daa1b30 100644 --- a/pkg/materialize/source_table.go +++ b/pkg/materialize/source_table.go @@ -178,16 +178,18 @@ func (b *SourceTableBuilder) Create() error { var options []string - if len(b.textColumns) > 0 { - s := strings.Join(b.textColumns, ", ") - options = append(options, fmt.Sprintf(`TEXT COLUMNS (%s)`, s)) - } - sourceType, err := b.GetSourceType() if err != nil { return err } + // Skip text columns for load-generator sources + if !strings.EqualFold(sourceType, "load-generator") && len(b.textColumns) > 0 { + s := strings.Join(b.textColumns, ", ") + options = append(options, fmt.Sprintf(`TEXT COLUMNS (%s)`, s)) + } + + // Add ignore columns only for MySQL sources if strings.EqualFold(sourceType, "mysql") && len(b.ignoreColumns) > 0 { s := strings.Join(b.ignoreColumns, ", ") options = append(options, fmt.Sprintf(`IGNORE COLUMNS (%s)`, s)) diff --git a/pkg/materialize/source_table_test.go b/pkg/materialize/source_table_test.go index 79360f07..d70dae0d 100644 --- a/pkg/materialize/source_table_test.go +++ b/pkg/materialize/source_table_test.go @@ -59,6 +59,30 @@ func TestSourceTableCreateWithMySQLSource(t *testing.T) { }) } +func TestSourceTableLoadgenCreate(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` + testhelpers.MockSourceScanWithType(mock, sourceTypeQuery, "load-generator") + + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."source" + \(REFERENCE "upstream_schema"."upstream_table"\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableBuilder(db, sourceTable) + b.Source(IdentifierSchemaStruct{Name: "source", SchemaName: "public", DatabaseName: "materialize"}) + b.UpstreamName("upstream_table") + b.UpstreamSchemaName("upstream_schema") + // Text columns are not supported for load-generator sources and should be ignored in the query builder + b.TextColumns([]string{"column1", "column2"}) + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + func TestGetSourceType(t *testing.T) { testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` diff --git a/pkg/resources/resource_source_table.go b/pkg/resources/resource_source_table.go index 93291025..86b24fb6 100644 --- a/pkg/resources/resource_source_table.go +++ b/pkg/resources/resource_source_table.go @@ -36,7 +36,7 @@ var sourceTableSchema = map[string]*schema.Schema{ Description: "The schema of the table in the upstream database.", }, "text_columns": { - Description: "Columns to be decoded as text.", + Description: "Columns to be decoded as text. Not supported for the load generator sources, if the source is a load generator, the attribute will be ignored.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, From d9626c08f2440ca3a9435e7999ca5746844a834d Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 13 Sep 2024 13:53:27 +0300 Subject: [PATCH 12/46] Refactor source table for individual sources --- docs/guides/materialize_source_table.md | 46 +- docs/resources/source_table_load_generator.md | 48 ++ ...{source_table.md => source_table_mysql.md} | 8 +- docs/resources/source_table_postgres.md | 49 ++ pkg/materialize/source_table.go | 82 +--- .../source_table_load_generator.go | 20 + .../source_table_load_generator_test.go | 56 +++ pkg/materialize/source_table_mysql.go | 126 +++++ pkg/materialize/source_table_mysql_test.go | 59 +++ pkg/materialize/source_table_postgres.go | 115 +++++ pkg/materialize/source_table_postgres_test.go | 58 +++ pkg/materialize/source_table_test.go | 128 ----- ...ptance_source_table_load_generator_test.go | 208 ++++++++ .../acceptance_source_table_mysql_test.go | 255 ++++++++++ .../acceptance_source_table_postgres_test.go | 267 ++++++++++ pkg/provider/acceptance_source_table_test.go | 458 ------------------ pkg/provider/provider.go | 4 +- pkg/resources/resource_source_table.go | 129 ----- .../resource_source_table_load_generator.go | 110 +++++ ...source_source_table_load_generator_test.go | 113 +++++ pkg/resources/resource_source_table_mysql.go | 202 ++++++++ ...go => resource_source_table_mysql_test.go} | 58 +-- .../resource_source_table_postgres.go | 187 +++++++ .../resource_source_table_postgres_test.go | 112 +++++ .../guides/materialize_source_table.md.tmpl | 46 +- 25 files changed, 2078 insertions(+), 866 deletions(-) create mode 100644 docs/resources/source_table_load_generator.md rename docs/resources/{source_table.md => source_table_mysql.md} (82%) create mode 100644 docs/resources/source_table_postgres.md create mode 100644 pkg/materialize/source_table_load_generator.go create mode 100644 pkg/materialize/source_table_load_generator_test.go create mode 100644 pkg/materialize/source_table_mysql.go create mode 100644 pkg/materialize/source_table_mysql_test.go create mode 100644 pkg/materialize/source_table_postgres.go create mode 100644 pkg/materialize/source_table_postgres_test.go create mode 100644 pkg/provider/acceptance_source_table_load_generator_test.go create mode 100644 pkg/provider/acceptance_source_table_mysql_test.go create mode 100644 pkg/provider/acceptance_source_table_postgres_test.go delete mode 100644 pkg/provider/acceptance_source_table_test.go create mode 100644 pkg/resources/resource_source_table_load_generator.go create mode 100644 pkg/resources/resource_source_table_load_generator_test.go create mode 100644 pkg/resources/resource_source_table_mysql.go rename pkg/resources/{resource_source_table_test.go => resource_source_table_mysql_test.go} (59%) create mode 100644 pkg/resources/resource_source_table_postgres.go create mode 100644 pkg/resources/resource_source_table_postgres_test.go diff --git a/docs/guides/materialize_source_table.md b/docs/guides/materialize_source_table.md index eaf224f5..8d768cd7 100644 --- a/docs/guides/materialize_source_table.md +++ b/docs/guides/materialize_source_table.md @@ -5,11 +5,13 @@ description: |- --- -# Source versioning: migrating to `materialize_source_table` Resource +# Source versioning: migrating to `materialize_source_table_{source}` Resource In previous versions of the Materialize Terraform provider, source tables were defined within the source resource itself and were considered subsources of the source rather than separate entities. -This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table` resource. +This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table_{source}` resource. + +For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. ## Old Approach @@ -38,18 +40,18 @@ The same approach was used for other source types such as Postgres and the load ## New Approach -The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table` resource. +The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table_mysql` resource. ## Manual Migration Process -This manual migration process requires users to create new source tables using the new `materialize_source_table` resource and then remove the old ones. +This manual migration process requires users to create new source tables using the new `materialize_source_table_{source}` resource and then remove the old ones. In this example, we will use MySQL as the source type. -### Step 1: Define `materialize_source_table` Resources +### Step 1: Define `materialize_source_table_mysql` Resources -Before making any changes to your existing source resources, create new `materialize_source_table` resources for each table that is currently defined within your sources. This ensures that the tables are preserved during the migration: +Before making any changes to your existing source resources, create new `materialize_source_table_mysql` resources for each table that is currently defined within your sources. This ensures that the tables are preserved during the migration: ```hcl -resource "materialize_source_table" "mysql_table_from_source" { +resource "materialize_source_table_mysql" "mysql_table_from_source" { name = "mysql_table1_from_source" schema_name = "public" database_name = "materialize" @@ -68,13 +70,13 @@ resource "materialize_source_table" "mysql_table_from_source" { ### Step 2: Apply the Changes -Run `terraform plan` and `terraform apply` to create the new `materialize_source_table` resources. This step ensures that the tables are defined separately from the source and are not removed from Materialize. +Run `terraform plan` and `terraform apply` to create the new `materialize_source_table_mysql` resources. This step ensures that the tables are defined separately from the source and are not removed from Materialize. > **Note:** This will start an ingestion process for the newly created source tables. ### Step 3: Remove Table Blocks from Source Resources -Once the new `materialize_source_table` resources are successfully created, you can safely remove the `table` blocks from your existing source resources: +Once the new `materialize_source_table_mysql` resources are successfully created, you can safely remove the `table` blocks from your existing source resources: ```hcl resource "materialize_source_mysql" "mysql_source" { @@ -84,6 +86,16 @@ resource "materialize_source_mysql" "mysql_source" { mysql_connection { name = materialize_connection_mysql.mysql_connection.name } + + // Remove the table blocks from here + - table { + - upstream_name = "mysql_table1" + - upstream_schema_name = "shop" + - name = "mysql_table1_local" + - + - ignore_columns = ["about"] + - + ... } ``` @@ -97,7 +109,9 @@ After removing the `table` blocks from your source resources, run `terraform pla After applying the changes, verify that your tables are still correctly set up in Materialize by checking the table definitions using Materialize’s SQL commands. -During the migration, you can use both the old `table` blocks and the new `materialize_source_table` resources simultaneously. This allows for a gradual transition until the old method is fully deprecated. +During the migration, you can use both the old `table` blocks and the new `materialize_source_table_{source}` resources simultaneously. This allows for a gradual transition until the old method is fully deprecated. + +The same approach can be used for other source types such as Postgres, eg. `materialize_source_table_postgres`. ## Automated Migration Process (TBD) @@ -105,12 +119,12 @@ During the migration, you can use both the old `table` blocks and the new `mater Once the migration on the Materialize side has been implemented, a more automated migration process will be available. The steps will include: -### Step 1: Define `materialize_source_table` Resources +### Step 1: Define `materialize_source_table_{source}` Resources -First, define the new `materialize_source_table` resources for each table: +First, define the new `materialize_source_table_mysql` resources for each table: ```hcl -resource "materialize_source_table" "mysql_table_from_source" { +resource "materialize_source_table_mysql" "mysql_table_from_source" { name = "mysql_table1_from_source" schema_name = "public" database_name = "materialize" @@ -150,10 +164,10 @@ resource "materialize_source_mysql" "mysql_source" { ### Step 3: Import the Existing Tables -You can then import the existing tables into the new `materialize_source_table` resources without disrupting your existing setup: +You can then import the existing tables into the new `materialize_source_table_mysql` resources without disrupting your existing setup: ```bash -terraform import materialize_source_table.mysql_table_from_source : +terraform import materialize_source_table_mysql.mysql_table_from_source : ``` Replace `` with the actual region and `` with the table ID. You can find the table ID by querying the `mz_tables` table. @@ -169,7 +183,7 @@ This approach allows you to migrate your tables safely without disrupting your e To import existing tables into your Terraform state using the manual migration process, use the following command: ```bash -terraform import materialize_source_table.table_name : +terraform import materialize_source_table_mysql.table_name : ``` Ensure you replace `` with the region where the table is located and `` with the ID of the table. diff --git a/docs/resources/source_table_load_generator.md b/docs/resources/source_table_load_generator.md new file mode 100644 index 00000000..c692c274 --- /dev/null +++ b/docs/resources/source_table_load_generator.md @@ -0,0 +1,48 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "materialize_source_table_load_generator Resource - terraform-provider-materialize" +subcategory: "" +description: |- + +--- + +# materialize_source_table_load_generator (Resource) + + + + + + +## Schema + +### Required + +- `name` (String) The identifier for the table. +- `source` (Block List, Min: 1, Max: 1) The source this table is created from. (see [below for nested schema](#nestedblock--source)) +- `upstream_name` (String) The name of the table in the upstream database. + +### Optional + +- `comment` (String) **Public Preview** Comment on an object in the database. +- `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `ownership_role` (String) The owernship role of the object. +- `region` (String) The region to use for the resource connection. If not set, the default region is used. +- `schema_name` (String) The identifier for the table schema in Materialize. Defaults to `public`. +- `upstream_schema_name` (String) The schema of the table in the upstream database. + +### Read-Only + +- `id` (String) The ID of this resource. +- `qualified_sql_name` (String) The fully qualified name of the table. + + +### Nested Schema for `source` + +Required: + +- `name` (String) The source name. + +Optional: + +- `database_name` (String) The source database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The source schema name. Defaults to `public`. diff --git a/docs/resources/source_table.md b/docs/resources/source_table_mysql.md similarity index 82% rename from docs/resources/source_table.md rename to docs/resources/source_table_mysql.md index e3760607..d83836f7 100644 --- a/docs/resources/source_table.md +++ b/docs/resources/source_table_mysql.md @@ -1,12 +1,12 @@ --- # generated by https://github.com/hashicorp/terraform-plugin-docs -page_title: "materialize_source_table Resource - terraform-provider-materialize" +page_title: "materialize_source_table_mysql Resource - terraform-provider-materialize" subcategory: "" description: |- --- -# materialize_source_table (Resource) +# materialize_source_table_mysql (Resource) @@ -25,11 +25,11 @@ description: |- - `comment` (String) **Public Preview** Comment on an object in the database. - `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. -- `ignore_columns` (List of String) Ignore specific columns when reading data from MySQL. Only compatible with MySQL sources, if the source is not MySQL, the attribute will be ignored. +- `ignore_columns` (List of String) Ignore specific columns when reading data from MySQL. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the table schema in Materialize. Defaults to `public`. -- `text_columns` (List of String) Columns to be decoded as text. Not supported for the load generator sources, if the source is a load generator, the attribute will be ignored. +- `text_columns` (List of String) Columns to be decoded as text. - `upstream_schema_name` (String) The schema of the table in the upstream database. ### Read-Only diff --git a/docs/resources/source_table_postgres.md b/docs/resources/source_table_postgres.md new file mode 100644 index 00000000..4b0881ec --- /dev/null +++ b/docs/resources/source_table_postgres.md @@ -0,0 +1,49 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "materialize_source_table_postgres Resource - terraform-provider-materialize" +subcategory: "" +description: |- + +--- + +# materialize_source_table_postgres (Resource) + + + + + + +## Schema + +### Required + +- `name` (String) The identifier for the table. +- `source` (Block List, Min: 1, Max: 1) The source this table is created from. (see [below for nested schema](#nestedblock--source)) +- `upstream_name` (String) The name of the table in the upstream database. + +### Optional + +- `comment` (String) **Public Preview** Comment on an object in the database. +- `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `ownership_role` (String) The owernship role of the object. +- `region` (String) The region to use for the resource connection. If not set, the default region is used. +- `schema_name` (String) The identifier for the table schema in Materialize. Defaults to `public`. +- `text_columns` (List of String) Columns to be decoded as text. +- `upstream_schema_name` (String) The schema of the table in the upstream database. + +### Read-Only + +- `id` (String) The ID of this resource. +- `qualified_sql_name` (String) The fully qualified name of the table. + + +### Nested Schema for `source` + +Required: + +- `name` (String) The source name. + +Optional: + +- `database_name` (String) The source database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The source schema name. Defaults to `public`. diff --git a/pkg/materialize/source_table.go b/pkg/materialize/source_table.go index 4daa1b30..42e66abd 100644 --- a/pkg/materialize/source_table.go +++ b/pkg/materialize/source_table.go @@ -26,7 +26,6 @@ type SourceTableParams struct { Privileges pq.StringArray `db:"privileges"` } -// TODO: Extend this query to include the upstream table name and schema name and the source var sourceTableQuery = NewBaseQuery(` SELECT mz_tables.id, @@ -97,9 +96,6 @@ type SourceTableBuilder struct { source IdentifierSchemaStruct upstreamName string upstreamSchemaName string - textColumns []string - ignoreColumns []string - sourceType string conn *sqlx.DB } @@ -113,27 +109,6 @@ func NewSourceTableBuilder(conn *sqlx.DB, obj MaterializeObject) *SourceTableBui } } -func (b *SourceTableBuilder) GetSourceType() (string, error) { - if b.sourceType != "" { - return b.sourceType, nil - } - - q := sourceQuery.QueryPredicate(map[string]string{ - "mz_sources.name": b.source.Name, - "mz_schemas.name": b.source.SchemaName, - "mz_databases.name": b.source.DatabaseName, - }) - - var s SourceParams - if err := b.conn.Get(&s, q); err != nil { - return "", err - } - - b.sourceType = s.SourceType.String - - return b.sourceType, nil -} - func (b *SourceTableBuilder) QualifiedName() string { return QualifiedName(b.databaseName, b.schemaName, b.tableName) } @@ -153,17 +128,20 @@ func (b *SourceTableBuilder) UpstreamSchemaName(n string) *SourceTableBuilder { return b } -func (b *SourceTableBuilder) TextColumns(c []string) *SourceTableBuilder { - b.textColumns = c - return b +func (b *SourceTableBuilder) Rename(newName string) error { + oldName := b.QualifiedName() + b.tableName = newName + newName = b.QualifiedName() + return b.ddl.rename(oldName, newName) } -func (b *SourceTableBuilder) IgnoreColumns(c []string) *SourceTableBuilder { - b.ignoreColumns = c - return b +func (b *SourceTableBuilder) Drop() error { + qn := b.QualifiedName() + return b.ddl.drop(qn) } -func (b *SourceTableBuilder) Create() error { +// BaseCreate provides a template for the Create method +func (b *SourceTableBuilder) BaseCreate(sourceType string, additionalOptions func() string) error { q := strings.Builder{} q.WriteString(fmt.Sprintf(`CREATE TABLE %s`, b.QualifiedName())) q.WriteString(fmt.Sprintf(` FROM SOURCE %s`, b.source.QualifiedName())) @@ -176,43 +154,13 @@ func (b *SourceTableBuilder) Create() error { q.WriteString(")") - var options []string - - sourceType, err := b.GetSourceType() - if err != nil { - return err - } - - // Skip text columns for load-generator sources - if !strings.EqualFold(sourceType, "load-generator") && len(b.textColumns) > 0 { - s := strings.Join(b.textColumns, ", ") - options = append(options, fmt.Sprintf(`TEXT COLUMNS (%s)`, s)) - } - - // Add ignore columns only for MySQL sources - if strings.EqualFold(sourceType, "mysql") && len(b.ignoreColumns) > 0 { - s := strings.Join(b.ignoreColumns, ", ") - options = append(options, fmt.Sprintf(`IGNORE COLUMNS (%s)`, s)) - } - - if len(options) > 0 { - q.WriteString(" WITH (") - q.WriteString(strings.Join(options, ", ")) - q.WriteString(")") + if additionalOptions != nil { + options := additionalOptions() + if options != "" { + q.WriteString(options) + } } q.WriteString(`;`) return b.ddl.exec(q.String()) } - -func (b *SourceTableBuilder) Rename(newName string) error { - oldName := b.QualifiedName() - b.tableName = newName - newName = b.QualifiedName() - return b.ddl.rename(oldName, newName) -} - -func (b *SourceTableBuilder) Drop() error { - qn := b.QualifiedName() - return b.ddl.drop(qn) -} diff --git a/pkg/materialize/source_table_load_generator.go b/pkg/materialize/source_table_load_generator.go new file mode 100644 index 00000000..bfb19bfd --- /dev/null +++ b/pkg/materialize/source_table_load_generator.go @@ -0,0 +1,20 @@ +package materialize + +import ( + "github.com/jmoiron/sqlx" +) + +// SourceTableLoadGenBuilder for Load Generator sources +type SourceTableLoadGenBuilder struct { + *SourceTableBuilder +} + +func NewSourceTableLoadGenBuilder(conn *sqlx.DB, obj MaterializeObject) *SourceTableLoadGenBuilder { + return &SourceTableLoadGenBuilder{ + SourceTableBuilder: NewSourceTableBuilder(conn, obj), + } +} + +func (b *SourceTableLoadGenBuilder) Create() error { + return b.BaseCreate("load-generator", nil) +} diff --git a/pkg/materialize/source_table_load_generator_test.go b/pkg/materialize/source_table_load_generator_test.go new file mode 100644 index 00000000..31136e90 --- /dev/null +++ b/pkg/materialize/source_table_load_generator_test.go @@ -0,0 +1,56 @@ +package materialize + +import ( + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/jmoiron/sqlx" +) + +var sourceTableLoadGen = MaterializeObject{Name: "table", SchemaName: "schema", DatabaseName: "database"} + +func TestSourceTableLoadgenCreate(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."source" + \(REFERENCE "upstream_schema"."upstream_table"\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableLoadGenBuilder(db, sourceTableLoadGen) + b.Source(IdentifierSchemaStruct{Name: "source", SchemaName: "public", DatabaseName: "materialize"}) + b.UpstreamName("upstream_table") + b.UpstreamSchemaName("upstream_schema") + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableLoadGenRename(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `ALTER TABLE "database"."schema"."table" RENAME TO "database"."schema"."new_table";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableLoadGenBuilder(db, sourceTableLoadGen) + if err := b.Rename("new_table"); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableLoadGenDrop(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `DROP TABLE "database"."schema"."table";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableLoadGenBuilder(db, sourceTableLoadGen) + if err := b.Drop(); err != nil { + t.Fatal(err) + } + }) +} diff --git a/pkg/materialize/source_table_mysql.go b/pkg/materialize/source_table_mysql.go new file mode 100644 index 00000000..a7e34072 --- /dev/null +++ b/pkg/materialize/source_table_mysql.go @@ -0,0 +1,126 @@ +package materialize + +import ( + "fmt" + "strings" + + "github.com/jmoiron/sqlx" + "github.com/lib/pq" +) + +// MySQL specific params and query +// TODO: Add upstream table name and schema name +type SourceTableMySQLParams struct { + SourceTableParams + IgnoreColumns pq.StringArray `db:"ignore_columns"` + TextColumns pq.StringArray `db:"text_columns"` +} + +var sourceTableMySQLQuery = ` + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.name AS source_name, + source_schemas.name AS source_schema_name, + source_databases.name AS source_database_name, + mz_sources.type AS source_type, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_sources + ON mz_tables.source_id = mz_sources.id + JOIN mz_schemas AS source_schemas + ON mz_sources.schema_id = source_schemas.id + JOIN mz_databases AS source_databases + ON source_schemas.database_id = source_databases.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN ( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + ) comments + ON mz_tables.id = comments.id +` + +func SourceTableMySQLId(conn *sqlx.DB, obj MaterializeObject) (string, error) { + p := map[string]string{ + "mz_tables.name": obj.Name, + "mz_schemas.name": obj.SchemaName, + "mz_databases.name": obj.DatabaseName, + } + q := NewBaseQuery(sourceTableMySQLQuery).QueryPredicate(p) + + var t SourceTableParams + if err := conn.Get(&t, q); err != nil { + return "", err + } + + return t.TableId.String, nil +} + +func ScanSourceTableMySQL(conn *sqlx.DB, id string) (SourceTableMySQLParams, error) { + q := NewBaseQuery(sourceTableMySQLQuery).QueryPredicate(map[string]string{"mz_tables.id": id}) + + var params SourceTableMySQLParams + if err := conn.Get(¶ms, q); err != nil { + return params, err + } + + return params, nil +} + +// SourceTableMySQLBuilder for MySQL sources +type SourceTableMySQLBuilder struct { + *SourceTableBuilder + textColumns []string + ignoreColumns []string +} + +func NewSourceTableMySQLBuilder(conn *sqlx.DB, obj MaterializeObject) *SourceTableMySQLBuilder { + return &SourceTableMySQLBuilder{ + SourceTableBuilder: NewSourceTableBuilder(conn, obj), + } +} + +func (b *SourceTableMySQLBuilder) TextColumns(c []string) *SourceTableMySQLBuilder { + b.textColumns = c + return b +} + +func (b *SourceTableMySQLBuilder) IgnoreColumns(c []string) *SourceTableMySQLBuilder { + b.ignoreColumns = c + return b +} + +func (b *SourceTableMySQLBuilder) Create() error { + return b.BaseCreate("mysql", func() string { + q := strings.Builder{} + var options []string + if len(b.textColumns) > 0 { + s := strings.Join(b.textColumns, ", ") + options = append(options, fmt.Sprintf(`TEXT COLUMNS (%s)`, s)) + } + + if len(b.ignoreColumns) > 0 { + s := strings.Join(b.ignoreColumns, ", ") + options = append(options, fmt.Sprintf(`IGNORE COLUMNS (%s)`, s)) + } + + if len(options) > 0 { + q.WriteString(" WITH (") + q.WriteString(strings.Join(options, ", ")) + q.WriteString(")") + } + + return q.String() + }) +} diff --git a/pkg/materialize/source_table_mysql_test.go b/pkg/materialize/source_table_mysql_test.go new file mode 100644 index 00000000..346edf1c --- /dev/null +++ b/pkg/materialize/source_table_mysql_test.go @@ -0,0 +1,59 @@ +package materialize + +import ( + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/jmoiron/sqlx" +) + +var sourceTableMySQL = MaterializeObject{Name: "table", SchemaName: "schema", DatabaseName: "database"} + +func TestSourceTableCreateWithMySQLSource(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."source" + \(REFERENCE "upstream_schema"."upstream_table"\) + WITH \(TEXT COLUMNS \(column1, column2\), IGNORE COLUMNS \(ignore1, ignore2\)\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableMySQLBuilder(db, sourceTableMySQL) + b.Source(IdentifierSchemaStruct{Name: "source", SchemaName: "public", DatabaseName: "materialize"}) + b.UpstreamName("upstream_table") + b.UpstreamSchemaName("upstream_schema") + b.TextColumns([]string{"column1", "column2"}) + b.IgnoreColumns([]string{"ignore1", "ignore2"}) + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableMySQLRename(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `ALTER TABLE "database"."schema"."table" RENAME TO "database"."schema"."new_table";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableMySQLBuilder(db, sourceTableMySQL) + if err := b.Rename("new_table"); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableMySQLDrop(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `DROP TABLE "database"."schema"."table";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableMySQLBuilder(db, sourceTableMySQL) + if err := b.Drop(); err != nil { + t.Fatal(err) + } + }) +} diff --git a/pkg/materialize/source_table_postgres.go b/pkg/materialize/source_table_postgres.go new file mode 100644 index 00000000..dc1f77db --- /dev/null +++ b/pkg/materialize/source_table_postgres.go @@ -0,0 +1,115 @@ +package materialize + +import ( + "fmt" + "strings" + + "github.com/jmoiron/sqlx" + "github.com/lib/pq" +) + +// Postgres specific params and query +type SourceTablePostgresParams struct { + SourceTableParams + // Add upstream table and schema name once supported + IgnoreColumns pq.StringArray `db:"ignore_columns"` + TextColumns pq.StringArray `db:"text_columns"` +} + +var sourceTablePostgresQuery = ` + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.name AS source_name, + source_schemas.name AS source_schema_name, + source_databases.name AS source_database_name, + mz_sources.type AS source_type, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_sources + ON mz_tables.source_id = mz_sources.id + JOIN mz_schemas AS source_schemas + ON mz_sources.schema_id = source_schemas.id + JOIN mz_databases AS source_databases + ON source_schemas.database_id = source_databases.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN ( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + ) comments + ON mz_tables.id = comments.id +` + +func SourceTablePostgresId(conn *sqlx.DB, obj MaterializeObject) (string, error) { + p := map[string]string{ + "mz_tables.name": obj.Name, + "mz_schemas.name": obj.SchemaName, + "mz_databases.name": obj.DatabaseName, + } + q := NewBaseQuery(sourceTablePostgresQuery).QueryPredicate(p) + + var t SourceTableParams + if err := conn.Get(&t, q); err != nil { + return "", err + } + + return t.TableId.String, nil +} + +func ScanSourceTablePostgres(conn *sqlx.DB, id string) (SourceTablePostgresParams, error) { + q := NewBaseQuery(sourceTablePostgresQuery).QueryPredicate(map[string]string{"mz_tables.id": id}) + + var params SourceTablePostgresParams + if err := conn.Get(¶ms, q); err != nil { + return params, err + } + + return params, nil +} + +// SourceTablePostgresBuilder for Postgres sources +type SourceTablePostgresBuilder struct { + *SourceTableBuilder + textColumns []string +} + +func NewSourceTablePostgresBuilder(conn *sqlx.DB, obj MaterializeObject) *SourceTablePostgresBuilder { + return &SourceTablePostgresBuilder{ + SourceTableBuilder: NewSourceTableBuilder(conn, obj), + } +} + +func (b *SourceTablePostgresBuilder) TextColumns(c []string) *SourceTablePostgresBuilder { + b.textColumns = c + return b +} + +func (b *SourceTablePostgresBuilder) Create() error { + return b.BaseCreate("postgres", func() string { + q := strings.Builder{} + var options []string + if len(b.textColumns) > 0 { + s := strings.Join(b.textColumns, ", ") + options = append(options, fmt.Sprintf(`TEXT COLUMNS (%s)`, s)) + } + + if len(options) > 0 { + q.WriteString(" WITH (") + q.WriteString(strings.Join(options, ", ")) + q.WriteString(")") + } + + return q.String() + }) +} diff --git a/pkg/materialize/source_table_postgres_test.go b/pkg/materialize/source_table_postgres_test.go new file mode 100644 index 00000000..8d120adc --- /dev/null +++ b/pkg/materialize/source_table_postgres_test.go @@ -0,0 +1,58 @@ +package materialize + +import ( + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/jmoiron/sqlx" +) + +var sourceTablePostgres = MaterializeObject{Name: "table", SchemaName: "schema", DatabaseName: "database"} + +func TestSourceTablePostgresCreate(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."source" + \(REFERENCE "upstream_schema"."upstream_table"\) + WITH \(TEXT COLUMNS \(column1, column2\)\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTablePostgresBuilder(db, sourceTablePostgres) + b.Source(IdentifierSchemaStruct{Name: "source", SchemaName: "public", DatabaseName: "materialize"}) + b.UpstreamName("upstream_table") + b.UpstreamSchemaName("upstream_schema") + b.TextColumns([]string{"column1", "column2"}) + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTablePostgresRename(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `ALTER TABLE "database"."schema"."table" RENAME TO "database"."schema"."new_table";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTablePostgresBuilder(db, sourceTablePostgres) + if err := b.Rename("new_table"); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTablePostgresDrop(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `DROP TABLE "database"."schema"."table";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTablePostgresBuilder(db, sourceTablePostgres) + if err := b.Drop(); err != nil { + t.Fatal(err) + } + }) +} diff --git a/pkg/materialize/source_table_test.go b/pkg/materialize/source_table_test.go index d70dae0d..0d3d5548 100644 --- a/pkg/materialize/source_table_test.go +++ b/pkg/materialize/source_table_test.go @@ -1,129 +1 @@ package materialize - -import ( - "testing" - - sqlmock "github.com/DATA-DOG/go-sqlmock" - "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" - "github.com/jmoiron/sqlx" -) - -var sourceTable = MaterializeObject{Name: "table", SchemaName: "schema", DatabaseName: "database"} - -func TestSourceTableCreate(t *testing.T) { - testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { - sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` - testhelpers.MockSourceScanWithType(mock, sourceTypeQuery, "kafka") - - mock.ExpectExec( - `CREATE TABLE "database"."schema"."table" - FROM SOURCE "materialize"."public"."source" - \(REFERENCE "upstream_schema"."upstream_table"\) - WITH \(TEXT COLUMNS \(column1, column2\)\);`, - ).WillReturnResult(sqlmock.NewResult(1, 1)) - - b := NewSourceTableBuilder(db, sourceTable) - b.Source(IdentifierSchemaStruct{Name: "source", SchemaName: "public", DatabaseName: "materialize"}) - b.UpstreamName("upstream_table") - b.UpstreamSchemaName("upstream_schema") - b.TextColumns([]string{"column1", "column2"}) - - if err := b.Create(); err != nil { - t.Fatal(err) - } - }) -} - -func TestSourceTableCreateWithMySQLSource(t *testing.T) { - testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { - sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` - testhelpers.MockSourceScanWithType(mock, sourceTypeQuery, "mysql") - - mock.ExpectExec( - `CREATE TABLE "database"."schema"."table" - FROM SOURCE "materialize"."public"."source" - \(REFERENCE "upstream_schema"."upstream_table"\) - WITH \(TEXT COLUMNS \(column1, column2\), IGNORE COLUMNS \(ignore1, ignore2\)\);`, - ).WillReturnResult(sqlmock.NewResult(1, 1)) - - b := NewSourceTableBuilder(db, sourceTable) - b.Source(IdentifierSchemaStruct{Name: "source", SchemaName: "public", DatabaseName: "materialize"}) - b.UpstreamName("upstream_table") - b.UpstreamSchemaName("upstream_schema") - b.TextColumns([]string{"column1", "column2"}) - b.IgnoreColumns([]string{"ignore1", "ignore2"}) - - if err := b.Create(); err != nil { - t.Fatal(err) - } - }) -} - -func TestSourceTableLoadgenCreate(t *testing.T) { - testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { - sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` - testhelpers.MockSourceScanWithType(mock, sourceTypeQuery, "load-generator") - - mock.ExpectExec( - `CREATE TABLE "database"."schema"."table" - FROM SOURCE "materialize"."public"."source" - \(REFERENCE "upstream_schema"."upstream_table"\);`, - ).WillReturnResult(sqlmock.NewResult(1, 1)) - - b := NewSourceTableBuilder(db, sourceTable) - b.Source(IdentifierSchemaStruct{Name: "source", SchemaName: "public", DatabaseName: "materialize"}) - b.UpstreamName("upstream_table") - b.UpstreamSchemaName("upstream_schema") - // Text columns are not supported for load-generator sources and should be ignored in the query builder - b.TextColumns([]string{"column1", "column2"}) - - if err := b.Create(); err != nil { - t.Fatal(err) - } - }) -} - -func TestGetSourceType(t *testing.T) { - testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { - sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` - testhelpers.MockSourceScanWithType(mock, sourceTypeQuery, "KAFKA") - - b := NewSourceTableBuilder(db, sourceTable) - b.Source(IdentifierSchemaStruct{Name: "source", SchemaName: "public", DatabaseName: "materialize"}) - - sourceType, err := b.GetSourceType() - if err != nil { - t.Fatal(err) - } - - if sourceType != "KAFKA" { - t.Fatalf("Expected source type 'KAFKA', got '%s'", sourceType) - } - }) -} - -func TestSourceTableRename(t *testing.T) { - testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { - mock.ExpectExec( - `ALTER TABLE "database"."schema"."table" RENAME TO "database"."schema"."new_table";`, - ).WillReturnResult(sqlmock.NewResult(1, 1)) - - b := NewSourceTableBuilder(db, sourceTable) - if err := b.Rename("new_table"); err != nil { - t.Fatal(err) - } - }) -} - -func TestSourceTableDrop(t *testing.T) { - testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { - mock.ExpectExec( - `DROP TABLE "database"."schema"."table";`, - ).WillReturnResult(sqlmock.NewResult(1, 1)) - - b := NewSourceTableBuilder(db, sourceTable) - if err := b.Drop(); err != nil { - t.Fatal(err) - } - }) -} diff --git a/pkg/provider/acceptance_source_table_load_generator_test.go b/pkg/provider/acceptance_source_table_load_generator_test.go new file mode 100644 index 00000000..2ceb95bc --- /dev/null +++ b/pkg/provider/acceptance_source_table_load_generator_test.go @@ -0,0 +1,208 @@ +package provider + +import ( + "database/sql" + "fmt" + "testing" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + "github.com/hashicorp/terraform-plugin-testing/helper/acctest" + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-plugin-testing/terraform" +) + +func TestAccSourceTableLoadGen_basic(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: nil, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableLoadGenBasicResource(nameSpace), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableLoadGenExists("materialize_source_table_load_generator.test_loadgen"), + resource.TestMatchResourceAttr("materialize_source_table_load_generator.test_loadgen", "id", terraformObjectIdRegex), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "name", nameSpace+"_table_loadgen2"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "database_name", "materialize"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "upstream_name", "bids"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "source.0.name", nameSpace+"_loadgen2"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "source.0.schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "source.0.database_name", "materialize"), + ), + }, + }, + }) +} + +func TestAccSourceTableLoadGen_update(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: testAccCheckAllSourceTableLoadGenDestroyed, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableLoadGenResource(nameSpace, "bids", "", ""), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableLoadGenExists("materialize_source_table_load_generator.test_loadgen"), + resource.TestMatchResourceAttr("materialize_source_table_load_generator.test_loadgen", "id", terraformObjectIdRegex), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "name", nameSpace+"_table_loadgen"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "database_name", "materialize"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "upstream_name", "bids"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "source.0.name", nameSpace+"_loadgen"), + ), + }, + { + Config: testAccSourceTableLoadGenResource(nameSpace, "bids", nameSpace+"_role", "Updated comment"), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableLoadGenExists("materialize_source_table_load_generator.test_loadgen"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "name", nameSpace+"_table_loadgen"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "database_name", "materialize"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "upstream_name", "bids"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "source.0.name", nameSpace+"_loadgen"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "ownership_role", nameSpace+"_role"), + resource.TestCheckResourceAttr("materialize_source_table_load_generator.test_loadgen", "comment", "Updated comment"), + ), + }, + }, + }) +} + +func TestAccSourceTableLoadGen_disappears(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: testAccCheckAllSourceTableLoadGenDestroyed, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableLoadGenResource(nameSpace, "bids", "mz_system", ""), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableLoadGenExists("materialize_source_table_load_generator.test"), + testAccCheckObjectDisappears( + materialize.MaterializeObject{ + ObjectType: "TABLE", + Name: nameSpace + "_table", + }, + ), + ), + PlanOnly: true, + ExpectNonEmptyPlan: true, + }, + }, + }) +} + +func testAccSourceTableLoadGenBasicResource(nameSpace string) string { + return fmt.Sprintf(` + resource "materialize_source_load_generator" "test_loadgen" { + name = "%[1]s_loadgen2" + load_generator_type = "AUCTION" + + schema_name = "public" + database_name = "materialize" + + auction_options { + tick_interval = "500ms" + } + } + + resource "materialize_source_table_load_generator" "test_loadgen" { + name = "%[1]s_table_loadgen2" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_load_generator.test_loadgen.name + schema_name = "public" + database_name = "materialize" + } + + upstream_name = "bids" + } + `, nameSpace) +} + +func testAccSourceTableLoadGenResource(nameSpace, upstreamName, ownershipRole, comment string) string { + return fmt.Sprintf(` + resource "materialize_source_load_generator" "test_loadgen" { + name = "%[1]s_loadgen" + load_generator_type = "AUCTION" + + schema_name = "public" + database_name = "materialize" + + auction_options { + tick_interval = "500ms" + } + } + + resource "materialize_role" "test_role" { + name = "%[1]s_role" + } + + resource "materialize_source_table_load_generator" "test_loadgen" { + name = "%[1]s_table_loadgen" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_load_generator.test_loadgen.name + schema_name = "public" + database_name = "materialize" + } + + upstream_name = "%[2]s" + ownership_role = "%[3]s" + comment = "%[4]s" + + depends_on = [materialize_role.test_role] + } + `, nameSpace, upstreamName, ownershipRole, comment) +} + +func testAccCheckSourceTableLoadGenExists(name string) resource.TestCheckFunc { + return func(s *terraform.State) error { + meta := testAccProvider.Meta() + db, _, err := utils.GetDBClientFromMeta(meta, nil) + if err != nil { + return fmt.Errorf("error getting DB client: %s", err) + } + r, ok := s.RootModule().Resources[name] + if !ok { + return fmt.Errorf("source table not found: %s", name) + } + _, err = materialize.ScanSourceTable(db, utils.ExtractId(r.Primary.ID)) + return err + } +} + +func testAccCheckAllSourceTableLoadGenDestroyed(s *terraform.State) error { + meta := testAccProvider.Meta() + db, _, err := utils.GetDBClientFromMeta(meta, nil) + if err != nil { + return fmt.Errorf("error getting DB client: %s", err) + } + + for _, r := range s.RootModule().Resources { + if r.Type != "materialize_source_table_load_generator" { + continue + } + + _, err := materialize.ScanSourceTable(db, utils.ExtractId(r.Primary.ID)) + if err == nil { + return fmt.Errorf("source table %v still exists", utils.ExtractId(r.Primary.ID)) + } else if err != sql.ErrNoRows { + return err + } + } + return nil +} diff --git a/pkg/provider/acceptance_source_table_mysql_test.go b/pkg/provider/acceptance_source_table_mysql_test.go new file mode 100644 index 00000000..30a45eee --- /dev/null +++ b/pkg/provider/acceptance_source_table_mysql_test.go @@ -0,0 +1,255 @@ +package provider + +import ( + "database/sql" + "fmt" + "testing" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + "github.com/hashicorp/terraform-plugin-testing/helper/acctest" + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-plugin-testing/terraform" +) + +func TestAccSourceTableMySQL_basic(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: nil, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableMySQLBasicResource(nameSpace), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table_mysql.test_mysql"), + resource.TestMatchResourceAttr("materialize_source_table_mysql.test_mysql", "id", terraformObjectIdRegex), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "name", nameSpace+"_table_mysql"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "database_name", "materialize"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "upstream_name", "mysql_table1"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "upstream_schema_name", "shop"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "ignore_columns.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "ignore_columns.0", "banned"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "source.0.name", nameSpace+"_source_mysql"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "source.0.schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "source.0.database_name", "materialize"), + // resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "text_columns.#", "1"), + // resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "text_columns.0", "about"), + ), + }, + }, + }) +} + +func TestAccSourceTableMySQL_update(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: nil, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableMySQLResource(nameSpace, "mysql_table2", "mz_system", ""), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table_mysql.test"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "name", nameSpace+"_table"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "upstream_name", "mysql_table2"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "ownership_role", "mz_system"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "comment", ""), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "source.0.name", nameSpace+"_source_mysql"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "source.0.schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "source.0.database_name", "materialize"), + ), + }, + { + Config: testAccSourceTableMySQLResource(nameSpace, "mysql_table1", nameSpace+"_role", "Updated comment"), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table_mysql.test"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "name", nameSpace+"_table"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "upstream_name", "mysql_table1"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "ownership_role", nameSpace+"_role"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "comment", "Updated comment"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "source.0.name", nameSpace+"_source_mysql"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test", "source.0.schema_name", "public"), + ), + }, + }, + }) +} + +func TestAccSourceTableMySQL_disappears(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: testAccCheckAllSourceTableDestroyed, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableMySQLResource(nameSpace, "mysql_table2", "mz_system", ""), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table_mysql.test"), + testAccCheckObjectDisappears( + materialize.MaterializeObject{ + ObjectType: "TABLE", + Name: nameSpace + "_table", + }, + ), + ), + PlanOnly: true, + ExpectNonEmptyPlan: true, + }, + }, + }) +} + +func testAccSourceTableMySQLBasicResource(nameSpace string) string { + return fmt.Sprintf(` + resource "materialize_secret" "mysql_password" { + name = "%[1]s_secret_mysql" + value = "c2VjcmV0Cg==" + } + + resource "materialize_connection_mysql" "mysql_connection" { + name = "%[1]s_connection_mysql" + host = "mysql" + port = 3306 + user { + text = "repluser" + } + password { + name = materialize_secret.mysql_password.name + } + } + + resource "materialize_source_mysql" "test_source_mysql" { + name = "%[1]s_source_mysql" + cluster_name = "quickstart" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } + + table { + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + name = "mysql_table1_local" + } + } + + resource "materialize_source_table_mysql" "test_mysql" { + name = "%[1]s_table_mysql" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.test_source_mysql.name + } + + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + ignore_columns = ["banned"] + } + `, nameSpace) +} + +func testAccSourceTableMySQLResource(nameSpace, upstreamName, ownershipRole, comment string) string { + return fmt.Sprintf(` + resource "materialize_secret" "mysql_password" { + name = "%[1]s_secret_mysql" + value = "c2VjcmV0Cg==" + } + + resource "materialize_connection_mysql" "mysql_connection" { + name = "%[1]s_connection_mysql" + host = "mysql" + port = 3306 + user { + text = "repluser" + } + password { + name = materialize_secret.mysql_password.name + } + } + + resource "materialize_source_mysql" "test_source_mysql" { + name = "%[1]s_source_mysql" + cluster_name = "quickstart" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } + + table { + upstream_name = "mysql_table1" + upstream_schema_name = "shop" + name = "mysql_table1_local" + } + } + + resource "materialize_role" "test_role" { + name = "%[1]s_role" + } + + resource "materialize_source_table_mysql" "test" { + name = "%[1]s_table" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.test_source_mysql.name + schema_name = "public" + database_name = "materialize" + } + + upstream_name = "%[2]s" + upstream_schema_name = "shop" + + ownership_role = "%[3]s" + comment = "%[4]s" + + depends_on = [materialize_role.test_role] + } + `, nameSpace, upstreamName, ownershipRole, comment) +} + +func testAccCheckSourceTableExists(name string) resource.TestCheckFunc { + return func(s *terraform.State) error { + meta := testAccProvider.Meta() + db, _, err := utils.GetDBClientFromMeta(meta, nil) + if err != nil { + return fmt.Errorf("error getting DB client: %s", err) + } + r, ok := s.RootModule().Resources[name] + if !ok { + return fmt.Errorf("source table not found: %s", name) + } + _, err = materialize.ScanSourceTable(db, utils.ExtractId(r.Primary.ID)) + return err + } +} + +func testAccCheckAllSourceTableDestroyed(s *terraform.State) error { + meta := testAccProvider.Meta() + db, _, err := utils.GetDBClientFromMeta(meta, nil) + if err != nil { + return fmt.Errorf("error getting DB client: %s", err) + } + + for _, r := range s.RootModule().Resources { + if r.Type != "materialize_source_table_mysql" { + continue + } + + _, err := materialize.ScanSourceTable(db, utils.ExtractId(r.Primary.ID)) + if err == nil { + return fmt.Errorf("source table %v still exists", utils.ExtractId(r.Primary.ID)) + } else if err != sql.ErrNoRows { + return err + } + } + return nil +} diff --git a/pkg/provider/acceptance_source_table_postgres_test.go b/pkg/provider/acceptance_source_table_postgres_test.go new file mode 100644 index 00000000..9ffb34cc --- /dev/null +++ b/pkg/provider/acceptance_source_table_postgres_test.go @@ -0,0 +1,267 @@ +package provider + +import ( + "database/sql" + "fmt" + "testing" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + "github.com/hashicorp/terraform-plugin-testing/helper/acctest" + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-plugin-testing/terraform" +) + +func TestAccSourceTablePostgres_basic(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: nil, + Steps: []resource.TestStep{ + { + Config: testAccSourceTablePostgresBasicResource(nameSpace), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTablePostgresExists("materialize_source_table_postgres.test_postgres"), + resource.TestMatchResourceAttr("materialize_source_table_postgres.test_postgres", "id", terraformObjectIdRegex), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "name", nameSpace+"_table_postgres"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "database_name", "materialize"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "text_columns.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "text_columns.0", "updated_at"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "upstream_name", "table2"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "upstream_schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "source.0.name", nameSpace+"_source_postgres"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "source.0.schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "source.0.database_name", "materialize"), + ), + }, + }, + }) +} + +func TestAccSourceTablePostgres_update(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: nil, + Steps: []resource.TestStep{ + { + Config: testAccSourceTablePostgresResource(nameSpace, "table2", "mz_system", ""), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTablePostgresExists("materialize_source_table_postgres.test"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "name", nameSpace+"_table"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "upstream_name", "table2"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "text_columns.#", "2"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "ownership_role", "mz_system"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "comment", ""), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "source.0.name", nameSpace+"_source"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "source.0.schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "source.0.database_name", "materialize"), + ), + }, + { + Config: testAccSourceTablePostgresResource(nameSpace, "table3", nameSpace+"_role", "Updated comment"), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTablePostgresExists("materialize_source_table_postgres.test"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "name", nameSpace+"_table"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "upstream_name", "table3"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "text_columns.#", "2"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "ownership_role", nameSpace+"_role"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "comment", "Updated comment"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "source.0.name", nameSpace+"_source"), + resource.TestCheckResourceAttr("materialize_source_table_postgres.test", "source.0.schema_name", "public"), + ), + }, + }, + }) +} + +func TestAccSourceTablePostgres_disappears(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: testAccCheckAllSourceTablePostgresDestroyed, + Steps: []resource.TestStep{ + { + Config: testAccSourceTablePostgresResource(nameSpace, "table2", "mz_system", ""), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTablePostgresExists("materialize_source_table_postgres.test"), + testAccCheckObjectDisappears( + materialize.MaterializeObject{ + ObjectType: "TABLE", + Name: nameSpace + "_table", + }, + ), + ), + PlanOnly: true, + ExpectNonEmptyPlan: true, + }, + }, + }) +} + +func testAccSourceTablePostgresBasicResource(nameSpace string) string { + return fmt.Sprintf(` + resource "materialize_secret" "postgres_password" { + name = "%[1]s_secret_postgres" + value = "c2VjcmV0Cg==" + } + + resource "materialize_connection_postgres" "postgres_connection" { + name = "%[1]s_connection_postgres" + host = "postgres" + port = 5432 + user { + text = "postgres" + } + password { + name = materialize_secret.postgres_password.name + } + database = "postgres" + } + + resource "materialize_source_postgres" "test_source_postgres" { + name = "%[1]s_source_postgres" + cluster_name = "quickstart" + + postgres_connection { + name = materialize_connection_postgres.postgres_connection.name + } + publication = "mz_source" + table { + upstream_name = "table2" + upstream_schema_name = "public" + } + } + + resource "materialize_source_table_postgres" "test_postgres" { + name = "%[1]s_table_postgres" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_postgres.test_source_postgres.name + } + + upstream_name = "table2" + upstream_schema_name = "public" + + text_columns = [ + "updated_at" + ] + } + `, nameSpace) +} + +func testAccSourceTablePostgresResource(nameSpace, upstreamName, ownershipRole, comment string) string { + return fmt.Sprintf(` + resource "materialize_secret" "postgres_password" { + name = "%[1]s_secret" + value = "c2VjcmV0Cg==" + } + + resource "materialize_connection_postgres" "postgres_connection" { + name = "%[1]s_connection" + host = "postgres" + port = 5432 + user { + text = "postgres" + } + password { + name = materialize_secret.postgres_password.name + database_name = materialize_secret.postgres_password.database_name + schema_name = materialize_secret.postgres_password.schema_name + } + database = "postgres" + } + + resource "materialize_source_postgres" "test_source" { + name = "%[1]s_source" + cluster_name = "quickstart" + + postgres_connection { + name = materialize_connection_postgres.postgres_connection.name + schema_name = materialize_connection_postgres.postgres_connection.schema_name + database_name = materialize_connection_postgres.postgres_connection.database_name + } + publication = "mz_source" + table { + upstream_name = "%[2]s" + upstream_schema_name = "public" + } + } + + resource "materialize_role" "test_role" { + name = "%[1]s_role" + } + + resource "materialize_source_table_postgres" "test" { + name = "%[1]s_table" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_postgres.test_source.name + schema_name = "public" + database_name = "materialize" + } + + upstream_name = "%[2]s" + upstream_schema_name = "public" + + text_columns = [ + "updated_at", + "id" + ] + + ownership_role = "%[3]s" + comment = "%[4]s" + + depends_on = [materialize_role.test_role] + } + `, nameSpace, upstreamName, ownershipRole, comment) +} + +func testAccCheckSourceTablePostgresExists(name string) resource.TestCheckFunc { + return func(s *terraform.State) error { + meta := testAccProvider.Meta() + db, _, err := utils.GetDBClientFromMeta(meta, nil) + if err != nil { + return fmt.Errorf("error getting DB client: %s", err) + } + r, ok := s.RootModule().Resources[name] + if !ok { + return fmt.Errorf("source table not found: %s", name) + } + _, err = materialize.ScanSourceTable(db, utils.ExtractId(r.Primary.ID)) + return err + } +} + +func testAccCheckAllSourceTablePostgresDestroyed(s *terraform.State) error { + meta := testAccProvider.Meta() + db, _, err := utils.GetDBClientFromMeta(meta, nil) + if err != nil { + return fmt.Errorf("error getting DB client: %s", err) + } + + for _, r := range s.RootModule().Resources { + if r.Type != "materialize_source_table_postgres" { + continue + } + + _, err := materialize.ScanSourceTable(db, utils.ExtractId(r.Primary.ID)) + if err == nil { + return fmt.Errorf("source table %v still exists", utils.ExtractId(r.Primary.ID)) + } else if err != sql.ErrNoRows { + return err + } + } + return nil +} diff --git a/pkg/provider/acceptance_source_table_test.go b/pkg/provider/acceptance_source_table_test.go deleted file mode 100644 index de133107..00000000 --- a/pkg/provider/acceptance_source_table_test.go +++ /dev/null @@ -1,458 +0,0 @@ -package provider - -import ( - "database/sql" - "fmt" - "testing" - - "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" - "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" - "github.com/hashicorp/terraform-plugin-testing/helper/acctest" - "github.com/hashicorp/terraform-plugin-testing/helper/resource" - "github.com/hashicorp/terraform-plugin-testing/terraform" -) - -func TestAccSourceTablePostgres_basic(t *testing.T) { - nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: nil, - Steps: []resource.TestStep{ - { - Config: testAccSourceTablePostgresBasicResource(nameSpace), - Check: resource.ComposeTestCheckFunc( - testAccCheckSourceTableExists("materialize_source_table.test_postgres"), - resource.TestMatchResourceAttr("materialize_source_table.test_postgres", "id", terraformObjectIdRegex), - resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "name", nameSpace+"_table_postgres"), - resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "database_name", "materialize"), - resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "schema_name", "public"), - resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "text_columns.#", "1"), - resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "text_columns.0", "updated_at"), - resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "upstream_name", "table2"), - resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "upstream_schema_name", "public"), - resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "source.#", "1"), - resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "source.0.name", nameSpace+"_source_postgres"), - resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "source.0.schema_name", "public"), - resource.TestCheckResourceAttr("materialize_source_table.test_postgres", "source.0.database_name", "materialize"), - ), - }, - }, - }) -} - -func TestAccSourceTableMySQL_basic(t *testing.T) { - nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: nil, - Steps: []resource.TestStep{ - { - Config: testAccSourceTableMySQLBasicResource(nameSpace), - Check: resource.ComposeTestCheckFunc( - testAccCheckSourceTableExists("materialize_source_table.test_mysql"), - resource.TestMatchResourceAttr("materialize_source_table.test_mysql", "id", terraformObjectIdRegex), - resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "name", nameSpace+"_table_mysql"), - resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "database_name", "materialize"), - resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "schema_name", "public"), - resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "upstream_name", "mysql_table1"), - resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "upstream_schema_name", "shop"), - resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "ignore_columns.#", "1"), - resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "ignore_columns.0", "banned"), - resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "source.#", "1"), - resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "source.0.name", nameSpace+"_source_mysql"), - resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "source.0.schema_name", "public"), - resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "source.0.database_name", "materialize"), - // resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "text_columns.#", "1"), - // resource.TestCheckResourceAttr("materialize_source_table.test_mysql", "text_columns.0", "about"), - ), - }, - }, - }) -} - -func TestAccSourceTableLoadGen_basic(t *testing.T) { - nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: nil, - Steps: []resource.TestStep{ - { - Config: testAccSourceTableLoadGenBasicResource(nameSpace), - Check: resource.ComposeTestCheckFunc( - testAccCheckSourceTableExists("materialize_source_table.test_loadgen"), - resource.TestMatchResourceAttr("materialize_source_table.test_loadgen", "id", terraformObjectIdRegex), - resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "name", nameSpace+"_table_loadgen"), - resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "database_name", "materialize"), - resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "schema_name", "public"), - resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "upstream_name", "bids"), - resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "source.#", "1"), - resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "source.0.name", nameSpace+"_loadgen"), - resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "source.0.schema_name", "public"), - resource.TestCheckResourceAttr("materialize_source_table.test_loadgen", "source.0.database_name", "materialize"), - ), - }, - }, - }) -} - -func TestAccSourceTable_update(t *testing.T) { - nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) - resource.Test(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: nil, - Steps: []resource.TestStep{ - { - Config: testAccSourceTableResource(nameSpace, "table2", "mz_system", ""), - Check: resource.ComposeTestCheckFunc( - testAccCheckSourceTableExists("materialize_source_table.test"), - resource.TestCheckResourceAttr("materialize_source_table.test", "name", nameSpace+"_table"), - resource.TestCheckResourceAttr("materialize_source_table.test", "upstream_name", "table2"), - resource.TestCheckResourceAttr("materialize_source_table.test", "text_columns.#", "2"), - resource.TestCheckResourceAttr("materialize_source_table.test", "ownership_role", "mz_system"), - resource.TestCheckResourceAttr("materialize_source_table.test", "comment", ""), - resource.TestCheckResourceAttr("materialize_source_table.test", "source.#", "1"), - resource.TestCheckResourceAttr("materialize_source_table.test", "source.0.name", nameSpace+"_source"), - resource.TestCheckResourceAttr("materialize_source_table.test", "source.0.schema_name", "public"), - resource.TestCheckResourceAttr("materialize_source_table.test", "source.0.database_name", "materialize"), - ), - }, - { - Config: testAccSourceTableResource(nameSpace, "table3", nameSpace+"_role", "Updated comment"), - Check: resource.ComposeTestCheckFunc( - testAccCheckSourceTableExists("materialize_source_table.test"), - resource.TestCheckResourceAttr("materialize_source_table.test", "name", nameSpace+"_table"), - resource.TestCheckResourceAttr("materialize_source_table.test", "upstream_name", "table3"), - resource.TestCheckResourceAttr("materialize_source_table.test", "text_columns.#", "2"), - resource.TestCheckResourceAttr("materialize_source_table.test", "ownership_role", nameSpace+"_role"), - resource.TestCheckResourceAttr("materialize_source_table.test", "comment", "Updated comment"), - resource.TestCheckResourceAttr("materialize_source_table.test", "source.#", "1"), - resource.TestCheckResourceAttr("materialize_source_table.test", "source.0.name", nameSpace+"_source"), - resource.TestCheckResourceAttr("materialize_source_table.test", "source.0.schema_name", "public"), - ), - }, - }, - }) -} - -func TestAccSourceTable_disappears(t *testing.T) { - nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) - resource.ParallelTest(t, resource.TestCase{ - PreCheck: func() { testAccPreCheck(t) }, - ProviderFactories: testAccProviderFactories, - CheckDestroy: testAccCheckAllSourceTableDestroyed, - Steps: []resource.TestStep{ - { - Config: testAccSourceTableResource(nameSpace, "table2", "mz_system", ""), - Check: resource.ComposeTestCheckFunc( - testAccCheckSourceTableExists("materialize_source_table.test"), - testAccCheckObjectDisappears( - materialize.MaterializeObject{ - ObjectType: "TABLE", - Name: nameSpace + "_table", - }, - ), - ), - PlanOnly: true, - ExpectNonEmptyPlan: true, - }, - }, - }) -} - -func testAccSourceTablePostgresBasicResource(nameSpace string) string { - return fmt.Sprintf(` - resource "materialize_secret" "postgres_password" { - name = "%[1]s_secret_postgres" - value = "c2VjcmV0Cg==" - } - - resource "materialize_connection_postgres" "postgres_connection" { - name = "%[1]s_connection_postgres" - host = "postgres" - port = 5432 - user { - text = "postgres" - } - password { - name = materialize_secret.postgres_password.name - } - database = "postgres" - } - - resource "materialize_source_postgres" "test_source_postgres" { - name = "%[1]s_source_postgres" - cluster_name = "quickstart" - - postgres_connection { - name = materialize_connection_postgres.postgres_connection.name - } - publication = "mz_source" - table { - upstream_name = "table2" - upstream_schema_name = "public" - } - } - - resource "materialize_source_table" "test_postgres" { - name = "%[1]s_table_postgres" - schema_name = "public" - database_name = "materialize" - - source { - name = materialize_source_postgres.test_source_postgres.name - } - - upstream_name = "table2" - upstream_schema_name = "public" - - text_columns = [ - "updated_at" - ] - } - `, nameSpace) -} - -func testAccSourceTableMySQLBasicResource(nameSpace string) string { - return fmt.Sprintf(` - resource "materialize_secret" "mysql_password" { - name = "%[1]s_secret_mysql" - value = "c2VjcmV0Cg==" - } - - resource "materialize_connection_mysql" "mysql_connection" { - name = "%[1]s_connection_mysql" - host = "mysql" - port = 3306 - user { - text = "repluser" - } - password { - name = materialize_secret.mysql_password.name - } - } - - resource "materialize_source_mysql" "test_source_mysql" { - name = "%[1]s_source_mysql" - cluster_name = "quickstart" - - mysql_connection { - name = materialize_connection_mysql.mysql_connection.name - } - - table { - upstream_name = "mysql_table1" - upstream_schema_name = "shop" - name = "mysql_table1_local" - } - } - - resource "materialize_source_table" "test_mysql" { - name = "%[1]s_table_mysql" - schema_name = "public" - database_name = "materialize" - - source { - name = materialize_source_mysql.test_source_mysql.name - } - - upstream_name = "mysql_table1" - upstream_schema_name = "shop" - ignore_columns = ["banned"] - } - `, nameSpace) -} - -func testAccSourceTableLoadGenBasicResource(nameSpace string) string { - return fmt.Sprintf(` - resource "materialize_source_load_generator" "test_loadgen" { - name = "%[1]s_loadgen" - load_generator_type = "AUCTION" - - auction_options { - tick_interval = "500ms" - } - } - - resource "materialize_source_table" "test_loadgen" { - name = "%[1]s_table_loadgen" - schema_name = "public" - database_name = "materialize" - - source { - name = materialize_source_load_generator.test_loadgen.name - } - - upstream_name = "bids" - } - `, nameSpace) -} - -func testAccSourceTableBasicResource(nameSpace string) string { - return fmt.Sprintf(` - resource "materialize_secret" "postgres_password" { - name = "%[1]s_secret" - value = "c2VjcmV0Cg==" - } - - resource "materialize_connection_postgres" "postgres_connection" { - name = "%[1]s_connection" - host = "postgres" - port = 5432 - user { - text = "postgres" - } - password { - name = materialize_secret.postgres_password.name - database_name = materialize_secret.postgres_password.database_name - schema_name = materialize_secret.postgres_password.schema_name - } - database = "postgres" - } - - resource "materialize_source_postgres" "test_source" { - name = "%[1]s_source" - cluster_name = "quickstart" - - postgres_connection { - name = materialize_connection_postgres.postgres_connection.name - schema_name = materialize_connection_postgres.postgres_connection.schema_name - database_name = materialize_connection_postgres.postgres_connection.database_name - } - publication = "mz_source" - table { - upstream_name = "table2" - upstream_schema_name = "public" - } - } - - resource "materialize_source_table" "test" { - name = "%[1]s_table" - schema_name = "public" - database_name = "materialize" - - source { - name = materialize_source_postgres.test_source.name - schema_name = "public" - database_name = "materialize" - } - - upstream_name = "table2" - upstream_schema_name = "public" - - text_columns = [ - "updated_at" - ] - } - `, nameSpace) -} - -func testAccSourceTableResource(nameSpace, upstreamName, ownershipRole, comment string) string { - return fmt.Sprintf(` - resource "materialize_secret" "postgres_password" { - name = "%[1]s_secret" - value = "c2VjcmV0Cg==" - } - - resource "materialize_connection_postgres" "postgres_connection" { - name = "%[1]s_connection" - host = "postgres" - port = 5432 - user { - text = "postgres" - } - password { - name = materialize_secret.postgres_password.name - database_name = materialize_secret.postgres_password.database_name - schema_name = materialize_secret.postgres_password.schema_name - } - database = "postgres" - } - - resource "materialize_source_postgres" "test_source" { - name = "%[1]s_source" - cluster_name = "quickstart" - - postgres_connection { - name = materialize_connection_postgres.postgres_connection.name - schema_name = materialize_connection_postgres.postgres_connection.schema_name - database_name = materialize_connection_postgres.postgres_connection.database_name - } - publication = "mz_source" - table { - upstream_name = "%[2]s" - upstream_schema_name = "public" - } - } - - resource "materialize_role" "test_role" { - name = "%[1]s_role" - } - - resource "materialize_source_table" "test" { - name = "%[1]s_table" - schema_name = "public" - database_name = "materialize" - - source { - name = materialize_source_postgres.test_source.name - schema_name = "public" - database_name = "materialize" - } - - upstream_name = "%[2]s" - upstream_schema_name = "public" - - text_columns = [ - "updated_at", - "id" - ] - - ownership_role = "%[3]s" - comment = "%[4]s" - - depends_on = [materialize_role.test_role] - } - `, nameSpace, upstreamName, ownershipRole, comment) -} - -func testAccCheckSourceTableExists(name string) resource.TestCheckFunc { - return func(s *terraform.State) error { - meta := testAccProvider.Meta() - db, _, err := utils.GetDBClientFromMeta(meta, nil) - if err != nil { - return fmt.Errorf("error getting DB client: %s", err) - } - r, ok := s.RootModule().Resources[name] - if !ok { - return fmt.Errorf("source table not found: %s", name) - } - _, err = materialize.ScanSourceTable(db, utils.ExtractId(r.Primary.ID)) - return err - } -} - -func testAccCheckAllSourceTableDestroyed(s *terraform.State) error { - meta := testAccProvider.Meta() - db, _, err := utils.GetDBClientFromMeta(meta, nil) - if err != nil { - return fmt.Errorf("error getting DB client: %s", err) - } - - for _, r := range s.RootModule().Resources { - if r.Type != "materialize_source_table" { - continue - } - - _, err := materialize.ScanSourceTable(db, utils.ExtractId(r.Primary.ID)) - if err == nil { - return fmt.Errorf("source table %v still exists", utils.ExtractId(r.Primary.ID)) - } else if err != sql.ErrNoRows { - return err - } - } - return nil -} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index c5f0886e..a52cd88b 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -133,7 +133,9 @@ func Provider(version string) *schema.Provider { "materialize_source_grant": resources.GrantSource(), "materialize_system_parameter": resources.SystemParameter(), "materialize_table": resources.Table(), - "materialize_source_table": resources.SourceTable(), + "materialize_source_table_load_generator": resources.SourceTableLoadGen(), + "materialize_source_table_mysql": resources.SourceTableMySQL(), + "materialize_source_table_postgres": resources.SourceTablePostgres(), "materialize_table_grant": resources.GrantTable(), "materialize_table_grant_default_privilege": resources.GrantTableDefaultPrivilege(), "materialize_type": resources.Type(), diff --git a/pkg/resources/resource_source_table.go b/pkg/resources/resource_source_table.go index 86b24fb6..9c42ccf7 100644 --- a/pkg/resources/resource_source_table.go +++ b/pkg/resources/resource_source_table.go @@ -3,7 +3,6 @@ package resources import ( "context" "database/sql" - "log" "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" @@ -12,134 +11,6 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) -var sourceTableSchema = map[string]*schema.Schema{ - "name": ObjectNameSchema("table", true, false), - "schema_name": SchemaNameSchema("table", false), - "database_name": DatabaseNameSchema("table", false), - "qualified_sql_name": QualifiedNameSchema("table"), - "source": IdentifierSchema(IdentifierSchemaParams{ - Elem: "source", - Description: "The source this table is created from.", - Required: true, - ForceNew: true, - }), - "upstream_name": { - Type: schema.TypeString, - Required: true, - ForceNew: true, - Description: "The name of the table in the upstream database.", - }, - "upstream_schema_name": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, - Description: "The schema of the table in the upstream database.", - }, - "text_columns": { - Description: "Columns to be decoded as text. Not supported for the load generator sources, if the source is a load generator, the attribute will be ignored.", - Type: schema.TypeList, - Elem: &schema.Schema{Type: schema.TypeString}, - Optional: true, - ForceNew: true, - }, - "ignore_columns": { - Description: "Ignore specific columns when reading data from MySQL. Only compatible with MySQL sources, if the source is not MySQL, the attribute will be ignored.", - Type: schema.TypeList, - Elem: &schema.Schema{Type: schema.TypeString}, - Optional: true, - ForceNew: true, - }, - "comment": CommentSchema(false), - "ownership_role": OwnershipRoleSchema(), - "region": RegionSchema(), -} - -func SourceTable() *schema.Resource { - return &schema.Resource{ - CreateContext: sourceTableCreate, - ReadContext: sourceTableRead, - UpdateContext: sourceTableUpdate, - DeleteContext: sourceTableDelete, - - Importer: &schema.ResourceImporter{ - StateContext: schema.ImportStatePassthroughContext, - }, - - Schema: sourceTableSchema, - } -} - -func sourceTableCreate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { - tableName := d.Get("name").(string) - schemaName := d.Get("schema_name").(string) - databaseName := d.Get("database_name").(string) - - metaDb, region, err := utils.GetDBClientFromMeta(meta, d) - if err != nil { - return diag.FromErr(err) - } - - o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} - b := materialize.NewSourceTableBuilder(metaDb, o) - - source := materialize.GetIdentifierSchemaStruct(d.Get("source")) - b.Source(source) - - b.UpstreamName(d.Get("upstream_name").(string)) - - if v, ok := d.GetOk("upstream_schema_name"); ok { - b.UpstreamSchemaName(v.(string)) - } - - if v, ok := d.GetOk("text_columns"); ok { - textColumns, err := materialize.GetSliceValueString("text_columns", v.([]interface{})) - if err != nil { - return diag.FromErr(err) - } - b.TextColumns(textColumns) - } - - if v, ok := d.GetOk("ignore_columns"); ok && len(v.([]interface{})) > 0 { - columns, err := materialize.GetSliceValueString("ignore_columns", v.([]interface{})) - if err != nil { - return diag.FromErr(err) - } - b.IgnoreColumns(columns) - } - - if err := b.Create(); err != nil { - return diag.FromErr(err) - } - - // Handle ownership - if v, ok := d.GetOk("ownership_role"); ok { - ownership := materialize.NewOwnershipBuilder(metaDb, o) - if err := ownership.Alter(v.(string)); err != nil { - log.Printf("[DEBUG] resource failed ownership, dropping object: %s", o.Name) - b.Drop() - return diag.FromErr(err) - } - } - - // Handle comments - if v, ok := d.GetOk("comment"); ok { - comment := materialize.NewCommentBuilder(metaDb, o) - if err := comment.Object(v.(string)); err != nil { - log.Printf("[DEBUG] resource failed comment, dropping object: %s", o.Name) - b.Drop() - return diag.FromErr(err) - } - } - - i, err := materialize.SourceTableId(metaDb, o) - if err != nil { - return diag.FromErr(err) - } - d.SetId(utils.TransformIdWithRegion(string(region), i)) - - return sourceTableRead(ctx, d, meta) -} - func sourceTableRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { i := d.Id() diff --git a/pkg/resources/resource_source_table_load_generator.go b/pkg/resources/resource_source_table_load_generator.go new file mode 100644 index 00000000..e895da06 --- /dev/null +++ b/pkg/resources/resource_source_table_load_generator.go @@ -0,0 +1,110 @@ +package resources + +import ( + "context" + "log" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +var sourceTableLoadGenSchema = map[string]*schema.Schema{ + "name": ObjectNameSchema("table", true, false), + "schema_name": SchemaNameSchema("table", false), + "database_name": DatabaseNameSchema("table", false), + "qualified_sql_name": QualifiedNameSchema("table"), + "source": IdentifierSchema(IdentifierSchemaParams{ + Elem: "source", + Description: "The source this table is created from.", + Required: true, + ForceNew: true, + }), + "upstream_name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "The name of the table in the upstream database.", + }, + "upstream_schema_name": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: "The schema of the table in the upstream database.", + }, + "comment": CommentSchema(false), + "ownership_role": OwnershipRoleSchema(), + "region": RegionSchema(), +} + +func SourceTableLoadGen() *schema.Resource { + return &schema.Resource{ + CreateContext: sourceTableLoadGenCreate, + ReadContext: sourceTableRead, + UpdateContext: sourceTableUpdate, + DeleteContext: sourceTableDelete, + + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + + Schema: sourceTableLoadGenSchema, + } +} + +func sourceTableLoadGenCreate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + tableName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewSourceTableLoadGenBuilder(metaDb, o) + + source := materialize.GetIdentifierSchemaStruct(d.Get("source")) + b.Source(source) + + b.UpstreamName(d.Get("upstream_name").(string)) + + if v, ok := d.GetOk("upstream_schema_name"); ok { + b.UpstreamSchemaName(v.(string)) + } + + if err := b.Create(); err != nil { + return diag.FromErr(err) + } + + // Handle ownership + if v, ok := d.GetOk("ownership_role"); ok { + ownership := materialize.NewOwnershipBuilder(metaDb, o) + if err := ownership.Alter(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed ownership, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + // Handle comments + if v, ok := d.GetOk("comment"); ok { + comment := materialize.NewCommentBuilder(metaDb, o) + if err := comment.Object(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed comment, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + i, err := materialize.SourceTableId(metaDb, o) + if err != nil { + return diag.FromErr(err) + } + d.SetId(utils.TransformIdWithRegion(string(region), i)) + + return sourceTableRead(ctx, d, meta) +} diff --git a/pkg/resources/resource_source_table_load_generator_test.go b/pkg/resources/resource_source_table_load_generator_test.go new file mode 100644 index 00000000..6643830b --- /dev/null +++ b/pkg/resources/resource_source_table_load_generator_test.go @@ -0,0 +1,113 @@ +package resources + +import ( + "context" + "testing" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/stretchr/testify/require" +) + +var inSourceTableLoadGen = map[string]interface{}{ + "name": "table", + "schema_name": "schema", + "database_name": "database", + "source": []interface{}{ + map[string]interface{}{ + "name": "loadgen", + "schema_name": "public", + "database_name": "materialize", + }, + }, + "upstream_name": "upstream_table", + "upstream_schema_name": "upstream_schema", + "text_columns": []interface{}{"column1", "column2"}, + "ignore_columns": []interface{}{"column3", "column4"}, +} + +func TestResourceSourceTableLoadGenCreate(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableLoadGen().Schema, inSourceTableLoadGen) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."loadgen" + \(REFERENCE "upstream_schema"."upstream_table"\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` + testhelpers.MockSourceTableScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableLoadGenCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableLoadGenRead(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableLoadGen().Schema, inSourceTableLoadGen) + d.SetId("u1") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableRead(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + + r.Equal("table", d.Get("name").(string)) + r.Equal("schema", d.Get("schema_name").(string)) + r.Equal("database", d.Get("database_name").(string)) + }) +} + +func TestResourceSourceTableLoadGenUpdate(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableLoadGen().Schema, inSourceTableLoadGen) + d.SetId("u1") + d.Set("name", "old_table") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + mock.ExpectExec(`ALTER TABLE "database"."schema"."" RENAME TO "database"."schema"."table"`).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableUpdate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableLoadGenDelete(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableLoadGen().Schema, inSourceTableLoadGen) + d.SetId("u1") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + mock.ExpectExec(`DROP TABLE "database"."schema"."table"`).WillReturnResult(sqlmock.NewResult(1, 1)) + + if err := sourceTableDelete(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} diff --git a/pkg/resources/resource_source_table_mysql.go b/pkg/resources/resource_source_table_mysql.go new file mode 100644 index 00000000..aa72bed6 --- /dev/null +++ b/pkg/resources/resource_source_table_mysql.go @@ -0,0 +1,202 @@ +package resources + +import ( + "context" + "database/sql" + "log" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +var sourceTableMySQLSchema = map[string]*schema.Schema{ + "name": ObjectNameSchema("table", true, false), + "schema_name": SchemaNameSchema("table", false), + "database_name": DatabaseNameSchema("table", false), + "qualified_sql_name": QualifiedNameSchema("table"), + "source": IdentifierSchema(IdentifierSchemaParams{ + Elem: "source", + Description: "The source this table is created from.", + Required: true, + ForceNew: true, + }), + "upstream_name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "The name of the table in the upstream database.", + }, + "upstream_schema_name": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: "The schema of the table in the upstream database.", + }, + "text_columns": { + Description: "Columns to be decoded as text.", + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + ForceNew: true, + }, + "ignore_columns": { + Description: "Ignore specific columns when reading data from MySQL.", + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + ForceNew: true, + }, + "comment": CommentSchema(false), + "ownership_role": OwnershipRoleSchema(), + "region": RegionSchema(), +} + +func SourceTableMySQL() *schema.Resource { + return &schema.Resource{ + CreateContext: sourceTableMySQLCreate, + ReadContext: sourceTableMySQLRead, + UpdateContext: sourceTableUpdate, + DeleteContext: sourceTableDelete, + + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + + Schema: sourceTableMySQLSchema, + } +} + +func sourceTableMySQLCreate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + tableName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewSourceTableMySQLBuilder(metaDb, o) + + source := materialize.GetIdentifierSchemaStruct(d.Get("source")) + b.Source(source) + + b.UpstreamName(d.Get("upstream_name").(string)) + + if v, ok := d.GetOk("upstream_schema_name"); ok { + b.UpstreamSchemaName(v.(string)) + } + + if v, ok := d.GetOk("text_columns"); ok { + textColumns, err := materialize.GetSliceValueString("text_columns", v.([]interface{})) + if err != nil { + return diag.FromErr(err) + } + b.TextColumns(textColumns) + } + + if v, ok := d.GetOk("ignore_columns"); ok && len(v.([]interface{})) > 0 { + columns, err := materialize.GetSliceValueString("ignore_columns", v.([]interface{})) + if err != nil { + return diag.FromErr(err) + } + b.IgnoreColumns(columns) + } + + if err := b.Create(); err != nil { + return diag.FromErr(err) + } + + // Handle ownership + if v, ok := d.GetOk("ownership_role"); ok { + ownership := materialize.NewOwnershipBuilder(metaDb, o) + if err := ownership.Alter(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed ownership, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + // Handle comments + if v, ok := d.GetOk("comment"); ok { + comment := materialize.NewCommentBuilder(metaDb, o) + if err := comment.Object(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed comment, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + i, err := materialize.SourceTableId(metaDb, o) + if err != nil { + return diag.FromErr(err) + } + d.SetId(utils.TransformIdWithRegion(string(region), i)) + + return sourceTableMySQLRead(ctx, d, meta) +} + +func sourceTableMySQLRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + i := d.Id() + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + t, err := materialize.ScanSourceTableMySQL(metaDb, utils.ExtractId(i)) + if err == sql.ErrNoRows { + d.SetId("") + return nil + } else if err != nil { + return diag.FromErr(err) + } + + d.SetId(utils.TransformIdWithRegion(string(region), i)) + + if err := d.Set("name", t.TableName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("schema_name", t.SchemaName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("database_name", t.DatabaseName.String); err != nil { + return diag.FromErr(err) + } + + source := []interface{}{ + map[string]interface{}{ + "name": t.SourceName.String, + "schema_name": t.SourceSchemaName.String, + "database_name": t.SourceDatabaseName.String, + }, + } + if err := d.Set("source", source); err != nil { + return diag.FromErr(err) + } + + // TODO: Set the upstream_name and upstream_schema_name once supported on the Materialize side + // if err := d.Set("upstream_name", t.UpstreamName.String); err != nil { + // return diag.FromErr(err) + // } + + // if err := d.Set("upstream_schema_name", t.UpstreamSchemaName.String); err != nil { + // return diag.FromErr(err) + // } + + if err := d.Set("ownership_role", t.OwnerName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("comment", t.Comment.String); err != nil { + return diag.FromErr(err) + } + + return nil +} diff --git a/pkg/resources/resource_source_table_test.go b/pkg/resources/resource_source_table_mysql_test.go similarity index 59% rename from pkg/resources/resource_source_table_test.go rename to pkg/resources/resource_source_table_mysql_test.go index 2262bc24..40a0ffed 100644 --- a/pkg/resources/resource_source_table_test.go +++ b/pkg/resources/resource_source_table_mysql_test.go @@ -4,15 +4,14 @@ import ( "context" "testing" + sqlmock "github.com/DATA-DOG/go-sqlmock" "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" - - sqlmock "github.com/DATA-DOG/go-sqlmock" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/stretchr/testify/require" ) -var inSourceTable = map[string]interface{}{ +var inSourceTableMySQL = map[string]interface{}{ "name": "table", "schema_name": "schema", "database_name": "database", @@ -29,16 +28,12 @@ var inSourceTable = map[string]interface{}{ "ignore_columns": []interface{}{"column3", "column4"}, } -func TestResourceSourceTableCreate(t *testing.T) { +func TestResourceSourceTableMySQLCreate(t *testing.T) { r := require.New(t) - d := schema.TestResourceDataRaw(t, SourceTable().Schema, inSourceTable) + d := schema.TestResourceDataRaw(t, SourceTableMySQL().Schema, inSourceTableMySQL) r.NotNil(d) testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { - // Expect source type query - sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` - testhelpers.MockSourceScanWithType(mock, sourceTypeQuery, "mysql") - // Create mock.ExpectExec( `CREATE TABLE "database"."schema"."table" @@ -55,46 +50,15 @@ func TestResourceSourceTableCreate(t *testing.T) { pp := `WHERE mz_tables.id = 'u1'` testhelpers.MockSourceTableScan(mock, pp) - if err := sourceTableCreate(context.TODO(), d, db); err != nil { - t.Fatal(err) - } - }) -} - -func TestResourceSourceTableCreateNonMySQL(t *testing.T) { - r := require.New(t) - d := schema.TestResourceDataRaw(t, SourceTable().Schema, inSourceTable) - r.NotNil(d) - - testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { - // Expect source type query - sourceTypeQuery := `WHERE mz_databases.name = 'materialize' AND mz_schemas.name = 'public' AND mz_sources.name = 'source'` - testhelpers.MockSourceScan(mock, sourceTypeQuery) - - // Create (without IGNORE COLUMNS) - mock.ExpectExec(`CREATE TABLE "database"."schema"."table" - FROM SOURCE "materialize"."public"."source" - \(REFERENCE "upstream_schema"."upstream_table"\) - WITH \(TEXT COLUMNS \(column1, column2\)\);`). - WillReturnResult(sqlmock.NewResult(1, 1)) - - // Query Id - ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` - testhelpers.MockSourceTableScan(mock, ip) - - // Query Params - pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) - - if err := sourceTableCreate(context.TODO(), d, db); err != nil { + if err := sourceTableMySQLCreate(context.TODO(), d, db); err != nil { t.Fatal(err) } }) } -func TestResourceSourceTableRead(t *testing.T) { +func TestResourceSourceTableMySQLRead(t *testing.T) { r := require.New(t) - d := schema.TestResourceDataRaw(t, SourceTable().Schema, inSourceTable) + d := schema.TestResourceDataRaw(t, SourceTableMySQL().Schema, inSourceTableMySQL) d.SetId("u1") r.NotNil(d) @@ -113,9 +77,9 @@ func TestResourceSourceTableRead(t *testing.T) { }) } -func TestResourceSourceTableUpdate(t *testing.T) { +func TestResourceSourceTableMySQLUpdate(t *testing.T) { r := require.New(t) - d := schema.TestResourceDataRaw(t, SourceTable().Schema, inSourceTable) + d := schema.TestResourceDataRaw(t, SourceTableMySQL().Schema, inSourceTableMySQL) d.SetId("u1") d.Set("name", "old_table") r.NotNil(d) @@ -133,9 +97,9 @@ func TestResourceSourceTableUpdate(t *testing.T) { }) } -func TestResourceSourceTableDelete(t *testing.T) { +func TestResourceSourceTableMySQLDelete(t *testing.T) { r := require.New(t) - d := schema.TestResourceDataRaw(t, SourceTable().Schema, inSourceTable) + d := schema.TestResourceDataRaw(t, SourceTableMySQL().Schema, inSourceTableMySQL) d.SetId("u1") r.NotNil(d) diff --git a/pkg/resources/resource_source_table_postgres.go b/pkg/resources/resource_source_table_postgres.go new file mode 100644 index 00000000..391f788a --- /dev/null +++ b/pkg/resources/resource_source_table_postgres.go @@ -0,0 +1,187 @@ +package resources + +import ( + "context" + "database/sql" + "log" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +var sourceTablePostgresSchema = map[string]*schema.Schema{ + "name": ObjectNameSchema("table", true, false), + "schema_name": SchemaNameSchema("table", false), + "database_name": DatabaseNameSchema("table", false), + "qualified_sql_name": QualifiedNameSchema("table"), + "source": IdentifierSchema(IdentifierSchemaParams{ + Elem: "source", + Description: "The source this table is created from.", + Required: true, + ForceNew: true, + }), + "upstream_name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "The name of the table in the upstream database.", + }, + "upstream_schema_name": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: "The schema of the table in the upstream database.", + }, + "text_columns": { + Description: "Columns to be decoded as text.", + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + ForceNew: true, + }, + "comment": CommentSchema(false), + "ownership_role": OwnershipRoleSchema(), + "region": RegionSchema(), +} + +func SourceTablePostgres() *schema.Resource { + return &schema.Resource{ + CreateContext: sourceTablePostgresCreate, + ReadContext: sourceTablePostgresRead, + UpdateContext: sourceTableUpdate, + DeleteContext: sourceTableDelete, + + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + + Schema: sourceTablePostgresSchema, + } +} + +func sourceTablePostgresCreate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + tableName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewSourceTablePostgresBuilder(metaDb, o) + + source := materialize.GetIdentifierSchemaStruct(d.Get("source")) + b.Source(source) + + b.UpstreamName(d.Get("upstream_name").(string)) + + if v, ok := d.GetOk("upstream_schema_name"); ok { + b.UpstreamSchemaName(v.(string)) + } + + if v, ok := d.GetOk("text_columns"); ok { + textColumns, err := materialize.GetSliceValueString("text_columns", v.([]interface{})) + if err != nil { + return diag.FromErr(err) + } + b.TextColumns(textColumns) + } + + if err := b.Create(); err != nil { + return diag.FromErr(err) + } + + // Handle ownership + if v, ok := d.GetOk("ownership_role"); ok { + ownership := materialize.NewOwnershipBuilder(metaDb, o) + if err := ownership.Alter(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed ownership, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + // Handle comments + if v, ok := d.GetOk("comment"); ok { + comment := materialize.NewCommentBuilder(metaDb, o) + if err := comment.Object(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed comment, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + i, err := materialize.SourceTableId(metaDb, o) + if err != nil { + return diag.FromErr(err) + } + d.SetId(utils.TransformIdWithRegion(string(region), i)) + + return sourceTablePostgresRead(ctx, d, meta) +} + +func sourceTablePostgresRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + i := d.Id() + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + t, err := materialize.ScanSourceTablePostgres(metaDb, utils.ExtractId(i)) + if err == sql.ErrNoRows { + d.SetId("") + return nil + } else if err != nil { + return diag.FromErr(err) + } + + d.SetId(utils.TransformIdWithRegion(string(region), i)) + + if err := d.Set("name", t.TableName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("schema_name", t.SchemaName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("database_name", t.DatabaseName.String); err != nil { + return diag.FromErr(err) + } + + source := []interface{}{ + map[string]interface{}{ + "name": t.SourceName.String, + "schema_name": t.SourceSchemaName.String, + "database_name": t.SourceDatabaseName.String, + }, + } + if err := d.Set("source", source); err != nil { + return diag.FromErr(err) + } + + // TODO: Set the upstream_name and upstream_schema_name once supported on the Materialize side + // if err := d.Set("upstream_name", t.UpstreamName.String); err != nil { + // return diag.FromErr(err) + // } + + // if err := d.Set("upstream_schema_name", t.UpstreamSchemaName.String); err != nil { + // return diag.FromErr(err) + // } + + if err := d.Set("ownership_role", t.OwnerName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("comment", t.Comment.String); err != nil { + return diag.FromErr(err) + } + + return nil +} diff --git a/pkg/resources/resource_source_table_postgres_test.go b/pkg/resources/resource_source_table_postgres_test.go new file mode 100644 index 00000000..ec655d16 --- /dev/null +++ b/pkg/resources/resource_source_table_postgres_test.go @@ -0,0 +1,112 @@ +package resources + +import ( + "context" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/stretchr/testify/require" +) + +var inSourceTablePostgres = map[string]interface{}{ + "name": "table", + "schema_name": "schema", + "database_name": "database", + "source": []interface{}{ + map[string]interface{}{ + "name": "source", + "schema_name": "public", + "database_name": "materialize", + }, + }, + "upstream_name": "upstream_table", + "upstream_schema_name": "upstream_schema", + "text_columns": []interface{}{"column1", "column2"}, + "ignore_columns": []interface{}{"column3", "column4"}, +} + +func TestResourceSourceTablePostgresCreate(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTablePostgres().Schema, inSourceTablePostgres) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec(`CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."source" + \(REFERENCE "upstream_schema"."upstream_table"\) + WITH \(TEXT COLUMNS \(column1, column2\)\);`). + WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` + testhelpers.MockSourceTableScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTablePostgresCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTablePostgresRead(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTablePostgres().Schema, inSourceTablePostgres) + d.SetId("u1") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableRead(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + + r.Equal("table", d.Get("name").(string)) + r.Equal("schema", d.Get("schema_name").(string)) + r.Equal("database", d.Get("database_name").(string)) + }) +} + +func TestResourceSourceTablePostgresUpdate(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTablePostgres().Schema, inSourceTablePostgres) + d.SetId("u1") + d.Set("name", "old_table") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + mock.ExpectExec(`ALTER TABLE "database"."schema"."" RENAME TO "database"."schema"."table"`).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableUpdate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTablePostgresDelete(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTablePostgres().Schema, inSourceTablePostgres) + d.SetId("u1") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + mock.ExpectExec(`DROP TABLE "database"."schema"."table"`).WillReturnResult(sqlmock.NewResult(1, 1)) + + if err := sourceTableDelete(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} diff --git a/templates/guides/materialize_source_table.md.tmpl b/templates/guides/materialize_source_table.md.tmpl index eaf224f5..8d768cd7 100644 --- a/templates/guides/materialize_source_table.md.tmpl +++ b/templates/guides/materialize_source_table.md.tmpl @@ -5,11 +5,13 @@ description: |- --- -# Source versioning: migrating to `materialize_source_table` Resource +# Source versioning: migrating to `materialize_source_table_{source}` Resource In previous versions of the Materialize Terraform provider, source tables were defined within the source resource itself and were considered subsources of the source rather than separate entities. -This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table` resource. +This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table_{source}` resource. + +For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. ## Old Approach @@ -38,18 +40,18 @@ The same approach was used for other source types such as Postgres and the load ## New Approach -The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table` resource. +The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table_mysql` resource. ## Manual Migration Process -This manual migration process requires users to create new source tables using the new `materialize_source_table` resource and then remove the old ones. +This manual migration process requires users to create new source tables using the new `materialize_source_table_{source}` resource and then remove the old ones. In this example, we will use MySQL as the source type. -### Step 1: Define `materialize_source_table` Resources +### Step 1: Define `materialize_source_table_mysql` Resources -Before making any changes to your existing source resources, create new `materialize_source_table` resources for each table that is currently defined within your sources. This ensures that the tables are preserved during the migration: +Before making any changes to your existing source resources, create new `materialize_source_table_mysql` resources for each table that is currently defined within your sources. This ensures that the tables are preserved during the migration: ```hcl -resource "materialize_source_table" "mysql_table_from_source" { +resource "materialize_source_table_mysql" "mysql_table_from_source" { name = "mysql_table1_from_source" schema_name = "public" database_name = "materialize" @@ -68,13 +70,13 @@ resource "materialize_source_table" "mysql_table_from_source" { ### Step 2: Apply the Changes -Run `terraform plan` and `terraform apply` to create the new `materialize_source_table` resources. This step ensures that the tables are defined separately from the source and are not removed from Materialize. +Run `terraform plan` and `terraform apply` to create the new `materialize_source_table_mysql` resources. This step ensures that the tables are defined separately from the source and are not removed from Materialize. > **Note:** This will start an ingestion process for the newly created source tables. ### Step 3: Remove Table Blocks from Source Resources -Once the new `materialize_source_table` resources are successfully created, you can safely remove the `table` blocks from your existing source resources: +Once the new `materialize_source_table_mysql` resources are successfully created, you can safely remove the `table` blocks from your existing source resources: ```hcl resource "materialize_source_mysql" "mysql_source" { @@ -84,6 +86,16 @@ resource "materialize_source_mysql" "mysql_source" { mysql_connection { name = materialize_connection_mysql.mysql_connection.name } + + // Remove the table blocks from here + - table { + - upstream_name = "mysql_table1" + - upstream_schema_name = "shop" + - name = "mysql_table1_local" + - + - ignore_columns = ["about"] + - + ... } ``` @@ -97,7 +109,9 @@ After removing the `table` blocks from your source resources, run `terraform pla After applying the changes, verify that your tables are still correctly set up in Materialize by checking the table definitions using Materialize’s SQL commands. -During the migration, you can use both the old `table` blocks and the new `materialize_source_table` resources simultaneously. This allows for a gradual transition until the old method is fully deprecated. +During the migration, you can use both the old `table` blocks and the new `materialize_source_table_{source}` resources simultaneously. This allows for a gradual transition until the old method is fully deprecated. + +The same approach can be used for other source types such as Postgres, eg. `materialize_source_table_postgres`. ## Automated Migration Process (TBD) @@ -105,12 +119,12 @@ During the migration, you can use both the old `table` blocks and the new `mater Once the migration on the Materialize side has been implemented, a more automated migration process will be available. The steps will include: -### Step 1: Define `materialize_source_table` Resources +### Step 1: Define `materialize_source_table_{source}` Resources -First, define the new `materialize_source_table` resources for each table: +First, define the new `materialize_source_table_mysql` resources for each table: ```hcl -resource "materialize_source_table" "mysql_table_from_source" { +resource "materialize_source_table_mysql" "mysql_table_from_source" { name = "mysql_table1_from_source" schema_name = "public" database_name = "materialize" @@ -150,10 +164,10 @@ resource "materialize_source_mysql" "mysql_source" { ### Step 3: Import the Existing Tables -You can then import the existing tables into the new `materialize_source_table` resources without disrupting your existing setup: +You can then import the existing tables into the new `materialize_source_table_mysql` resources without disrupting your existing setup: ```bash -terraform import materialize_source_table.mysql_table_from_source : +terraform import materialize_source_table_mysql.mysql_table_from_source : ``` Replace `` with the actual region and `` with the table ID. You can find the table ID by querying the `mz_tables` table. @@ -169,7 +183,7 @@ This approach allows you to migrate your tables safely without disrupting your e To import existing tables into your Terraform state using the manual migration process, use the following command: ```bash -terraform import materialize_source_table.table_name : +terraform import materialize_source_table_mysql.table_name : ``` Ensure you replace `` with the region where the table is located and `` with the ID of the table. From 1b6ccc2314bf4ba97cd7945b76cfb278b5b8cba2 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 13 Sep 2024 15:25:45 +0300 Subject: [PATCH 13/46] Add datasource --- docs/data-sources/source_table.md | 65 ++++++++ docs/data-sources/table.md | 13 ++ docs/resources/source_mysql.md | 4 - docs/resources/source_table_load_generator.md | 32 +++- docs/resources/source_table_mysql.md | 35 ++++ docs/resources/source_table_postgres.md | 34 ++++ .../materialize_source_table/data-source.tf | 10 ++ .../materialize_table/data-source.tf | 10 ++ .../materialize_source_mysql/resource.tf | 4 - .../import.sh | 5 + .../resource.tf | 15 ++ .../materialize_source_table_mysql/import.sh | 5 + .../resource.tf | 20 +++ .../import.sh | 5 + .../resource.tf | 19 +++ pkg/datasources/datasource_source_table.go | 157 ++++++++++++++++++ .../datasource_source_table_test.go | 54 ++++++ pkg/materialize/source_table.go | 15 ++ ...acceptance_datasource_source_table_test.go | 67 ++++++++ pkg/provider/provider.go | 1 + .../resource_source_table_load_generator.go | 2 +- 21 files changed, 562 insertions(+), 10 deletions(-) create mode 100644 docs/data-sources/source_table.md create mode 100644 examples/data-sources/materialize_source_table/data-source.tf create mode 100644 examples/data-sources/materialize_table/data-source.tf create mode 100644 examples/resources/materialize_source_table_load_generator/import.sh create mode 100644 examples/resources/materialize_source_table_load_generator/resource.tf create mode 100644 examples/resources/materialize_source_table_mysql/import.sh create mode 100644 examples/resources/materialize_source_table_mysql/resource.tf create mode 100644 examples/resources/materialize_source_table_postgres/import.sh create mode 100644 examples/resources/materialize_source_table_postgres/resource.tf create mode 100644 pkg/datasources/datasource_source_table.go create mode 100644 pkg/datasources/datasource_source_table_test.go create mode 100644 pkg/provider/acceptance_datasource_source_table_test.go diff --git a/docs/data-sources/source_table.md b/docs/data-sources/source_table.md new file mode 100644 index 00000000..29a9fa8b --- /dev/null +++ b/docs/data-sources/source_table.md @@ -0,0 +1,65 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "materialize_source_table Data Source - terraform-provider-materialize" +subcategory: "" +description: |- + +--- + +# materialize_source_table (Data Source) + + + +## Example Usage + +```terraform +data "materialize_source_table" "all" {} + +data "materialize_source_table" "materialize" { + database_name = "materialize" +} + +data "materialize_source_table" "materialize_schema" { + database_name = "materialize" + schema_name = "schema" +} +``` + + +## Schema + +### Optional + +- `database_name` (String) Limit tables to a specific database +- `region` (String) The region in which the resource is located. +- `schema_name` (String) Limit tables to a specific schema within a specific database + +### Read-Only + +- `id` (String) The ID of this resource. +- `tables` (List of Object) The source tables in the account (see [below for nested schema](#nestedatt--tables)) + + +### Nested Schema for `tables` + +Read-Only: + +- `comment` (String) +- `database_name` (String) +- `id` (String) +- `name` (String) +- `owner_name` (String) +- `schema_name` (String) +- `source` (List of Object) (see [below for nested schema](#nestedobjatt--tables--source)) +- `source_type` (String) +- `upstream_name` (String) +- `upstream_schema_name` (String) + + +### Nested Schema for `tables.source` + +Read-Only: + +- `database_name` (String) +- `name` (String) +- `schema_name` (String) diff --git a/docs/data-sources/table.md b/docs/data-sources/table.md index 2a29c7f4..61cc3f5d 100644 --- a/docs/data-sources/table.md +++ b/docs/data-sources/table.md @@ -10,7 +10,20 @@ description: |- +## Example Usage +```terraform +data "materialize_table" "all" {} + +data "materialize_table" "materialize" { + database_name = "materialize" +} + +data "materialize_table" "materialize_schema" { + database_name = "materialize" + schema_name = "schema" +} +``` ## Schema diff --git a/docs/resources/source_mysql.md b/docs/resources/source_mysql.md index 67559839..a7829e0d 100644 --- a/docs/resources/source_mysql.md +++ b/docs/resources/source_mysql.md @@ -36,10 +36,6 @@ resource "materialize_source_mysql" "test" { name = "mysql_table2_local" } } - -# CREATE SOURCE schema.source_mysql -# FROM MYSQL CONNECTION "database"."schema"."mysql_connection" (PUBLICATION 'mz_source') -# FOR TABLES (shop.mysql_table1 AS mysql_table1_local, shop.mysql_table2 AS mysql_table2_local); ``` diff --git a/docs/resources/source_table_load_generator.md b/docs/resources/source_table_load_generator.md index c692c274..c959aa9e 100644 --- a/docs/resources/source_table_load_generator.md +++ b/docs/resources/source_table_load_generator.md @@ -10,7 +10,25 @@ description: |- +## Example Usage +```terraform +resource "materialize_source_table_load_generator" "load_generator_table_from_source" { + name = "load_generator_table_from_source" + schema_name = "public" + database_name = "materialize" + + # The load generator source must be of type: `auction_options`, `marketing_options`, and `tpch_options` load generator sources. + source { + name = materialize_source_load_generator.example.name + schema_name = materialize_source_load_generator.example.schema_name + database_name = materialize_source_load_generator.example.database_name + } + + upstream_name = "load_generator_table_name" # The name of the table from the load generator + +} +``` ## Schema @@ -18,7 +36,7 @@ description: |- ### Required - `name` (String) The identifier for the table. -- `source` (Block List, Min: 1, Max: 1) The source this table is created from. (see [below for nested schema](#nestedblock--source)) +- `source` (Block List, Min: 1, Max: 1) The source this table is created from. Compatible with `auction_options`, `marketing_options`, and `tpch_options` load generator sources. (see [below for nested schema](#nestedblock--source)) - `upstream_name` (String) The name of the table in the upstream database. ### Optional @@ -46,3 +64,15 @@ Optional: - `database_name` (String) The source database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `schema_name` (String) The source schema name. Defaults to `public`. + +## Import + +Import is supported using the following syntax: + +```shell +# Source tables can be imported using the source id: +terraform import materialize_source_table_load_generator.example_source_table_loadgen : + +# Source id and information be found in the `mz_catalog.mz_tables` table +# The region is the region where the database is located (e.g. aws/us-east-1) +``` diff --git a/docs/resources/source_table_mysql.md b/docs/resources/source_table_mysql.md index d83836f7..d0a73d8e 100644 --- a/docs/resources/source_table_mysql.md +++ b/docs/resources/source_table_mysql.md @@ -10,7 +10,30 @@ description: |- +## Example Usage +```terraform +resource "materialize_source_table_mysql" "mysql_table_from_source" { + name = "mysql_table_from_source" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.example.name + schema_name = materialize_source_mysql.example.schema_name + database_name = materialize_source_mysql.example.database_name + } + + upstream_name = "mysql_table_name" # The name of the table in the MySQL database + upstream_schema_name = "mysql_db_name" # The name of the database in the MySQL database + + text_columns = [ + "updated_at" + ] + + ignore_columns = ["about"] +} +``` ## Schema @@ -48,3 +71,15 @@ Optional: - `database_name` (String) The source database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `schema_name` (String) The source schema name. Defaults to `public`. + +## Import + +Import is supported using the following syntax: + +```shell +# Source tables can be imported using the source id: +terraform import materialize_source_table_mysql.example_source_table_mysql : + +# Source id and information be found in the `mz_catalog.mz_tables` table +# The region is the region where the database is located (e.g. aws/us-east-1) +``` diff --git a/docs/resources/source_table_postgres.md b/docs/resources/source_table_postgres.md index 4b0881ec..626fd652 100644 --- a/docs/resources/source_table_postgres.md +++ b/docs/resources/source_table_postgres.md @@ -10,7 +10,29 @@ description: |- +## Example Usage +```terraform +resource "materialize_source_table_postgres" "postgres_table_from_source" { + name = "postgres_table_from_source" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_postgres.example.name + schema_name = materialize_source_postgres.example.schema_name + database_name = materialize_source_postgres.example.database_name + } + + upstream_name = "postgres_table_name" # The name of the table in the postgres database + upstream_schema_name = "postgres_schema_name" # The name of the database in the postgres database + + text_columns = [ + "updated_at" + ] + +} +``` ## Schema @@ -47,3 +69,15 @@ Optional: - `database_name` (String) The source database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `schema_name` (String) The source schema name. Defaults to `public`. + +## Import + +Import is supported using the following syntax: + +```shell +# Source tables can be imported using the source id: +terraform import materialize_source_table_postgres.example_source_table_postgres : + +# Source id and information be found in the `mz_catalog.mz_tables` table +# The region is the region where the database is located (e.g. aws/us-east-1) +``` diff --git a/examples/data-sources/materialize_source_table/data-source.tf b/examples/data-sources/materialize_source_table/data-source.tf new file mode 100644 index 00000000..30e5c940 --- /dev/null +++ b/examples/data-sources/materialize_source_table/data-source.tf @@ -0,0 +1,10 @@ +data "materialize_source_table" "all" {} + +data "materialize_source_table" "materialize" { + database_name = "materialize" +} + +data "materialize_source_table" "materialize_schema" { + database_name = "materialize" + schema_name = "schema" +} diff --git a/examples/data-sources/materialize_table/data-source.tf b/examples/data-sources/materialize_table/data-source.tf new file mode 100644 index 00000000..9f79d339 --- /dev/null +++ b/examples/data-sources/materialize_table/data-source.tf @@ -0,0 +1,10 @@ +data "materialize_table" "all" {} + +data "materialize_table" "materialize" { + database_name = "materialize" +} + +data "materialize_table" "materialize_schema" { + database_name = "materialize" + schema_name = "schema" +} diff --git a/examples/resources/materialize_source_mysql/resource.tf b/examples/resources/materialize_source_mysql/resource.tf index e891a474..72fbb0f6 100644 --- a/examples/resources/materialize_source_mysql/resource.tf +++ b/examples/resources/materialize_source_mysql/resource.tf @@ -21,7 +21,3 @@ resource "materialize_source_mysql" "test" { name = "mysql_table2_local" } } - -# CREATE SOURCE schema.source_mysql -# FROM MYSQL CONNECTION "database"."schema"."mysql_connection" (PUBLICATION 'mz_source') -# FOR TABLES (shop.mysql_table1 AS mysql_table1_local, shop.mysql_table2 AS mysql_table2_local); diff --git a/examples/resources/materialize_source_table_load_generator/import.sh b/examples/resources/materialize_source_table_load_generator/import.sh new file mode 100644 index 00000000..50e09749 --- /dev/null +++ b/examples/resources/materialize_source_table_load_generator/import.sh @@ -0,0 +1,5 @@ +# Source tables can be imported using the source id: +terraform import materialize_source_table_load_generator.example_source_table_loadgen : + +# Source id and information be found in the `mz_catalog.mz_tables` table +# The region is the region where the database is located (e.g. aws/us-east-1) diff --git a/examples/resources/materialize_source_table_load_generator/resource.tf b/examples/resources/materialize_source_table_load_generator/resource.tf new file mode 100644 index 00000000..190a148d --- /dev/null +++ b/examples/resources/materialize_source_table_load_generator/resource.tf @@ -0,0 +1,15 @@ +resource "materialize_source_table_load_generator" "load_generator_table_from_source" { + name = "load_generator_table_from_source" + schema_name = "public" + database_name = "materialize" + + # The load generator source must be of type: `auction_options`, `marketing_options`, and `tpch_options` load generator sources. + source { + name = materialize_source_load_generator.example.name + schema_name = materialize_source_load_generator.example.schema_name + database_name = materialize_source_load_generator.example.database_name + } + + upstream_name = "load_generator_table_name" # The name of the table from the load generator + +} diff --git a/examples/resources/materialize_source_table_mysql/import.sh b/examples/resources/materialize_source_table_mysql/import.sh new file mode 100644 index 00000000..4ed92a32 --- /dev/null +++ b/examples/resources/materialize_source_table_mysql/import.sh @@ -0,0 +1,5 @@ +# Source tables can be imported using the source id: +terraform import materialize_source_table_mysql.example_source_table_mysql : + +# Source id and information be found in the `mz_catalog.mz_tables` table +# The region is the region where the database is located (e.g. aws/us-east-1) diff --git a/examples/resources/materialize_source_table_mysql/resource.tf b/examples/resources/materialize_source_table_mysql/resource.tf new file mode 100644 index 00000000..78a02460 --- /dev/null +++ b/examples/resources/materialize_source_table_mysql/resource.tf @@ -0,0 +1,20 @@ +resource "materialize_source_table_mysql" "mysql_table_from_source" { + name = "mysql_table_from_source" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.example.name + schema_name = materialize_source_mysql.example.schema_name + database_name = materialize_source_mysql.example.database_name + } + + upstream_name = "mysql_table_name" # The name of the table in the MySQL database + upstream_schema_name = "mysql_db_name" # The name of the database in the MySQL database + + text_columns = [ + "updated_at" + ] + + ignore_columns = ["about"] +} diff --git a/examples/resources/materialize_source_table_postgres/import.sh b/examples/resources/materialize_source_table_postgres/import.sh new file mode 100644 index 00000000..1580dd37 --- /dev/null +++ b/examples/resources/materialize_source_table_postgres/import.sh @@ -0,0 +1,5 @@ +# Source tables can be imported using the source id: +terraform import materialize_source_table_postgres.example_source_table_postgres : + +# Source id and information be found in the `mz_catalog.mz_tables` table +# The region is the region where the database is located (e.g. aws/us-east-1) diff --git a/examples/resources/materialize_source_table_postgres/resource.tf b/examples/resources/materialize_source_table_postgres/resource.tf new file mode 100644 index 00000000..9744511f --- /dev/null +++ b/examples/resources/materialize_source_table_postgres/resource.tf @@ -0,0 +1,19 @@ +resource "materialize_source_table_postgres" "postgres_table_from_source" { + name = "postgres_table_from_source" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_postgres.example.name + schema_name = materialize_source_postgres.example.schema_name + database_name = materialize_source_postgres.example.database_name + } + + upstream_name = "postgres_table_name" # The name of the table in the postgres database + upstream_schema_name = "postgres_schema_name" # The name of the database in the postgres database + + text_columns = [ + "updated_at" + ] + +} diff --git a/pkg/datasources/datasource_source_table.go b/pkg/datasources/datasource_source_table.go new file mode 100644 index 00000000..8d39b625 --- /dev/null +++ b/pkg/datasources/datasource_source_table.go @@ -0,0 +1,157 @@ +package datasources + +import ( + "context" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +func SourceTable() *schema.Resource { + return &schema.Resource{ + ReadContext: sourceTableRead, + Schema: map[string]*schema.Schema{ + "database_name": { + Type: schema.TypeString, + Optional: true, + Description: "Limit tables to a specific database", + }, + "schema_name": { + Type: schema.TypeString, + Optional: true, + Description: "Limit tables to a specific schema within a specific database", + RequiredWith: []string{"database_name"}, + }, + "tables": { + Type: schema.TypeList, + Computed: true, + Description: "The source tables in the account", + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "id": { + Type: schema.TypeString, + Computed: true, + Description: "The ID of the source table", + }, + "name": { + Type: schema.TypeString, + Computed: true, + Description: "The name of the source table", + }, + "schema_name": { + Type: schema.TypeString, + Computed: true, + Description: "The schema name of the source table", + }, + "database_name": { + Type: schema.TypeString, + Computed: true, + Description: "The database name of the source table", + }, + "source": { + Type: schema.TypeList, + Computed: true, + Description: "Information about the source", + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "name": { + Type: schema.TypeString, + Computed: true, + Description: "The name of the source", + }, + "schema_name": { + Type: schema.TypeString, + Computed: true, + Description: "The schema name of the source", + }, + "database_name": { + Type: schema.TypeString, + Computed: true, + Description: "The database name of the source", + }, + }, + }, + }, + "source_type": { + Type: schema.TypeString, + Computed: true, + Description: "The type of the source", + }, + "upstream_name": { + Type: schema.TypeString, + Computed: true, + Description: "The name of the upstream table", + }, + "upstream_schema_name": { + Type: schema.TypeString, + Computed: true, + Description: "The schema name of the upstream table", + }, + "comment": { + Type: schema.TypeString, + Computed: true, + Description: "The comment on the source table", + }, + "owner_name": { + Type: schema.TypeString, + Computed: true, + Description: "The name of the owner of the source table", + }, + }, + }, + }, + "region": RegionSchema(), + }, + } +} + +func sourceTableRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + var diags diag.Diagnostics + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + dataSource, err := materialize.ListSourceTables(metaDb, schemaName, databaseName) + if err != nil { + return diag.FromErr(err) + } + + tableFormats := []map[string]interface{}{} + for _, p := range dataSource { + tableMap := map[string]interface{}{ + "id": p.TableId.String, + "name": p.TableName.String, + "schema_name": p.SchemaName.String, + "database_name": p.DatabaseName.String, + "source_type": p.SourceType.String, + "upstream_name": p.UpstreamName.String, + "upstream_schema_name": p.UpstreamSchemaName.String, + "comment": p.Comment.String, + "owner_name": p.OwnerName.String, + } + + sourceMap := map[string]interface{}{ + "name": p.SourceName.String, + "schema_name": p.SourceSchemaName.String, + "database_name": p.SourceDatabaseName.String, + } + tableMap["source"] = []interface{}{sourceMap} + + tableFormats = append(tableFormats, tableMap) + } + + if err := d.Set("tables", tableFormats); err != nil { + return diag.FromErr(err) + } + + SetId(string(region), "source_tables", databaseName, schemaName, d) + + return diags +} diff --git a/pkg/datasources/datasource_source_table_test.go b/pkg/datasources/datasource_source_table_test.go new file mode 100644 index 00000000..7b084a08 --- /dev/null +++ b/pkg/datasources/datasource_source_table_test.go @@ -0,0 +1,54 @@ +package datasources + +import ( + "context" + "testing" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/stretchr/testify/require" +) + +func TestSourceTableDatasource(t *testing.T) { + r := require.New(t) + + in := map[string]interface{}{ + "schema_name": "schema", + "database_name": "database", + } + d := schema.TestResourceDataRaw(t, SourceTable().Schema, in) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + p := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema'` + testhelpers.MockSourceTableScan(mock, p) + + if err := sourceTableRead(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + + // Verify the results + tables := d.Get("tables").([]interface{}) + r.Equal(1, len(tables)) + + table := tables[0].(map[string]interface{}) + r.Equal("u1", table["id"]) + r.Equal("table", table["name"]) + r.Equal("schema", table["schema_name"]) + r.Equal("database", table["database_name"]) + r.Equal("KAFKA", table["source_type"]) + // TODO: Update once upstream_name and upstream_schema_name are supported + r.Equal("", table["upstream_name"]) + r.Equal("", table["upstream_schema_name"]) + r.Equal("comment", table["comment"]) + r.Equal("materialize", table["owner_name"]) + + source := table["source"].([]interface{})[0].(map[string]interface{}) + r.Equal("source", source["name"]) + r.Equal("public", source["schema_name"]) + r.Equal("materialize", source["database_name"]) + }) +} diff --git a/pkg/materialize/source_table.go b/pkg/materialize/source_table.go index 42e66abd..c24c3ec0 100644 --- a/pkg/materialize/source_table.go +++ b/pkg/materialize/source_table.go @@ -164,3 +164,18 @@ func (b *SourceTableBuilder) BaseCreate(sourceType string, additionalOptions fun q.WriteString(`;`) return b.ddl.exec(q.String()) } + +func ListSourceTables(conn *sqlx.DB, schemaName, databaseName string) ([]SourceTableParams, error) { + p := map[string]string{ + "mz_schemas.name": schemaName, + "mz_databases.name": databaseName, + } + q := sourceTableQuery.QueryPredicate(p) + + var c []SourceTableParams + if err := conn.Select(&c, q); err != nil { + return c, err + } + + return c, nil +} diff --git a/pkg/provider/acceptance_datasource_source_table_test.go b/pkg/provider/acceptance_datasource_source_table_test.go new file mode 100644 index 00000000..ab7befe3 --- /dev/null +++ b/pkg/provider/acceptance_datasource_source_table_test.go @@ -0,0 +1,67 @@ +package provider + +import ( + "fmt" + "testing" + + "github.com/hashicorp/terraform-plugin-testing/helper/acctest" + "github.com/hashicorp/terraform-plugin-testing/helper/resource" +) + +func TestAccDataSourceSourceTable_basic(t *testing.T) { + nameSpace := acctest.RandomWithPrefix("tf_test") + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + Steps: []resource.TestStep{ + { + Config: testAccDataSourceSourceTable(nameSpace), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("data.materialize_source_table.test", "tables.0.name", fmt.Sprintf("%s_table", nameSpace)), + resource.TestCheckResourceAttr("data.materialize_source_table.test", "tables.0.schema_name", "public"), + resource.TestCheckResourceAttr("data.materialize_source_table.test", "tables.0.database_name", "materialize"), + resource.TestCheckResourceAttr("data.materialize_source_table.test", "tables.0.source.#", "1"), + resource.TestCheckResourceAttr("data.materialize_source_table.test", "tables.0.source.0.name", fmt.Sprintf("%s_source", nameSpace)), + resource.TestCheckResourceAttr("data.materialize_source_table.test", "tables.0.source.0.schema_name", "public"), + resource.TestCheckResourceAttr("data.materialize_source_table.test", "tables.0.source.0.database_name", "materialize"), + resource.TestCheckResourceAttr("data.materialize_source_table.test", "tables.0.source_type", "load-generator"), + resource.TestCheckResourceAttr("data.materialize_source_table.test", "tables.0.comment", "test comment"), + resource.TestCheckResourceAttrSet("data.materialize_source_table.test", "tables.0.owner_name"), + ), + }, + }, + }) +} + +func testAccDataSourceSourceTable(nameSpace string) string { + return fmt.Sprintf(` +resource "materialize_source_load_generator" "test" { + name = "%[1]s_source" + schema_name = "public" + database_name = "materialize" + load_generator_type = "AUCTION" + auction_options { + tick_interval = "1s" + } +} + +resource "materialize_source_table_load_generator" "test" { + name = "%[1]s_table" + schema_name = "public" + database_name = "materialize" + source { + name = materialize_source_load_generator.test.name + schema_name = materialize_source_load_generator.test.schema_name + database_name = materialize_source_load_generator.test.database_name + } + upstream_name = "bids" + comment = "test comment" +} + +data "materialize_source_table" "test" { + schema_name = "public" + database_name = "materialize" + depends_on = [materialize_source_table_load_generator.test] +} +`, nameSpace) +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index a52cd88b..32079903 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -161,6 +161,7 @@ func Provider(version string) *schema.Provider { "materialize_secret": datasources.Secret(), "materialize_sink": datasources.Sink(), "materialize_source": datasources.Source(), + "materialize_source_table": datasources.SourceTable(), "materialize_scim_groups": datasources.SCIMGroups(), "materialize_scim_configs": datasources.SCIMConfigs(), "materialize_sso_config": datasources.SSOConfig(), diff --git a/pkg/resources/resource_source_table_load_generator.go b/pkg/resources/resource_source_table_load_generator.go index e895da06..39d321e1 100644 --- a/pkg/resources/resource_source_table_load_generator.go +++ b/pkg/resources/resource_source_table_load_generator.go @@ -18,7 +18,7 @@ var sourceTableLoadGenSchema = map[string]*schema.Schema{ "qualified_sql_name": QualifiedNameSchema("table"), "source": IdentifierSchema(IdentifierSchemaParams{ Elem: "source", - Description: "The source this table is created from.", + Description: "The source this table is created from. Compatible with `auction_options`, `marketing_options`, and `tpch_options` load generator sources.", Required: true, ForceNew: true, }), From a8a21e4299cf03ba3608ab3f4f3c54e815fd3583 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 13 Sep 2024 15:27:58 +0300 Subject: [PATCH 14/46] Format examples --- docs/resources/source_table_load_generator.md | 8 ++++---- docs/resources/source_table_mysql.md | 10 +++++----- docs/resources/source_table_postgres.md | 10 +++++----- .../resource.tf | 8 ++++---- .../materialize_source_table_mysql/resource.tf | 10 +++++----- .../materialize_source_table_postgres/resource.tf | 10 +++++----- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/docs/resources/source_table_load_generator.md b/docs/resources/source_table_load_generator.md index c959aa9e..a8f1770e 100644 --- a/docs/resources/source_table_load_generator.md +++ b/docs/resources/source_table_load_generator.md @@ -14,9 +14,9 @@ description: |- ```terraform resource "materialize_source_table_load_generator" "load_generator_table_from_source" { - name = "load_generator_table_from_source" - schema_name = "public" - database_name = "materialize" + name = "load_generator_table_from_source" + schema_name = "public" + database_name = "materialize" # The load generator source must be of type: `auction_options`, `marketing_options`, and `tpch_options` load generator sources. source { @@ -25,7 +25,7 @@ resource "materialize_source_table_load_generator" "load_generator_table_from_so database_name = materialize_source_load_generator.example.database_name } - upstream_name = "load_generator_table_name" # The name of the table from the load generator + upstream_name = "load_generator_table_name" # The name of the table from the load generator } ``` diff --git a/docs/resources/source_table_mysql.md b/docs/resources/source_table_mysql.md index d0a73d8e..309374e2 100644 --- a/docs/resources/source_table_mysql.md +++ b/docs/resources/source_table_mysql.md @@ -14,9 +14,9 @@ description: |- ```terraform resource "materialize_source_table_mysql" "mysql_table_from_source" { - name = "mysql_table_from_source" - schema_name = "public" - database_name = "materialize" + name = "mysql_table_from_source" + schema_name = "public" + database_name = "materialize" source { name = materialize_source_mysql.example.name @@ -24,8 +24,8 @@ resource "materialize_source_table_mysql" "mysql_table_from_source" { database_name = materialize_source_mysql.example.database_name } - upstream_name = "mysql_table_name" # The name of the table in the MySQL database - upstream_schema_name = "mysql_db_name" # The name of the database in the MySQL database + upstream_name = "mysql_table_name" # The name of the table in the MySQL database + upstream_schema_name = "mysql_db_name" # The name of the database in the MySQL database text_columns = [ "updated_at" diff --git a/docs/resources/source_table_postgres.md b/docs/resources/source_table_postgres.md index 626fd652..8eebc851 100644 --- a/docs/resources/source_table_postgres.md +++ b/docs/resources/source_table_postgres.md @@ -14,9 +14,9 @@ description: |- ```terraform resource "materialize_source_table_postgres" "postgres_table_from_source" { - name = "postgres_table_from_source" - schema_name = "public" - database_name = "materialize" + name = "postgres_table_from_source" + schema_name = "public" + database_name = "materialize" source { name = materialize_source_postgres.example.name @@ -24,8 +24,8 @@ resource "materialize_source_table_postgres" "postgres_table_from_source" { database_name = materialize_source_postgres.example.database_name } - upstream_name = "postgres_table_name" # The name of the table in the postgres database - upstream_schema_name = "postgres_schema_name" # The name of the database in the postgres database + upstream_name = "postgres_table_name" # The name of the table in the postgres database + upstream_schema_name = "postgres_schema_name" # The name of the database in the postgres database text_columns = [ "updated_at" diff --git a/examples/resources/materialize_source_table_load_generator/resource.tf b/examples/resources/materialize_source_table_load_generator/resource.tf index 190a148d..4bc698ea 100644 --- a/examples/resources/materialize_source_table_load_generator/resource.tf +++ b/examples/resources/materialize_source_table_load_generator/resource.tf @@ -1,7 +1,7 @@ resource "materialize_source_table_load_generator" "load_generator_table_from_source" { - name = "load_generator_table_from_source" - schema_name = "public" - database_name = "materialize" + name = "load_generator_table_from_source" + schema_name = "public" + database_name = "materialize" # The load generator source must be of type: `auction_options`, `marketing_options`, and `tpch_options` load generator sources. source { @@ -10,6 +10,6 @@ resource "materialize_source_table_load_generator" "load_generator_table_from_so database_name = materialize_source_load_generator.example.database_name } - upstream_name = "load_generator_table_name" # The name of the table from the load generator + upstream_name = "load_generator_table_name" # The name of the table from the load generator } diff --git a/examples/resources/materialize_source_table_mysql/resource.tf b/examples/resources/materialize_source_table_mysql/resource.tf index 78a02460..bedac347 100644 --- a/examples/resources/materialize_source_table_mysql/resource.tf +++ b/examples/resources/materialize_source_table_mysql/resource.tf @@ -1,7 +1,7 @@ resource "materialize_source_table_mysql" "mysql_table_from_source" { - name = "mysql_table_from_source" - schema_name = "public" - database_name = "materialize" + name = "mysql_table_from_source" + schema_name = "public" + database_name = "materialize" source { name = materialize_source_mysql.example.name @@ -9,8 +9,8 @@ resource "materialize_source_table_mysql" "mysql_table_from_source" { database_name = materialize_source_mysql.example.database_name } - upstream_name = "mysql_table_name" # The name of the table in the MySQL database - upstream_schema_name = "mysql_db_name" # The name of the database in the MySQL database + upstream_name = "mysql_table_name" # The name of the table in the MySQL database + upstream_schema_name = "mysql_db_name" # The name of the database in the MySQL database text_columns = [ "updated_at" diff --git a/examples/resources/materialize_source_table_postgres/resource.tf b/examples/resources/materialize_source_table_postgres/resource.tf index 9744511f..60bac8be 100644 --- a/examples/resources/materialize_source_table_postgres/resource.tf +++ b/examples/resources/materialize_source_table_postgres/resource.tf @@ -1,7 +1,7 @@ resource "materialize_source_table_postgres" "postgres_table_from_source" { - name = "postgres_table_from_source" - schema_name = "public" - database_name = "materialize" + name = "postgres_table_from_source" + schema_name = "public" + database_name = "materialize" source { name = materialize_source_postgres.example.name @@ -9,8 +9,8 @@ resource "materialize_source_table_postgres" "postgres_table_from_source" { database_name = materialize_source_postgres.example.database_name } - upstream_name = "postgres_table_name" # The name of the table in the postgres database - upstream_schema_name = "postgres_schema_name" # The name of the database in the postgres database + upstream_name = "postgres_table_name" # The name of the table in the postgres database + upstream_schema_name = "postgres_schema_name" # The name of the database in the postgres database text_columns = [ "updated_at" From 18900b273ba254756c5e8dfc69bfeca07fd1c300 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 16 Sep 2024 15:18:38 +0300 Subject: [PATCH 15/46] Add Kafka source table resource --- docs/resources/source_kafka.md | 26 +- docs/resources/source_load_generator.md | 2 +- docs/resources/source_mysql.md | 6 +- docs/resources/source_postgres.md | 4 +- docs/resources/source_table_kafka.md | 325 +++++++++++++ pkg/materialize/format_specs.go | 2 +- pkg/materialize/source_table_kafka.go | 332 +++++++++++++ pkg/materialize/source_table_kafka_test.go | 123 +++++ .../source_table_load_generator.go | 1 - pkg/provider/provider.go | 1 + pkg/resources/resource_source_kafka.go | 39 +- .../resource_source_load_generator.go | 2 +- pkg/resources/resource_source_mysql.go | 14 +- pkg/resources/resource_source_postgres.go | 12 +- pkg/resources/resource_source_table_kafka.go | 349 +++++++++++++ .../resource_source_table_kafka_test.go | 458 ++++++++++++++++++ 16 files changed, 1648 insertions(+), 48 deletions(-) create mode 100644 docs/resources/source_table_kafka.md create mode 100644 pkg/materialize/source_table_kafka.go create mode 100644 pkg/materialize/source_table_kafka_test.go create mode 100644 pkg/resources/resource_source_table_kafka.go create mode 100644 pkg/resources/resource_source_table_kafka_test.go diff --git a/docs/resources/source_kafka.md b/docs/resources/source_kafka.md index 9095035f..01f6a47f 100644 --- a/docs/resources/source_kafka.md +++ b/docs/resources/source_kafka.md @@ -56,25 +56,25 @@ resource "materialize_source_kafka" "example_source_kafka" { - `cluster_name` (String) The cluster to maintain this source. - `comment` (String) Comment on an object in the database. - `database_name` (String) The identifier for the source database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. -- `envelope` (Block List, Max: 1) How Materialize should interpret records (e.g. append-only, upsert).. (see [below for nested schema](#nestedblock--envelope)) +- `envelope` (Block List, Max: 1, Deprecated) How Materialize should interpret records (e.g. append-only, upsert). Deprecated: Use the new materialize_source_table_kafka resource instead. (see [below for nested schema](#nestedblock--envelope)) - `expose_progress` (Block List, Max: 1) The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`. (see [below for nested schema](#nestedblock--expose_progress)) - `format` (Block List, Max: 1) How to decode raw bytes from different formats into data structures Materialize can understand at runtime. (see [below for nested schema](#nestedblock--format)) -- `include_headers` (Boolean) Include message headers. -- `include_headers_alias` (String) Provide an alias for the headers column. -- `include_key` (Boolean) Include a column containing the Kafka message key. -- `include_key_alias` (String) Provide an alias for the key column. -- `include_offset` (Boolean) Include an offset column containing the Kafka message offset. -- `include_offset_alias` (String) Provide an alias for the offset column. -- `include_partition` (Boolean) Include a partition column containing the Kafka message partition -- `include_partition_alias` (String) Provide an alias for the partition column. -- `include_timestamp` (Boolean) Include a timestamp column containing the Kafka message timestamp. -- `include_timestamp_alias` (String) Provide an alias for the timestamp column. +- `include_headers` (Boolean, Deprecated) Include message headers. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `include_headers_alias` (String, Deprecated) Provide an alias for the headers column. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `include_key` (Boolean, Deprecated) Include a column containing the Kafka message key. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `include_key_alias` (String, Deprecated) Provide an alias for the key column. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `include_offset` (Boolean, Deprecated) Include an offset column containing the Kafka message offset. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `include_offset_alias` (String, Deprecated) Provide an alias for the offset column. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `include_partition` (Boolean, Deprecated) Include a partition column containing the Kafka message partition. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `include_partition_alias` (String, Deprecated) Provide an alias for the partition column. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `include_timestamp` (Boolean, Deprecated) Include a timestamp column containing the Kafka message timestamp. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `include_timestamp_alias` (String, Deprecated) Provide an alias for the timestamp column. Deprecated: Use the new materialize_source_table_kafka resource instead. - `key_format` (Block List, Max: 1) Set the key format explicitly. (see [below for nested schema](#nestedblock--key_format)) - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `start_offset` (List of Number) Read partitions from the specified offset. -- `start_timestamp` (Number) Use the specified value to set `START OFFSET` based on the Kafka timestamp. +- `start_offset` (List of Number, Deprecated) Read partitions from the specified offset. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `start_timestamp` (Number, Deprecated) Use the specified value to set `START OFFSET` based on the Kafka timestamp. Deprecated: Use the new materialize_source_table_kafka resource instead. - `value_format` (Block List, Max: 1) Set the value format explicitly. (see [below for nested schema](#nestedblock--value_format)) ### Read-Only diff --git a/docs/resources/source_load_generator.md b/docs/resources/source_load_generator.md index 4dddf435..cb3f0bda 100644 --- a/docs/resources/source_load_generator.md +++ b/docs/resources/source_load_generator.md @@ -40,7 +40,7 @@ resource "materialize_source_load_generator" "example_source_load_generator" { ### Optional -- `all_tables` (Boolean) Whether to include all tables in the source. Compatible with `auction_options`, `marketing_options`, and `tpch_options`. If not specified, use the `materialize_source_table` resource to specify tables to include. +- `all_tables` (Boolean) Whether to include all tables in the source. Compatible with `auction_options`, `marketing_options`, and `tpch_options`. If not specified, use the `materialize_source_table_load_generator` resource to specify tables to include. - `auction_options` (Block List, Max: 1) Auction Options. (see [below for nested schema](#nestedblock--auction_options)) - `cluster_name` (String) The cluster to maintain this source. - `comment` (String) Comment on an object in the database. diff --git a/docs/resources/source_mysql.md b/docs/resources/source_mysql.md index a7829e0d..01773461 100644 --- a/docs/resources/source_mysql.md +++ b/docs/resources/source_mysql.md @@ -53,12 +53,12 @@ resource "materialize_source_mysql" "test" { - `comment` (String) Comment on an object in the database. - `database_name` (String) The identifier for the source database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `expose_progress` (Block List, Max: 1) The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`. (see [below for nested schema](#nestedblock--expose_progress)) -- `ignore_columns` (List of String, Deprecated) Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead. +- `ignore_columns` (List of String, Deprecated) Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_mysql resource instead. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `table` (Block Set, Deprecated) Specify the tables to be included in the source. Deprecated: Use the new materialize_source_table resource instead. (see [below for nested schema](#nestedblock--table)) -- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead. +- `table` (Block Set, Deprecated) Specify the tables to be included in the source. Deprecated: Use the new materialize_source_table_mysql resource instead. (see [below for nested schema](#nestedblock--table)) +- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_mysql resource instead. ### Read-Only diff --git a/docs/resources/source_postgres.md b/docs/resources/source_postgres.md index c98e17d8..829ca9cf 100644 --- a/docs/resources/source_postgres.md +++ b/docs/resources/source_postgres.md @@ -62,8 +62,8 @@ resource "materialize_source_postgres" "example_source_postgres" { - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `table` (Block Set, Deprecated) Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new materialize_source_table resource instead. (see [below for nested schema](#nestedblock--table)) -- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead. +- `table` (Block Set, Deprecated) Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new materialize_source_table_postgres resource instead. (see [below for nested schema](#nestedblock--table)) +- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_postgres resource instead. ### Read-Only diff --git a/docs/resources/source_table_kafka.md b/docs/resources/source_table_kafka.md new file mode 100644 index 00000000..130e144e --- /dev/null +++ b/docs/resources/source_table_kafka.md @@ -0,0 +1,325 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "materialize_source_table_kafka Resource - terraform-provider-materialize" +subcategory: "" +description: |- + A Kafka source describes a Kafka cluster you want Materialize to read data from. +--- + +# materialize_source_table_kafka (Resource) + +A Kafka source describes a Kafka cluster you want Materialize to read data from. + + + + +## Schema + +### Required + +- `name` (String) The identifier for the source. +- `source` (Block List, Min: 1, Max: 1) The source this table is created from. (see [below for nested schema](#nestedblock--source)) +- `upstream_name` (String) The name of the table in the upstream database. + +### Optional + +- `comment` (String) **Public Preview** Comment on an object in the database. +- `database_name` (String) The identifier for the source database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `envelope` (Block List, Max: 1) How Materialize should interpret records (e.g. append-only, upsert).. (see [below for nested schema](#nestedblock--envelope)) +- `expose_progress` (Block List, Max: 1) The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`. (see [below for nested schema](#nestedblock--expose_progress)) +- `format` (Block List, Max: 1) How to decode raw bytes from different formats into data structures Materialize can understand at runtime. (see [below for nested schema](#nestedblock--format)) +- `include_headers` (Boolean) Include message headers. +- `include_headers_alias` (String) Provide an alias for the headers column. +- `include_key` (Boolean) Include a column containing the Kafka message key. +- `include_key_alias` (String) Provide an alias for the key column. +- `include_offset` (Boolean) Include an offset column containing the Kafka message offset. +- `include_offset_alias` (String) Provide an alias for the offset column. +- `include_partition` (Boolean) Include a partition column containing the Kafka message partition +- `include_partition_alias` (String) Provide an alias for the partition column. +- `include_timestamp` (Boolean) Include a timestamp column containing the Kafka message timestamp. +- `include_timestamp_alias` (String) Provide an alias for the timestamp column. +- `key_format` (Block List, Max: 1) Set the key format explicitly. (see [below for nested schema](#nestedblock--key_format)) +- `ownership_role` (String) The owernship role of the object. +- `region` (String) The region to use for the resource connection. If not set, the default region is used. +- `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. +- `start_offset` (List of Number) Read partitions from the specified offset. +- `start_timestamp` (Number) Use the specified value to set `START OFFSET` based on the Kafka timestamp. +- `upstream_schema_name` (String) The schema of the table in the upstream database. +- `value_format` (Block List, Max: 1) Set the value format explicitly. (see [below for nested schema](#nestedblock--value_format)) + +### Read-Only + +- `id` (String) The ID of this resource. +- `qualified_sql_name` (String) The fully qualified name of the source. + + +### Nested Schema for `source` + +Required: + +- `name` (String) The source name. + +Optional: + +- `database_name` (String) The source database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The source schema name. Defaults to `public`. + + + +### Nested Schema for `envelope` + +Optional: + +- `debezium` (Boolean) Use the Debezium envelope, which uses a diff envelope to handle CRUD operations. +- `none` (Boolean) Use an append-only envelope. This means that records will only be appended and cannot be updated or deleted. +- `upsert` (Boolean) Use the upsert envelope, which uses message keys to handle CRUD operations. +- `upsert_options` (Block List, Max: 1) Options for the upsert envelope. (see [below for nested schema](#nestedblock--envelope--upsert_options)) + + +### Nested Schema for `envelope.upsert_options` + +Optional: + +- `value_decoding_errors` (Block List, Max: 1) Specify how to handle value decoding errors in the upsert envelope. (see [below for nested schema](#nestedblock--envelope--upsert_options--value_decoding_errors)) + + +### Nested Schema for `envelope.upsert_options.value_decoding_errors` + +Optional: + +- `inline` (Block List, Max: 1) Configuration for inline value decoding errors. (see [below for nested schema](#nestedblock--envelope--upsert_options--value_decoding_errors--inline)) + + +### Nested Schema for `envelope.upsert_options.value_decoding_errors.inline` + +Optional: + +- `alias` (String) Specify an alias for the value decoding errors column, to use an alternative name for the error column. If not specified, the column name will be `error`. +- `enabled` (Boolean) Enable inline value decoding errors. + + + + + + +### Nested Schema for `expose_progress` + +Required: + +- `name` (String) The expose_progress name. + +Optional: + +- `database_name` (String) The expose_progress database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The expose_progress schema name. Defaults to `public`. + + + +### Nested Schema for `format` + +Optional: + +- `avro` (Block List, Max: 1) Avro format. (see [below for nested schema](#nestedblock--format--avro)) +- `bytes` (Boolean) BYTES format. +- `csv` (Block List, Max: 2) CSV format. (see [below for nested schema](#nestedblock--format--csv)) +- `json` (Boolean) JSON format. +- `protobuf` (Block List, Max: 1) Protobuf format. (see [below for nested schema](#nestedblock--format--protobuf)) +- `text` (Boolean) Text format. + + +### Nested Schema for `format.avro` + +Required: + +- `schema_registry_connection` (Block List, Min: 1, Max: 1) The name of a schema registry connection. (see [below for nested schema](#nestedblock--format--avro--schema_registry_connection)) + +Optional: + +- `key_strategy` (String) How Materialize will define the Avro schema reader key strategy. +- `value_strategy` (String) How Materialize will define the Avro schema reader value strategy. + + +### Nested Schema for `format.avro.schema_registry_connection` + +Required: + +- `name` (String) The schema_registry_connection name. + +Optional: + +- `database_name` (String) The schema_registry_connection database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The schema_registry_connection schema name. Defaults to `public`. + + + + +### Nested Schema for `format.csv` + +Optional: + +- `column` (Number) The columns to use for the source. +- `delimited_by` (String) The delimiter to use for the source. +- `header` (List of String) The number of columns and the name of each column using the header row. + + + +### Nested Schema for `format.protobuf` + +Required: + +- `message` (String) The name of the Protobuf message to use for the source. +- `schema_registry_connection` (Block List, Min: 1, Max: 1) The name of a schema registry connection. (see [below for nested schema](#nestedblock--format--protobuf--schema_registry_connection)) + + +### Nested Schema for `format.protobuf.schema_registry_connection` + +Required: + +- `name` (String) The schema_registry_connection name. + +Optional: + +- `database_name` (String) The schema_registry_connection database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The schema_registry_connection schema name. Defaults to `public`. + + + + + +### Nested Schema for `key_format` + +Optional: + +- `avro` (Block List, Max: 1) Avro format. (see [below for nested schema](#nestedblock--key_format--avro)) +- `bytes` (Boolean) BYTES format. +- `csv` (Block List, Max: 2) CSV format. (see [below for nested schema](#nestedblock--key_format--csv)) +- `json` (Boolean) JSON format. +- `protobuf` (Block List, Max: 1) Protobuf format. (see [below for nested schema](#nestedblock--key_format--protobuf)) +- `text` (Boolean) Text format. + + +### Nested Schema for `key_format.avro` + +Required: + +- `schema_registry_connection` (Block List, Min: 1, Max: 1) The name of a schema registry connection. (see [below for nested schema](#nestedblock--key_format--avro--schema_registry_connection)) + +Optional: + +- `key_strategy` (String) How Materialize will define the Avro schema reader key strategy. +- `value_strategy` (String) How Materialize will define the Avro schema reader value strategy. + + +### Nested Schema for `key_format.avro.schema_registry_connection` + +Required: + +- `name` (String) The schema_registry_connection name. + +Optional: + +- `database_name` (String) The schema_registry_connection database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The schema_registry_connection schema name. Defaults to `public`. + + + + +### Nested Schema for `key_format.csv` + +Optional: + +- `column` (Number) The columns to use for the source. +- `delimited_by` (String) The delimiter to use for the source. +- `header` (List of String) The number of columns and the name of each column using the header row. + + + +### Nested Schema for `key_format.protobuf` + +Required: + +- `message` (String) The name of the Protobuf message to use for the source. +- `schema_registry_connection` (Block List, Min: 1, Max: 1) The name of a schema registry connection. (see [below for nested schema](#nestedblock--key_format--protobuf--schema_registry_connection)) + + +### Nested Schema for `key_format.protobuf.schema_registry_connection` + +Required: + +- `name` (String) The schema_registry_connection name. + +Optional: + +- `database_name` (String) The schema_registry_connection database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The schema_registry_connection schema name. Defaults to `public`. + + + + + +### Nested Schema for `value_format` + +Optional: + +- `avro` (Block List, Max: 1) Avro format. (see [below for nested schema](#nestedblock--value_format--avro)) +- `bytes` (Boolean) BYTES format. +- `csv` (Block List, Max: 2) CSV format. (see [below for nested schema](#nestedblock--value_format--csv)) +- `json` (Boolean) JSON format. +- `protobuf` (Block List, Max: 1) Protobuf format. (see [below for nested schema](#nestedblock--value_format--protobuf)) +- `text` (Boolean) Text format. + + +### Nested Schema for `value_format.avro` + +Required: + +- `schema_registry_connection` (Block List, Min: 1, Max: 1) The name of a schema registry connection. (see [below for nested schema](#nestedblock--value_format--avro--schema_registry_connection)) + +Optional: + +- `key_strategy` (String) How Materialize will define the Avro schema reader key strategy. +- `value_strategy` (String) How Materialize will define the Avro schema reader value strategy. + + +### Nested Schema for `value_format.avro.schema_registry_connection` + +Required: + +- `name` (String) The schema_registry_connection name. + +Optional: + +- `database_name` (String) The schema_registry_connection database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The schema_registry_connection schema name. Defaults to `public`. + + + + +### Nested Schema for `value_format.csv` + +Optional: + +- `column` (Number) The columns to use for the source. +- `delimited_by` (String) The delimiter to use for the source. +- `header` (List of String) The number of columns and the name of each column using the header row. + + + +### Nested Schema for `value_format.protobuf` + +Required: + +- `message` (String) The name of the Protobuf message to use for the source. +- `schema_registry_connection` (Block List, Min: 1, Max: 1) The name of a schema registry connection. (see [below for nested schema](#nestedblock--value_format--protobuf--schema_registry_connection)) + + +### Nested Schema for `value_format.protobuf.schema_registry_connection` + +Required: + +- `name` (String) The schema_registry_connection name. + +Optional: + +- `database_name` (String) The schema_registry_connection database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The schema_registry_connection schema name. Defaults to `public`. diff --git a/pkg/materialize/format_specs.go b/pkg/materialize/format_specs.go index ad3178fd..d9c51d48 100644 --- a/pkg/materialize/format_specs.go +++ b/pkg/materialize/format_specs.go @@ -73,7 +73,7 @@ func GetFormatSpecStruc(v interface{}) SourceFormatSpecStruct { } if protobuf, ok := u["protobuf"]; ok && protobuf != nil && len(protobuf.([]interface{})) > 0 { if csr, ok := protobuf.([]interface{})[0].(map[string]interface{})["schema_registry_connection"]; ok { - message := protobuf.([]interface{})[0].(map[string]interface{})["message_name"].(string) + message := protobuf.([]interface{})[0].(map[string]interface{})["message"].(string) format.Protobuf = &ProtobufFormatSpec{ SchemaRegistryConnection: GetIdentifierSchemaStruct(csr), MessageName: message, diff --git a/pkg/materialize/source_table_kafka.go b/pkg/materialize/source_table_kafka.go new file mode 100644 index 00000000..dddc3ffb --- /dev/null +++ b/pkg/materialize/source_table_kafka.go @@ -0,0 +1,332 @@ +package materialize + +import ( + "fmt" + "strings" + + "github.com/jmoiron/sqlx" +) + +type SourceTableKafkaBuilder struct { + *SourceTableBuilder + includeKey bool + includeHeaders bool + includePartition bool + includeOffset bool + includeTimestamp bool + keyAlias string + headersAlias string + partitionAlias string + offsetAlias string + timestampAlias string + format SourceFormatSpecStruct + keyFormat SourceFormatSpecStruct + valueFormat SourceFormatSpecStruct + envelope KafkaSourceEnvelopeStruct + startOffset []int + startTimestamp int + exposeProgress IdentifierSchemaStruct +} + +func NewSourceTableKafkaBuilder(conn *sqlx.DB, obj MaterializeObject) *SourceTableKafkaBuilder { + return &SourceTableKafkaBuilder{ + SourceTableBuilder: NewSourceTableBuilder(conn, obj), + } +} + +func (b *SourceTableKafkaBuilder) IncludeKey() *SourceTableKafkaBuilder { + b.includeKey = true + return b +} + +func (b *SourceTableKafkaBuilder) IncludeHeaders() *SourceTableKafkaBuilder { + b.includeHeaders = true + return b +} + +func (b *SourceTableKafkaBuilder) IncludePartition() *SourceTableKafkaBuilder { + b.includePartition = true + return b +} + +func (b *SourceTableKafkaBuilder) IncludeOffset() *SourceTableKafkaBuilder { + b.includeOffset = true + return b +} + +func (b *SourceTableKafkaBuilder) IncludeTimestamp() *SourceTableKafkaBuilder { + b.includeTimestamp = true + return b +} + +func (b *SourceTableKafkaBuilder) IncludeKeyAlias(alias string) *SourceTableKafkaBuilder { + b.includeKey = true + b.keyAlias = alias + return b +} + +func (b *SourceTableKafkaBuilder) IncludeHeadersAlias(alias string) *SourceTableKafkaBuilder { + b.includeHeaders = true + b.headersAlias = alias + return b +} + +func (b *SourceTableKafkaBuilder) IncludePartitionAlias(alias string) *SourceTableKafkaBuilder { + b.includePartition = true + b.partitionAlias = alias + return b +} + +func (b *SourceTableKafkaBuilder) IncludeOffsetAlias(alias string) *SourceTableKafkaBuilder { + b.includeOffset = true + b.offsetAlias = alias + return b +} + +func (b *SourceTableKafkaBuilder) IncludeTimestampAlias(alias string) *SourceTableKafkaBuilder { + b.includeTimestamp = true + b.timestampAlias = alias + return b +} + +func (b *SourceTableKafkaBuilder) Format(f SourceFormatSpecStruct) *SourceTableKafkaBuilder { + b.format = f + return b +} + +func (b *SourceTableKafkaBuilder) Envelope(e KafkaSourceEnvelopeStruct) *SourceTableKafkaBuilder { + b.envelope = e + return b +} + +func (b *SourceTableKafkaBuilder) KeyFormat(k SourceFormatSpecStruct) *SourceTableKafkaBuilder { + b.keyFormat = k + return b +} + +func (b *SourceTableKafkaBuilder) ValueFormat(v SourceFormatSpecStruct) *SourceTableKafkaBuilder { + b.valueFormat = v + return b +} + +func (b *SourceTableKafkaBuilder) StartOffset(s []int) *SourceTableKafkaBuilder { + b.startOffset = s + return b +} + +func (b *SourceTableKafkaBuilder) StartTimestamp(s int) *SourceTableKafkaBuilder { + b.startTimestamp = s + return b +} + +func (b *SourceTableKafkaBuilder) ExposeProgress(e IdentifierSchemaStruct) *SourceTableKafkaBuilder { + b.exposeProgress = e + return b +} + +func (b *SourceTableKafkaBuilder) Create() error { + return b.BaseCreate("kafka", func() string { + q := strings.Builder{} + var options []string + + // Format + if b.format.Avro != nil { + if b.format.Avro.SchemaRegistryConnection.Name != "" { + options = append(options, fmt.Sprintf(`FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QualifiedName(b.format.Avro.SchemaRegistryConnection.DatabaseName, b.format.Avro.SchemaRegistryConnection.SchemaName, b.format.Avro.SchemaRegistryConnection.Name))) + } + if b.format.Avro.KeyStrategy != "" { + options = append(options, fmt.Sprintf(`KEY STRATEGY %s`, b.format.Avro.KeyStrategy)) + } + if b.format.Avro.ValueStrategy != "" { + options = append(options, fmt.Sprintf(`VALUE STRATEGY %s`, b.format.Avro.ValueStrategy)) + } + } + + if b.format.Protobuf != nil { + if b.format.Protobuf.SchemaRegistryConnection.Name != "" && b.format.Protobuf.MessageName != "" { + options = append(options, fmt.Sprintf(`FORMAT PROTOBUF MESSAGE '%s' USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, b.format.Protobuf.MessageName, QualifiedName(b.format.Protobuf.SchemaRegistryConnection.DatabaseName, b.format.Protobuf.SchemaRegistryConnection.SchemaName, b.format.Protobuf.SchemaRegistryConnection.Name))) + } else if b.format.Protobuf.SchemaRegistryConnection.Name != "" { + options = append(options, fmt.Sprintf(`FORMAT PROTOBUF USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QualifiedName(b.format.Protobuf.SchemaRegistryConnection.DatabaseName, b.format.Protobuf.SchemaRegistryConnection.SchemaName, b.format.Protobuf.SchemaRegistryConnection.Name))) + } + } + + if b.format.Csv != nil { + if b.format.Csv.Columns > 0 { + options = append(options, fmt.Sprintf(`FORMAT CSV WITH %d COLUMNS`, b.format.Csv.Columns)) + } + if b.format.Csv.Header != nil { + options = append(options, fmt.Sprintf(`FORMAT CSV WITH HEADER ( %s )`, strings.Join(b.format.Csv.Header, ", "))) + } + if b.format.Csv.DelimitedBy != "" { + options = append(options, fmt.Sprintf(`DELIMITER '%s'`, b.format.Csv.DelimitedBy)) + } + } + + if b.format.Bytes { + options = append(options, `FORMAT BYTES`) + } + if b.format.Text { + options = append(options, `FORMAT TEXT`) + } + if b.format.Json { + options = append(options, `FORMAT JSON`) + } + + // Key Format + if b.keyFormat.Avro != nil { + if b.keyFormat.Avro.SchemaRegistryConnection.Name != "" { + options = append(options, fmt.Sprintf(`KEY FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QualifiedName(b.keyFormat.Avro.SchemaRegistryConnection.DatabaseName, b.keyFormat.Avro.SchemaRegistryConnection.SchemaName, b.keyFormat.Avro.SchemaRegistryConnection.Name))) + } + if b.keyFormat.Avro.KeyStrategy != "" { + options = append(options, fmt.Sprintf(`KEY STRATEGY %s`, b.keyFormat.Avro.KeyStrategy)) + } + if b.keyFormat.Avro.ValueStrategy != "" { + options = append(options, fmt.Sprintf(`VALUE STRATEGY %s`, b.keyFormat.Avro.ValueStrategy)) + } + } + + if b.keyFormat.Protobuf != nil { + if b.keyFormat.Protobuf.SchemaRegistryConnection.Name != "" && b.keyFormat.Protobuf.MessageName != "" { + options = append(options, fmt.Sprintf(`KEY FORMAT PROTOBUF MESSAGE '%s' USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, b.keyFormat.Protobuf.MessageName, QualifiedName(b.keyFormat.Protobuf.SchemaRegistryConnection.DatabaseName, b.keyFormat.Protobuf.SchemaRegistryConnection.SchemaName, b.keyFormat.Protobuf.SchemaRegistryConnection.Name))) + } else if b.keyFormat.Protobuf.SchemaRegistryConnection.Name != "" { + options = append(options, fmt.Sprintf(`KEY FORMAT PROTOBUF USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QualifiedName(b.keyFormat.Protobuf.SchemaRegistryConnection.DatabaseName, b.keyFormat.Protobuf.SchemaRegistryConnection.SchemaName, b.keyFormat.Protobuf.SchemaRegistryConnection.Name))) + } + } + + if b.keyFormat.Csv != nil { + if b.keyFormat.Csv.Columns > 0 { + options = append(options, fmt.Sprintf(`KEY FORMAT CSV WITH %d COLUMNS`, b.keyFormat.Csv.Columns)) + } + if b.keyFormat.Csv.Header != nil { + options = append(options, fmt.Sprintf(`KEY FORMAT CSV WITH HEADER ( %s )`, strings.Join(b.keyFormat.Csv.Header, ", "))) + } + if b.keyFormat.Csv.DelimitedBy != "" { + options = append(options, fmt.Sprintf(`KEY DELIMITER '%s'`, b.keyFormat.Csv.DelimitedBy)) + } + } + + if b.keyFormat.Bytes { + options = append(options, `KEY FORMAT BYTES`) + } + if b.keyFormat.Text { + options = append(options, `KEY FORMAT TEXT`) + } + if b.keyFormat.Json { + options = append(options, `KEY FORMAT JSON`) + } + + // Value Format + if b.valueFormat.Avro != nil { + if b.valueFormat.Avro.SchemaRegistryConnection.Name != "" { + options = append(options, fmt.Sprintf(`VALUE FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QualifiedName(b.valueFormat.Avro.SchemaRegistryConnection.DatabaseName, b.valueFormat.Avro.SchemaRegistryConnection.SchemaName, b.valueFormat.Avro.SchemaRegistryConnection.Name))) + } + if b.valueFormat.Avro.KeyStrategy != "" { + options = append(options, fmt.Sprintf(`VALUE STRATEGY %s`, b.valueFormat.Avro.KeyStrategy)) + } + if b.valueFormat.Avro.ValueStrategy != "" { + options = append(options, fmt.Sprintf(`VALUE STRATEGY %s`, b.valueFormat.Avro.ValueStrategy)) + } + } + + if b.valueFormat.Protobuf != nil { + if b.valueFormat.Protobuf.SchemaRegistryConnection.Name != "" && b.valueFormat.Protobuf.MessageName != "" { + options = append(options, fmt.Sprintf(`VALUE FORMAT PROTOBUF MESSAGE '%s' USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, b.valueFormat.Protobuf.MessageName, QualifiedName(b.valueFormat.Protobuf.SchemaRegistryConnection.DatabaseName, b.valueFormat.Protobuf.SchemaRegistryConnection.SchemaName, b.valueFormat.Protobuf.SchemaRegistryConnection.Name))) + } else if b.valueFormat.Protobuf.SchemaRegistryConnection.Name != "" { + options = append(options, fmt.Sprintf(`VALUE FORMAT PROTOBUF USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QualifiedName(b.valueFormat.Protobuf.SchemaRegistryConnection.DatabaseName, b.valueFormat.Protobuf.SchemaRegistryConnection.SchemaName, b.valueFormat.Protobuf.SchemaRegistryConnection.Name))) + } + } + + if b.valueFormat.Csv != nil { + if b.valueFormat.Csv.Columns > 0 { + options = append(options, fmt.Sprintf(`VALUE FORMAT CSV WITH %d COLUMNS`, b.valueFormat.Csv.Columns)) + } + if b.valueFormat.Csv.Header != nil { + options = append(options, fmt.Sprintf(`VALUE FORMAT CSV WITH HEADER ( %s )`, strings.Join(b.valueFormat.Csv.Header, ", "))) + } + if b.valueFormat.Csv.DelimitedBy != "" { + options = append(options, fmt.Sprintf(`VALUE DELIMITER '%s'`, b.valueFormat.Csv.DelimitedBy)) + } + } + + if b.valueFormat.Bytes { + options = append(options, `VALUE FORMAT BYTES`) + } + if b.valueFormat.Text { + options = append(options, `VALUE FORMAT TEXT`) + } + if b.valueFormat.Json { + options = append(options, `VALUE FORMAT JSON`) + } + + // Metadata + var metadataOptions []string + if b.includeKey { + if b.keyAlias != "" { + metadataOptions = append(metadataOptions, fmt.Sprintf("KEY AS %s", b.keyAlias)) + } else { + metadataOptions = append(metadataOptions, "KEY") + } + } + if b.includeHeaders { + if b.headersAlias != "" { + metadataOptions = append(metadataOptions, fmt.Sprintf("HEADERS AS %s", b.headersAlias)) + } else { + metadataOptions = append(metadataOptions, "HEADERS") + } + } + if b.includePartition { + if b.partitionAlias != "" { + metadataOptions = append(metadataOptions, fmt.Sprintf("PARTITION AS %s", b.partitionAlias)) + } else { + metadataOptions = append(metadataOptions, "PARTITION") + } + } + if b.includeOffset { + if b.offsetAlias != "" { + metadataOptions = append(metadataOptions, fmt.Sprintf("OFFSET AS %s", b.offsetAlias)) + } else { + metadataOptions = append(metadataOptions, "OFFSET") + } + } + if b.includeTimestamp { + if b.timestampAlias != "" { + metadataOptions = append(metadataOptions, fmt.Sprintf("TIMESTAMP AS %s", b.timestampAlias)) + } else { + metadataOptions = append(metadataOptions, "TIMESTAMP") + } + } + if len(metadataOptions) > 0 { + options = append(options, fmt.Sprintf(`INCLUDE %s`, strings.Join(metadataOptions, ", "))) + } + + // Envelope + if b.envelope.Debezium { + options = append(options, `ENVELOPE DEBEZIUM`) + } + if b.envelope.Upsert { + upsertOption := "ENVELOPE UPSERT" + if b.envelope.UpsertOptions != nil { + inlineOptions := b.envelope.UpsertOptions.ValueDecodingErrors.Inline + if inlineOptions.Enabled { + upsertOption += " (VALUE DECODING ERRORS = (INLINE" + if inlineOptions.Alias != "" { + upsertOption += fmt.Sprintf(" AS %s", inlineOptions.Alias) + } + upsertOption += "))" + } + } + options = append(options, upsertOption) + } + if b.envelope.None { + options = append(options, `ENVELOPE NONE`) + } + + // Expose Progress + if b.exposeProgress.Name != "" { + options = append(options, fmt.Sprintf(`EXPOSE PROGRESS AS %s`, b.exposeProgress.QualifiedName())) + } + + q.WriteString(strings.Join(options, " ")) + return " " + q.String() + }) +} diff --git a/pkg/materialize/source_table_kafka_test.go b/pkg/materialize/source_table_kafka_test.go new file mode 100644 index 00000000..cbc85468 --- /dev/null +++ b/pkg/materialize/source_table_kafka_test.go @@ -0,0 +1,123 @@ +package materialize + +import ( + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/jmoiron/sqlx" +) + +func TestResourceSourceTableKafkaCreate(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."source" + FROM SOURCE "database"."schema"."kafka_source" + \(REFERENCE "upstream_table"\) + FORMAT JSON + INCLUDE KEY AS message_key, HEADERS AS message_headers, PARTITION AS message_partition + ENVELOPE UPSERT + EXPOSE PROGRESS AS "database"."schema"."progress";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + o := MaterializeObject{Name: "source", SchemaName: "schema", DatabaseName: "database"} + b := NewSourceTableKafkaBuilder(db, o) + b.Source(IdentifierSchemaStruct{Name: "kafka_source", DatabaseName: "database", SchemaName: "schema"}) + b.UpstreamName("upstream_table") + b.Format(SourceFormatSpecStruct{Json: true}) + b.IncludeKey() + b.IncludeKeyAlias("message_key") + b.IncludeHeaders() + b.IncludeHeadersAlias("message_headers") + b.IncludePartition() + b.IncludePartitionAlias("message_partition") + b.Envelope(KafkaSourceEnvelopeStruct{Upsert: true}) + b.ExposeProgress(IdentifierSchemaStruct{Name: "progress", DatabaseName: "database", SchemaName: "schema"}) + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableKafkaCreateWithAvroFormat(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."source" + FROM SOURCE "database"."schema"."kafka_source" + \(REFERENCE "upstream_table"\) + FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION "database"."schema"."schema_registry" + KEY STRATEGY EXTRACT + VALUE STRATEGY EXTRACT + INCLUDE TIMESTAMP + ENVELOPE DEBEZIUM;`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + o := MaterializeObject{Name: "source", SchemaName: "schema", DatabaseName: "database"} + b := NewSourceTableKafkaBuilder(db, o) + b.Source(IdentifierSchemaStruct{Name: "kafka_source", DatabaseName: "database", SchemaName: "schema"}) + b.UpstreamName("upstream_table") + b.Format(SourceFormatSpecStruct{ + Avro: &AvroFormatSpec{ + SchemaRegistryConnection: IdentifierSchemaStruct{Name: "schema_registry", DatabaseName: "database", SchemaName: "schema"}, + KeyStrategy: "EXTRACT", + ValueStrategy: "EXTRACT", + }, + }) + b.IncludeTimestamp() + b.Envelope(KafkaSourceEnvelopeStruct{Debezium: true}) + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableKafkaCreateWithUpsertOptions(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."source" + FROM SOURCE "database"."schema"."kafka_source" + \(REFERENCE "upstream_table"\) + FORMAT JSON + INCLUDE KEY, HEADERS, PARTITION, OFFSET, TIMESTAMP + ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS my_error_col\)\) + EXPOSE PROGRESS AS "database"."schema"."progress";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + o := MaterializeObject{Name: "source", SchemaName: "schema", DatabaseName: "database"} + b := NewSourceTableKafkaBuilder(db, o) + b.Source(IdentifierSchemaStruct{Name: "kafka_source", DatabaseName: "database", SchemaName: "schema"}) + b.UpstreamName("upstream_table") + b.Format(SourceFormatSpecStruct{Json: true}) + b.IncludeKey() + b.IncludeHeaders() + b.IncludePartition() + b.IncludeOffset() + b.IncludeTimestamp() + b.Envelope(KafkaSourceEnvelopeStruct{ + Upsert: true, + UpsertOptions: &UpsertOptionsStruct{ + ValueDecodingErrors: struct { + Inline struct { + Enabled bool + Alias string + } + }{ + Inline: struct { + Enabled bool + Alias string + }{ + Enabled: true, + Alias: "my_error_col", + }, + }, + }, + }) + b.ExposeProgress(IdentifierSchemaStruct{Name: "progress", DatabaseName: "database", SchemaName: "schema"}) + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} diff --git a/pkg/materialize/source_table_load_generator.go b/pkg/materialize/source_table_load_generator.go index bfb19bfd..c3f81906 100644 --- a/pkg/materialize/source_table_load_generator.go +++ b/pkg/materialize/source_table_load_generator.go @@ -4,7 +4,6 @@ import ( "github.com/jmoiron/sqlx" ) -// SourceTableLoadGenBuilder for Load Generator sources type SourceTableLoadGenBuilder struct { *SourceTableBuilder } diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 32079903..09132ad9 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -133,6 +133,7 @@ func Provider(version string) *schema.Provider { "materialize_source_grant": resources.GrantSource(), "materialize_system_parameter": resources.SystemParameter(), "materialize_table": resources.Table(), + "materialize_source_table_kafka": resources.SourceTableKafka(), "materialize_source_table_load_generator": resources.SourceTableLoadGen(), "materialize_source_table_mysql": resources.SourceTableMySQL(), "materialize_source_table_postgres": resources.SourceTablePostgres(), diff --git a/pkg/resources/resource_source_kafka.go b/pkg/resources/resource_source_kafka.go index 57ff25b7..d9d650e5 100644 --- a/pkg/resources/resource_source_kafka.go +++ b/pkg/resources/resource_source_kafka.go @@ -32,62 +32,72 @@ var sourceKafkaSchema = map[string]*schema.Schema{ ForceNew: true, }, "include_key": { - Description: "Include a column containing the Kafka message key.", + Description: "Include a column containing the Kafka message key. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, }, "include_key_alias": { - Description: "Provide an alias for the key column.", + Description: "Provide an alias for the key column. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeString, Optional: true, ForceNew: true, }, "include_headers": { - Description: "Include message headers.", + Description: "Include message headers. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, Default: false, }, "include_headers_alias": { - Description: "Provide an alias for the headers column.", + Description: "Provide an alias for the headers column. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeString, Optional: true, ForceNew: true, }, "include_partition": { - Description: "Include a partition column containing the Kafka message partition", + Description: "Include a partition column containing the Kafka message partition. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, }, "include_partition_alias": { - Description: "Provide an alias for the partition column.", + Description: "Provide an alias for the partition column. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeString, Optional: true, ForceNew: true, }, "include_offset": { - Description: "Include an offset column containing the Kafka message offset.", + Description: "Include an offset column containing the Kafka message offset. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, }, "include_offset_alias": { - Description: "Provide an alias for the offset column.", + Description: "Provide an alias for the offset column. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeString, Optional: true, ForceNew: true, }, "include_timestamp": { - Description: "Include a timestamp column containing the Kafka message timestamp.", + Description: "Include a timestamp column containing the Kafka message timestamp. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, }, "include_timestamp_alias": { - Description: "Provide an alias for the timestamp column.", + Description: "Provide an alias for the timestamp column. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeString, Optional: true, ForceNew: true, @@ -96,7 +106,8 @@ var sourceKafkaSchema = map[string]*schema.Schema{ "key_format": FormatSpecSchema("key_format", "Set the key format explicitly.", false), "value_format": FormatSpecSchema("value_format", "Set the value format explicitly.", false), "envelope": { - Description: "How Materialize should interpret records (e.g. append-only, upsert)..", + Description: "How Materialize should interpret records (e.g. append-only, upsert). Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeList, MaxItems: 1, Elem: &schema.Resource{ @@ -171,7 +182,8 @@ var sourceKafkaSchema = map[string]*schema.Schema{ ForceNew: true, }, "start_offset": { - Description: "Read partitions from the specified offset.", + Description: "Read partitions from the specified offset. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeInt}, Optional: true, @@ -179,7 +191,8 @@ var sourceKafkaSchema = map[string]*schema.Schema{ ConflictsWith: []string{"start_timestamp"}, }, "start_timestamp": { - Description: "Use the specified value to set `START OFFSET` based on the Kafka timestamp.", + Description: "Use the specified value to set `START OFFSET` based on the Kafka timestamp. Deprecated: Use the new materialize_source_table_kafka resource instead.", + Deprecated: "Use the new materialize_source_table_kafka resource instead.", Type: schema.TypeInt, Optional: true, ForceNew: true, diff --git a/pkg/resources/resource_source_load_generator.go b/pkg/resources/resource_source_load_generator.go index 0cd4ef8a..7b396f36 100644 --- a/pkg/resources/resource_source_load_generator.go +++ b/pkg/resources/resource_source_load_generator.go @@ -175,7 +175,7 @@ var sourceLoadgenSchema = map[string]*schema.Schema{ ConflictsWith: []string{"counter_options", "auction_options", "marketing_options", "tpch_options"}, }, "all_tables": { - Description: "Whether to include all tables in the source. Compatible with `auction_options`, `marketing_options`, and `tpch_options`. If not specified, use the `materialize_source_table` resource to specify tables to include.", + Description: "Whether to include all tables in the source. Compatible with `auction_options`, `marketing_options`, and `tpch_options`. If not specified, use the `materialize_source_table_load_generator` resource to specify tables to include.", Type: schema.TypeBool, Optional: true, Default: false, diff --git a/pkg/resources/resource_source_mysql.go b/pkg/resources/resource_source_mysql.go index 01dc5dd1..40ef1681 100644 --- a/pkg/resources/resource_source_mysql.go +++ b/pkg/resources/resource_source_mysql.go @@ -27,22 +27,22 @@ var sourceMySQLSchema = map[string]*schema.Schema{ ForceNew: true, }), "ignore_columns": { - Description: "Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead.", - Deprecated: "Use the new materialize_source_table resource instead.", + Description: "Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_mysql resource instead.", + Deprecated: "Use the new materialize_source_table_mysql resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "text_columns": { - Description: "Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead.", - Deprecated: "Use the new materialize_source_table resource instead.", + Description: "Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_mysql resource instead.", + Deprecated: "Use the new materialize_source_table_mysql resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "table": { - Description: "Specify the tables to be included in the source. Deprecated: Use the new materialize_source_table resource instead.", - Deprecated: "Use the new materialize_source_table resource instead.", + Description: "Specify the tables to be included in the source. Deprecated: Use the new materialize_source_table_mysql resource instead.", + Deprecated: "Use the new materialize_source_table_mysql resource instead.", Type: schema.TypeSet, Optional: true, Elem: &schema.Resource{ @@ -81,7 +81,7 @@ var sourceMySQLSchema = map[string]*schema.Schema{ }, "all_tables": { Description: "Include all tables in the source. If `table` is specified, this will be ignored.", - Deprecated: "Use the new materialize_source_table resource instead.", + Deprecated: "Use the new materialize_source_table_mysql resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, diff --git a/pkg/resources/resource_source_postgres.go b/pkg/resources/resource_source_postgres.go index bea0cad5..040eb16a 100644 --- a/pkg/resources/resource_source_postgres.go +++ b/pkg/resources/resource_source_postgres.go @@ -33,15 +33,15 @@ var sourcePostgresSchema = map[string]*schema.Schema{ ForceNew: true, }, "text_columns": { - Description: "Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table resource instead.", - Deprecated: "Use the new materialize_source_table resource instead.", + Description: "Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_postgres resource instead.", + Deprecated: "Use the new materialize_source_table_postgres resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "table": { - Description: "Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new materialize_source_table resource instead.", - Deprecated: "Use the new materialize_source_table resource instead.", + Description: "Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new materialize_source_table_postgres resource instead.", + Deprecated: "Use the new materialize_source_table_postgres resource instead.", Type: schema.TypeSet, Optional: true, Elem: &schema.Resource{ @@ -204,7 +204,7 @@ func sourcePostgresCreate(ctx context.Context, d *schema.ResourceData, meta any) } if v, ok := d.GetOk("table"); ok { - log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_source_table resource instead.") + log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_source_table_postgres resource instead.") tables := v.(*schema.Set).List() t := materialize.GetTableStruct(tables) b.Table(t) @@ -291,7 +291,7 @@ func sourcePostgresUpdate(ctx context.Context, d *schema.ResourceData, meta any) } if d.HasChange("table") { - log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_source_table resource instead.") + log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_source_table_postgres resource instead.") ot, nt := d.GetChange("table") addTables := materialize.DiffTableStructs(nt.(*schema.Set).List(), ot.(*schema.Set).List()) dropTables := materialize.DiffTableStructs(ot.(*schema.Set).List(), nt.(*schema.Set).List()) diff --git a/pkg/resources/resource_source_table_kafka.go b/pkg/resources/resource_source_table_kafka.go new file mode 100644 index 00000000..f60d63f9 --- /dev/null +++ b/pkg/resources/resource_source_table_kafka.go @@ -0,0 +1,349 @@ +package resources + +import ( + "context" + "log" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +var sourceTableKafkaSchema = map[string]*schema.Schema{ + "name": ObjectNameSchema("source", true, false), + "schema_name": SchemaNameSchema("source", false), + "database_name": DatabaseNameSchema("source", false), + "qualified_sql_name": QualifiedNameSchema("source"), + "comment": CommentSchema(false), + "source": IdentifierSchema(IdentifierSchemaParams{ + Elem: "source", + Description: "The source this table is created from.", + Required: true, + ForceNew: true, + }), + "upstream_name": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + Description: "The name of the table in the upstream database.", + }, + "upstream_schema_name": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + Description: "The schema of the table in the upstream database.", + }, + "include_key": { + Description: "Include a column containing the Kafka message key.", + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + }, + "include_key_alias": { + Description: "Provide an alias for the key column.", + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + "include_headers": { + Description: "Include message headers.", + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + Default: false, + }, + "include_headers_alias": { + Description: "Provide an alias for the headers column.", + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + "include_partition": { + Description: "Include a partition column containing the Kafka message partition", + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + }, + "include_partition_alias": { + Description: "Provide an alias for the partition column.", + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + "include_offset": { + Description: "Include an offset column containing the Kafka message offset.", + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + }, + "include_offset_alias": { + Description: "Provide an alias for the offset column.", + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + "include_timestamp": { + Description: "Include a timestamp column containing the Kafka message timestamp.", + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + }, + "include_timestamp_alias": { + Description: "Provide an alias for the timestamp column.", + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + "format": FormatSpecSchema("format", "How to decode raw bytes from different formats into data structures Materialize can understand at runtime.", false), + "key_format": FormatSpecSchema("key_format", "Set the key format explicitly.", false), + "value_format": FormatSpecSchema("value_format", "Set the value format explicitly.", false), + "envelope": { + Description: "How Materialize should interpret records (e.g. append-only, upsert)..", + Type: schema.TypeList, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "upsert": { + Description: "Use the upsert envelope, which uses message keys to handle CRUD operations.", + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + ConflictsWith: []string{"envelope.0.debezium", "envelope.0.none"}, + }, + "debezium": { + Description: "Use the Debezium envelope, which uses a diff envelope to handle CRUD operations.", + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + ConflictsWith: []string{"envelope.0.upsert", "envelope.0.none", "envelope.0.upsert_options"}, + }, + "none": { + Description: "Use an append-only envelope. This means that records will only be appended and cannot be updated or deleted.", + Type: schema.TypeBool, + Optional: true, + ForceNew: true, + ConflictsWith: []string{"envelope.0.upsert", "envelope.0.debezium", "envelope.0.upsert_options"}, + }, + "upsert_options": { + Description: "Options for the upsert envelope.", + Type: schema.TypeList, + MaxItems: 1, + Optional: true, + ForceNew: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "value_decoding_errors": { + Description: "Specify how to handle value decoding errors in the upsert envelope.", + Type: schema.TypeList, + MaxItems: 1, + Optional: true, + ForceNew: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "inline": { + Description: "Configuration for inline value decoding errors.", + Type: schema.TypeList, + MaxItems: 1, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "enabled": { + Description: "Enable inline value decoding errors.", + Type: schema.TypeBool, + Optional: true, + Default: false, + }, + "alias": { + Description: "Specify an alias for the value decoding errors column, to use an alternative name for the error column. If not specified, the column name will be `error`.", + Type: schema.TypeString, + Optional: true, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + Optional: true, + ForceNew: true, + }, + "start_offset": { + Description: "Read partitions from the specified offset.", + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeInt}, + Optional: true, + ForceNew: true, + ConflictsWith: []string{"start_timestamp"}, + }, + "start_timestamp": { + Description: "Use the specified value to set `START OFFSET` based on the Kafka timestamp.", + Type: schema.TypeInt, + Optional: true, + ForceNew: true, + ConflictsWith: []string{"start_offset"}, + }, + "expose_progress": IdentifierSchema(IdentifierSchemaParams{ + Elem: "expose_progress", + Description: "The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`.", + Required: false, + ForceNew: true, + }), + "ownership_role": OwnershipRoleSchema(), + "region": RegionSchema(), +} + +func SourceTableKafka() *schema.Resource { + return &schema.Resource{ + Description: "A Kafka source describes a Kafka cluster you want Materialize to read data from.", + + CreateContext: sourceTableKafkaCreate, + ReadContext: sourceTableRead, + UpdateContext: sourceTableUpdate, + DeleteContext: sourceTableDelete, + + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + + Schema: sourceTableKafkaSchema, + } +} + +func sourceTableKafkaCreate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + sourceName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + o := materialize.MaterializeObject{ObjectType: "SOURCE", Name: sourceName, SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewSourceTableKafkaBuilder(metaDb, o) + + source := materialize.GetIdentifierSchemaStruct(d.Get("source")) + b.Source(source) + + b.UpstreamName(d.Get("upstream_name").(string)) + + if v, ok := d.GetOk("upstream_schema_name"); ok { + b.UpstreamSchemaName(v.(string)) + } + + if v, ok := d.GetOk("include_key"); ok && v.(bool) { + if alias, ok := d.GetOk("include_key_alias"); ok { + b.IncludeKeyAlias(alias.(string)) + } else { + b.IncludeKey() + } + } + + if v, ok := d.GetOk("include_partition"); ok && v.(bool) { + if alias, ok := d.GetOk("include_partition_alias"); ok { + b.IncludePartitionAlias(alias.(string)) + } else { + b.IncludePartition() + } + } + + if v, ok := d.GetOk("include_offset"); ok && v.(bool) { + if alias, ok := d.GetOk("include_offset_alias"); ok { + b.IncludeOffsetAlias(alias.(string)) + } else { + b.IncludeOffset() + } + } + + if v, ok := d.GetOk("include_timestamp"); ok && v.(bool) { + if alias, ok := d.GetOk("include_timestamp_alias"); ok { + b.IncludeTimestampAlias(alias.(string)) + } else { + b.IncludeTimestamp() + } + } + + if v, ok := d.GetOk("include_headers"); ok && v.(bool) { + if alias, ok := d.GetOk("include_headers_alias"); ok { + b.IncludeHeadersAlias(alias.(string)) + } else { + b.IncludeHeaders() + } + } + + if v, ok := d.GetOk("format"); ok { + format := materialize.GetFormatSpecStruc(v) + b.Format(format) + } + + if v, ok := d.GetOk("key_format"); ok { + format := materialize.GetFormatSpecStruc(v) + b.KeyFormat(format) + } + + if v, ok := d.GetOk("value_format"); ok { + format := materialize.GetFormatSpecStruc(v) + b.ValueFormat(format) + } + + if v, ok := d.GetOk("envelope"); ok { + envelope := materialize.GetSourceKafkaEnvelopeStruct(v) + b.Envelope(envelope) + } + + if v, ok := d.GetOk("start_offset"); ok { + so := materialize.GetSliceValueInt(v.([]interface{})) + b.StartOffset(so) + } + + if v, ok := d.GetOk("start_timestamp"); ok { + b.StartTimestamp(v.(int)) + } + + if v, ok := d.GetOk("expose_progress"); ok { + e := materialize.GetIdentifierSchemaStruct(v) + b.ExposeProgress(e) + } + + // create resource + if err := b.Create(); err != nil { + return diag.FromErr(err) + } + + // ownership + if v, ok := d.GetOk("ownership_role"); ok { + ownership := materialize.NewOwnershipBuilder(metaDb, o) + + if err := ownership.Alter(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed ownership, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + // object comment + if v, ok := d.GetOk("comment"); ok { + comment := materialize.NewCommentBuilder(metaDb, o) + + if err := comment.Object(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed comment, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + // set id + i, err := materialize.SourceTableId(metaDb, o) + if err != nil { + return diag.FromErr(err) + } + d.SetId(utils.TransformIdWithRegion(string(region), i)) + + return sourceTableRead(ctx, d, meta) +} diff --git a/pkg/resources/resource_source_table_kafka_test.go b/pkg/resources/resource_source_table_kafka_test.go new file mode 100644 index 00000000..b13e9d7c --- /dev/null +++ b/pkg/resources/resource_source_table_kafka_test.go @@ -0,0 +1,458 @@ +package resources + +import ( + "context" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/stretchr/testify/require" +) + +var inSourceTableKafka = map[string]interface{}{ + "name": "table", + "schema_name": "schema", + "database_name": "database", + "source": []interface{}{ + map[string]interface{}{ + "name": "kafka_source", + "schema_name": "public", + "database_name": "materialize", + }, + }, + "upstream_name": "upstream_table", + "include_key": true, + "include_key_alias": "message_key", + "include_headers": true, + "include_headers_alias": "message_headers", + "include_partition": true, + "include_partition_alias": "message_partition", + "format": []interface{}{ + map[string]interface{}{ + "json": true, + }, + }, + "envelope": []interface{}{ + map[string]interface{}{ + "upsert": true, + "upsert_options": []interface{}{ + map[string]interface{}{ + "value_decoding_errors": []interface{}{ + map[string]interface{}{ + "inline": []interface{}{ + map[string]interface{}{ + "enabled": true, + "alias": "decoding_error", + }, + }, + }, + }, + }, + }, + }, + }, +} + +func TestResourceSourceTableKafkaCreate(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableKafka().Schema, inSourceTableKafka) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."kafka_source" + \(REFERENCE "upstream_table"\) + FORMAT JSON + INCLUDE KEY AS message_key, HEADERS AS message_headers, PARTITION AS message_partition + ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS decoding_error\)\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` + testhelpers.MockSourceTableScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableKafkaRead(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableKafka().Schema, inSourceTableKafka) + d.SetId("u1") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableRead(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + + r.Equal("table", d.Get("name").(string)) + r.Equal("schema", d.Get("schema_name").(string)) + r.Equal("database", d.Get("database_name").(string)) + }) +} + +func TestResourceSourceTableKafkaUpdate(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableKafka().Schema, inSourceTableKafka) + d.SetId("u1") + d.Set("name", "old_table") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + mock.ExpectExec(`ALTER TABLE "database"."schema"."" RENAME TO "database"."schema"."table"`).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableUpdate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableKafkaDelete(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableKafka().Schema, inSourceTableKafka) + d.SetId("u1") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + mock.ExpectExec(`DROP TABLE "database"."schema"."table"`).WillReturnResult(sqlmock.NewResult(1, 1)) + + if err := sourceTableDelete(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableKafkaCreateWithAvroFormat(t *testing.T) { + r := require.New(t) + inSourceTableKafkaAvro := map[string]interface{}{ + "name": "table_avro", + "schema_name": "schema", + "database_name": "database", + "source": []interface{}{ + map[string]interface{}{ + "name": "kafka_source", + "schema_name": "public", + "database_name": "materialize", + }, + }, + "upstream_name": "upstream_table", + "format": []interface{}{ + map[string]interface{}{ + "avro": []interface{}{ + map[string]interface{}{ + "schema_registry_connection": []interface{}{ + map[string]interface{}{ + "name": "sr_conn", + "schema_name": "public", + "database_name": "materialize", + }, + }, + }, + }, + }, + }, + "envelope": []interface{}{ + map[string]interface{}{ + "debezium": true, + }, + }, + } + d := schema.TestResourceDataRaw(t, SourceTableKafka().Schema, inSourceTableKafkaAvro) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table_avro" + FROM SOURCE "materialize"."public"."kafka_source" + \(REFERENCE "upstream_table"\) + FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION "materialize"."public"."sr_conn" + ENVELOPE DEBEZIUM;`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table_avro'` + testhelpers.MockSourceTableScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableKafkaCreateIncludeTrueNoAlias(t *testing.T) { + r := require.New(t) + + testInSourceTableKafka := inSourceTableKafka + testInSourceTableKafka["include_key"] = true + delete(testInSourceTableKafka, "include_key_alias") + testInSourceTableKafka["include_headers"] = true + delete(testInSourceTableKafka, "include_headers_alias") + testInSourceTableKafka["include_partition"] = true + delete(testInSourceTableKafka, "include_partition_alias") + testInSourceTableKafka["include_offset"] = true + testInSourceTableKafka["include_timestamp"] = true + + d := schema.TestResourceDataRaw(t, SourceTableKafka().Schema, testInSourceTableKafka) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."kafka_source" + \(REFERENCE "upstream_table"\) + FORMAT JSON + INCLUDE KEY, HEADERS, PARTITION, OFFSET, TIMESTAMP + ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS decoding_error\)\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` + testhelpers.MockSourceTableScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableKafkaCreateIncludeFalseWithAlias(t *testing.T) { + r := require.New(t) + + testInSourceTableKafka := inSourceTableKafka + testInSourceTableKafka["include_key"] = false + testInSourceTableKafka["include_headers"] = false + testInSourceTableKafka["include_partition"] = false + testInSourceTableKafka["include_offset"] = false + testInSourceTableKafka["include_timestamp"] = false + + d := schema.TestResourceDataRaw(t, SourceTableKafka().Schema, testInSourceTableKafka) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table" + FROM SOURCE "materialize"."public"."kafka_source" + \(REFERENCE "upstream_table"\) + FORMAT JSON + ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS decoding_error\)\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` + testhelpers.MockSourceTableScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableKafkaCreateWithCSVFormat(t *testing.T) { + r := require.New(t) + inSourceTableKafkaCSV := map[string]interface{}{ + "name": "table_csv", + "schema_name": "schema", + "database_name": "database", + "source": []interface{}{ + map[string]interface{}{ + "name": "kafka_source", + "schema_name": "public", + "database_name": "materialize", + }, + }, + "upstream_name": "upstream_table", + "format": []interface{}{ + map[string]interface{}{ + "csv": []interface{}{ + map[string]interface{}{ + "delimited_by": ",", + "header": []interface{}{"column1", "column2", "column3"}, + }, + }, + }, + }, + } + d := schema.TestResourceDataRaw(t, SourceTableKafka().Schema, inSourceTableKafkaCSV) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table_csv" + FROM SOURCE "materialize"."public"."kafka_source" + \(REFERENCE "upstream_table"\) + FORMAT CSV WITH HEADER \( column1, column2, column3 \) DELIMITER ',';`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table_csv'` + testhelpers.MockSourceTableScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableKafkaCreateWithKeyAndValueFormat(t *testing.T) { + r := require.New(t) + inSourceTableKafkaKeyValue := map[string]interface{}{ + "name": "table_key_value", + "schema_name": "schema", + "database_name": "database", + "source": []interface{}{ + map[string]interface{}{ + "name": "kafka_source", + "schema_name": "public", + "database_name": "materialize", + }, + }, + "upstream_name": "upstream_table", + "key_format": []interface{}{ + map[string]interface{}{ + "json": true, + }, + }, + "value_format": []interface{}{ + map[string]interface{}{ + "avro": []interface{}{ + map[string]interface{}{ + "schema_registry_connection": []interface{}{ + map[string]interface{}{ + "name": "sr_conn", + "schema_name": "public", + "database_name": "materialize", + }, + }, + }, + }, + }, + }, + } + d := schema.TestResourceDataRaw(t, SourceTableKafka().Schema, inSourceTableKafkaKeyValue) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table_key_value" + FROM SOURCE "materialize"."public"."kafka_source" + \(REFERENCE "upstream_table"\) + KEY FORMAT JSON + VALUE FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION "materialize"."public"."sr_conn";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table_key_value'` + testhelpers.MockSourceTableScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableKafkaCreateWithProtobufFormat(t *testing.T) { + r := require.New(t) + inSourceTableKafkaProtobuf := map[string]interface{}{ + "name": "table_protobuf", + "schema_name": "schema", + "database_name": "database", + "source": []interface{}{ + map[string]interface{}{ + "name": "kafka_source", + "schema_name": "public", + "database_name": "materialize", + }, + }, + "upstream_name": "upstream_table", + "format": []interface{}{ + map[string]interface{}{ + "protobuf": []interface{}{ + map[string]interface{}{ + "schema_registry_connection": []interface{}{ + map[string]interface{}{ + "name": "sr_conn", + "schema_name": "public", + "database_name": "materialize", + }, + }, + "message": "MyMessage", + }, + }, + }, + }, + "envelope": []interface{}{ + map[string]interface{}{ + "none": true, + }, + }, + } + d := schema.TestResourceDataRaw(t, SourceTableKafka().Schema, inSourceTableKafkaProtobuf) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec( + `CREATE TABLE "database"."schema"."table_protobuf" + FROM SOURCE "materialize"."public"."kafka_source" + \(REFERENCE "upstream_table"\) + FORMAT PROTOBUF MESSAGE 'MyMessage' USING CONFLUENT SCHEMA REGISTRY CONNECTION "materialize"."public"."sr_conn" + ENVELOPE NONE;`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table_protobuf'` + testhelpers.MockSourceTableScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableScan(mock, pp) + + if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} From fe382fac2ea61a0adec22ec0ca4072d9b5b5b546 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 16 Sep 2024 18:20:48 +0300 Subject: [PATCH 16/46] Review updates --- docs/guides/materialize_source_table.md | 2 +- docs/resources/source_kafka.md | 4 +-- docs/resources/source_table_kafka.md | 5 +-- docs/resources/source_table_mysql.md | 2 +- pkg/materialize/source_table_kafka.go | 12 ------- pkg/materialize/source_table_mysql.go | 18 +++++----- pkg/materialize/source_table_mysql_test.go | 4 +-- .../acceptance_source_table_mysql_test.go | 6 ++-- pkg/resources/resource_source_kafka.go | 6 ++-- pkg/resources/resource_source_table_kafka.go | 36 +------------------ pkg/resources/resource_source_table_mysql.go | 10 +++--- .../resource_source_table_mysql_test.go | 4 +-- .../guides/materialize_source_table.md.tmpl | 2 +- 13 files changed, 30 insertions(+), 81 deletions(-) diff --git a/docs/guides/materialize_source_table.md b/docs/guides/materialize_source_table.md index 8d768cd7..3ada7b22 100644 --- a/docs/guides/materialize_source_table.md +++ b/docs/guides/materialize_source_table.md @@ -11,7 +11,7 @@ In previous versions of the Materialize Terraform provider, source tables were d This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table_{source}` resource. -For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. +For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. For Kafka sources, you will need to create at least one `materialize_source_table_kafka` table to hold data for the kafka topic. ## Old Approach diff --git a/docs/resources/source_kafka.md b/docs/resources/source_kafka.md index 01f6a47f..593f5511 100644 --- a/docs/resources/source_kafka.md +++ b/docs/resources/source_kafka.md @@ -73,8 +73,8 @@ resource "materialize_source_kafka" "example_source_kafka" { - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `start_offset` (List of Number, Deprecated) Read partitions from the specified offset. Deprecated: Use the new materialize_source_table_kafka resource instead. -- `start_timestamp` (Number, Deprecated) Use the specified value to set `START OFFSET` based on the Kafka timestamp. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `start_offset` (List of Number) Read partitions from the specified offset. +- `start_timestamp` (Number) Use the specified value to set `START OFFSET` based on the Kafka timestamp. - `value_format` (Block List, Max: 1) Set the value format explicitly. (see [below for nested schema](#nestedblock--value_format)) ### Read-Only diff --git a/docs/resources/source_table_kafka.md b/docs/resources/source_table_kafka.md index 130e144e..63cc3081 100644 --- a/docs/resources/source_table_kafka.md +++ b/docs/resources/source_table_kafka.md @@ -19,7 +19,7 @@ A Kafka source describes a Kafka cluster you want Materialize to read data from. - `name` (String) The identifier for the source. - `source` (Block List, Min: 1, Max: 1) The source this table is created from. (see [below for nested schema](#nestedblock--source)) -- `upstream_name` (String) The name of the table in the upstream database. +- `upstream_name` (String) The name of the Kafka topic in the upstream Kafka cluster. ### Optional @@ -42,9 +42,6 @@ A Kafka source describes a Kafka cluster you want Materialize to read data from. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `start_offset` (List of Number) Read partitions from the specified offset. -- `start_timestamp` (Number) Use the specified value to set `START OFFSET` based on the Kafka timestamp. -- `upstream_schema_name` (String) The schema of the table in the upstream database. - `value_format` (Block List, Max: 1) Set the value format explicitly. (see [below for nested schema](#nestedblock--value_format)) ### Read-Only diff --git a/docs/resources/source_table_mysql.md b/docs/resources/source_table_mysql.md index 309374e2..3ac79870 100644 --- a/docs/resources/source_table_mysql.md +++ b/docs/resources/source_table_mysql.md @@ -48,7 +48,7 @@ resource "materialize_source_table_mysql" "mysql_table_from_source" { - `comment` (String) **Public Preview** Comment on an object in the database. - `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. -- `ignore_columns` (List of String) Ignore specific columns when reading data from MySQL. +- `exclude_columns` (List of String) Exclude specific columns when reading data from MySQL. The option used to be called `ignore_columns`. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the table schema in Materialize. Defaults to `public`. diff --git a/pkg/materialize/source_table_kafka.go b/pkg/materialize/source_table_kafka.go index dddc3ffb..f06696d3 100644 --- a/pkg/materialize/source_table_kafka.go +++ b/pkg/materialize/source_table_kafka.go @@ -23,8 +23,6 @@ type SourceTableKafkaBuilder struct { keyFormat SourceFormatSpecStruct valueFormat SourceFormatSpecStruct envelope KafkaSourceEnvelopeStruct - startOffset []int - startTimestamp int exposeProgress IdentifierSchemaStruct } @@ -109,16 +107,6 @@ func (b *SourceTableKafkaBuilder) ValueFormat(v SourceFormatSpecStruct) *SourceT return b } -func (b *SourceTableKafkaBuilder) StartOffset(s []int) *SourceTableKafkaBuilder { - b.startOffset = s - return b -} - -func (b *SourceTableKafkaBuilder) StartTimestamp(s int) *SourceTableKafkaBuilder { - b.startTimestamp = s - return b -} - func (b *SourceTableKafkaBuilder) ExposeProgress(e IdentifierSchemaStruct) *SourceTableKafkaBuilder { b.exposeProgress = e return b diff --git a/pkg/materialize/source_table_mysql.go b/pkg/materialize/source_table_mysql.go index a7e34072..b55b01c9 100644 --- a/pkg/materialize/source_table_mysql.go +++ b/pkg/materialize/source_table_mysql.go @@ -12,8 +12,8 @@ import ( // TODO: Add upstream table name and schema name type SourceTableMySQLParams struct { SourceTableParams - IgnoreColumns pq.StringArray `db:"ignore_columns"` - TextColumns pq.StringArray `db:"text_columns"` + ExcludeColumns pq.StringArray `db:"exclude_columns"` + TextColumns pq.StringArray `db:"text_columns"` } var sourceTableMySQLQuery = ` @@ -81,8 +81,8 @@ func ScanSourceTableMySQL(conn *sqlx.DB, id string) (SourceTableMySQLParams, err // SourceTableMySQLBuilder for MySQL sources type SourceTableMySQLBuilder struct { *SourceTableBuilder - textColumns []string - ignoreColumns []string + textColumns []string + excludeColumns []string } func NewSourceTableMySQLBuilder(conn *sqlx.DB, obj MaterializeObject) *SourceTableMySQLBuilder { @@ -96,8 +96,8 @@ func (b *SourceTableMySQLBuilder) TextColumns(c []string) *SourceTableMySQLBuild return b } -func (b *SourceTableMySQLBuilder) IgnoreColumns(c []string) *SourceTableMySQLBuilder { - b.ignoreColumns = c +func (b *SourceTableMySQLBuilder) ExcludeColumns(c []string) *SourceTableMySQLBuilder { + b.excludeColumns = c return b } @@ -110,9 +110,9 @@ func (b *SourceTableMySQLBuilder) Create() error { options = append(options, fmt.Sprintf(`TEXT COLUMNS (%s)`, s)) } - if len(b.ignoreColumns) > 0 { - s := strings.Join(b.ignoreColumns, ", ") - options = append(options, fmt.Sprintf(`IGNORE COLUMNS (%s)`, s)) + if len(b.excludeColumns) > 0 { + s := strings.Join(b.excludeColumns, ", ") + options = append(options, fmt.Sprintf(`EXCLUDE COLUMNS (%s)`, s)) } if len(options) > 0 { diff --git a/pkg/materialize/source_table_mysql_test.go b/pkg/materialize/source_table_mysql_test.go index 346edf1c..31b1082b 100644 --- a/pkg/materialize/source_table_mysql_test.go +++ b/pkg/materialize/source_table_mysql_test.go @@ -16,7 +16,7 @@ func TestSourceTableCreateWithMySQLSource(t *testing.T) { `CREATE TABLE "database"."schema"."table" FROM SOURCE "materialize"."public"."source" \(REFERENCE "upstream_schema"."upstream_table"\) - WITH \(TEXT COLUMNS \(column1, column2\), IGNORE COLUMNS \(ignore1, ignore2\)\);`, + WITH \(TEXT COLUMNS \(column1, column2\), EXCLUDE COLUMNS \(exclude1, exclude2\)\);`, ).WillReturnResult(sqlmock.NewResult(1, 1)) b := NewSourceTableMySQLBuilder(db, sourceTableMySQL) @@ -24,7 +24,7 @@ func TestSourceTableCreateWithMySQLSource(t *testing.T) { b.UpstreamName("upstream_table") b.UpstreamSchemaName("upstream_schema") b.TextColumns([]string{"column1", "column2"}) - b.IgnoreColumns([]string{"ignore1", "ignore2"}) + b.ExcludeColumns([]string{"exclude1", "exclude2"}) if err := b.Create(); err != nil { t.Fatal(err) diff --git a/pkg/provider/acceptance_source_table_mysql_test.go b/pkg/provider/acceptance_source_table_mysql_test.go index 30a45eee..835d4470 100644 --- a/pkg/provider/acceptance_source_table_mysql_test.go +++ b/pkg/provider/acceptance_source_table_mysql_test.go @@ -29,8 +29,8 @@ func TestAccSourceTableMySQL_basic(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "schema_name", "public"), resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "upstream_name", "mysql_table1"), resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "upstream_schema_name", "shop"), - resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "ignore_columns.#", "1"), - resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "ignore_columns.0", "banned"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "exclude_columns.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "exclude_columns.0", "banned"), resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "source.#", "1"), resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "source.0.name", nameSpace+"_source_mysql"), resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "source.0.schema_name", "public"), @@ -151,7 +151,7 @@ func testAccSourceTableMySQLBasicResource(nameSpace string) string { upstream_name = "mysql_table1" upstream_schema_name = "shop" - ignore_columns = ["banned"] + exclude_columns = ["banned"] } `, nameSpace) } diff --git a/pkg/resources/resource_source_kafka.go b/pkg/resources/resource_source_kafka.go index d9d650e5..7641e9ac 100644 --- a/pkg/resources/resource_source_kafka.go +++ b/pkg/resources/resource_source_kafka.go @@ -182,8 +182,7 @@ var sourceKafkaSchema = map[string]*schema.Schema{ ForceNew: true, }, "start_offset": { - Description: "Read partitions from the specified offset. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Read partitions from the specified offset.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeInt}, Optional: true, @@ -191,8 +190,7 @@ var sourceKafkaSchema = map[string]*schema.Schema{ ConflictsWith: []string{"start_timestamp"}, }, "start_timestamp": { - Description: "Use the specified value to set `START OFFSET` based on the Kafka timestamp. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Use the specified value to set `START OFFSET` based on the Kafka timestamp.", Type: schema.TypeInt, Optional: true, ForceNew: true, diff --git a/pkg/resources/resource_source_table_kafka.go b/pkg/resources/resource_source_table_kafka.go index f60d63f9..c03d63ab 100644 --- a/pkg/resources/resource_source_table_kafka.go +++ b/pkg/resources/resource_source_table_kafka.go @@ -27,13 +27,7 @@ var sourceTableKafkaSchema = map[string]*schema.Schema{ Type: schema.TypeString, Required: true, ForceNew: true, - Description: "The name of the table in the upstream database.", - }, - "upstream_schema_name": { - Type: schema.TypeString, - Optional: true, - ForceNew: true, - Description: "The schema of the table in the upstream database.", + Description: "The name of the Kafka topic in the upstream Kafka cluster.", }, "include_key": { Description: "Include a column containing the Kafka message key.", @@ -174,21 +168,6 @@ var sourceTableKafkaSchema = map[string]*schema.Schema{ Optional: true, ForceNew: true, }, - "start_offset": { - Description: "Read partitions from the specified offset.", - Type: schema.TypeList, - Elem: &schema.Schema{Type: schema.TypeInt}, - Optional: true, - ForceNew: true, - ConflictsWith: []string{"start_timestamp"}, - }, - "start_timestamp": { - Description: "Use the specified value to set `START OFFSET` based on the Kafka timestamp.", - Type: schema.TypeInt, - Optional: true, - ForceNew: true, - ConflictsWith: []string{"start_offset"}, - }, "expose_progress": IdentifierSchema(IdentifierSchemaParams{ Elem: "expose_progress", Description: "The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`.", @@ -233,10 +212,6 @@ func sourceTableKafkaCreate(ctx context.Context, d *schema.ResourceData, meta an b.UpstreamName(d.Get("upstream_name").(string)) - if v, ok := d.GetOk("upstream_schema_name"); ok { - b.UpstreamSchemaName(v.(string)) - } - if v, ok := d.GetOk("include_key"); ok && v.(bool) { if alias, ok := d.GetOk("include_key_alias"); ok { b.IncludeKeyAlias(alias.(string)) @@ -297,15 +272,6 @@ func sourceTableKafkaCreate(ctx context.Context, d *schema.ResourceData, meta an b.Envelope(envelope) } - if v, ok := d.GetOk("start_offset"); ok { - so := materialize.GetSliceValueInt(v.([]interface{})) - b.StartOffset(so) - } - - if v, ok := d.GetOk("start_timestamp"); ok { - b.StartTimestamp(v.(int)) - } - if v, ok := d.GetOk("expose_progress"); ok { e := materialize.GetIdentifierSchemaStruct(v) b.ExposeProgress(e) diff --git a/pkg/resources/resource_source_table_mysql.go b/pkg/resources/resource_source_table_mysql.go index aa72bed6..17346dee 100644 --- a/pkg/resources/resource_source_table_mysql.go +++ b/pkg/resources/resource_source_table_mysql.go @@ -42,8 +42,8 @@ var sourceTableMySQLSchema = map[string]*schema.Schema{ Optional: true, ForceNew: true, }, - "ignore_columns": { - Description: "Ignore specific columns when reading data from MySQL.", + "exclude_columns": { + Description: "Exclude specific columns when reading data from MySQL. The option used to be called `ignore_columns`.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, @@ -99,12 +99,12 @@ func sourceTableMySQLCreate(ctx context.Context, d *schema.ResourceData, meta an b.TextColumns(textColumns) } - if v, ok := d.GetOk("ignore_columns"); ok && len(v.([]interface{})) > 0 { - columns, err := materialize.GetSliceValueString("ignore_columns", v.([]interface{})) + if v, ok := d.GetOk("exclude_columns"); ok && len(v.([]interface{})) > 0 { + columns, err := materialize.GetSliceValueString("exclude_columns", v.([]interface{})) if err != nil { return diag.FromErr(err) } - b.IgnoreColumns(columns) + b.ExcludeColumns(columns) } if err := b.Create(); err != nil { diff --git a/pkg/resources/resource_source_table_mysql_test.go b/pkg/resources/resource_source_table_mysql_test.go index 40a0ffed..a082899d 100644 --- a/pkg/resources/resource_source_table_mysql_test.go +++ b/pkg/resources/resource_source_table_mysql_test.go @@ -25,7 +25,7 @@ var inSourceTableMySQL = map[string]interface{}{ "upstream_name": "upstream_table", "upstream_schema_name": "upstream_schema", "text_columns": []interface{}{"column1", "column2"}, - "ignore_columns": []interface{}{"column3", "column4"}, + "exclude_columns": []interface{}{"column3", "column4"}, } func TestResourceSourceTableMySQLCreate(t *testing.T) { @@ -39,7 +39,7 @@ func TestResourceSourceTableMySQLCreate(t *testing.T) { `CREATE TABLE "database"."schema"."table" FROM SOURCE "materialize"."public"."source" \(REFERENCE "upstream_schema"."upstream_table"\) - WITH \(TEXT COLUMNS \(column1, column2\), IGNORE COLUMNS \(column3, column4\)\);`, + WITH \(TEXT COLUMNS \(column1, column2\), EXCLUDE COLUMNS \(column3, column4\)\);`, ).WillReturnResult(sqlmock.NewResult(1, 1)) // Query Id diff --git a/templates/guides/materialize_source_table.md.tmpl b/templates/guides/materialize_source_table.md.tmpl index 8d768cd7..3ada7b22 100644 --- a/templates/guides/materialize_source_table.md.tmpl +++ b/templates/guides/materialize_source_table.md.tmpl @@ -11,7 +11,7 @@ In previous versions of the Materialize Terraform provider, source tables were d This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table_{source}` resource. -For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. +For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. For Kafka sources, you will need to create at least one `materialize_source_table_kafka` table to hold data for the kafka topic. ## Old Approach From 3026124693a748890206b417f975e658daac6422 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 16 Sep 2024 23:17:29 +0300 Subject: [PATCH 17/46] Update guide migration guide --- docs/guides/materialize_source_table.md | 187 ++++++++++-------- .../guides/materialize_source_table.md.tmpl | 187 ++++++++++-------- 2 files changed, 220 insertions(+), 154 deletions(-) diff --git a/docs/guides/materialize_source_table.md b/docs/guides/materialize_source_table.md index 3ada7b22..119ab0e9 100644 --- a/docs/guides/materialize_source_table.md +++ b/docs/guides/materialize_source_table.md @@ -1,10 +1,3 @@ ---- -page_title: "Source versioning: migrating to `materialize_source_table` Resource" -subcategory: "" -description: |- - ---- - # Source versioning: migrating to `materialize_source_table_{source}` Resource In previous versions of the Materialize Terraform provider, source tables were defined within the source resource itself and were considered subsources of the source rather than separate entities. @@ -36,19 +29,42 @@ resource "materialize_source_mysql" "mysql_source" { } ``` -The same approach was used for other source types such as Postgres and the load generator sources. +### Example: Kafka Source + +```hcl +resource "materialize_source_kafka" "example_source_kafka_format_text" { + name = "source_kafka_text" + comment = "source kafka comment" + cluster_name = materialize_cluster.cluster_source.name + topic = "topic1" + + kafka_connection { + name = materialize_connection_kafka.kafka_connection.name + schema_name = materialize_connection_kafka.kafka_connection.schema_name + database_name = materialize_connection_kafka.kafka_connection.database_name + } + key_format { + text = true + } + value_format { + text = true + } +} +``` ## New Approach -The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table_mysql` resource. +The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table_{source}` resource. ## Manual Migration Process -This manual migration process requires users to create new source tables using the new `materialize_source_table_{source}` resource and then remove the old ones. In this example, we will use MySQL as the source type. +This manual migration process requires users to create new source tables using the new `materialize_source_table_{source}` resource and then remove the old ones. We'll cover examples for both MySQL and Kafka sources. + +### Step 1: Define `materialize_source_table_{source}` Resources -### Step 1: Define `materialize_source_table_mysql` Resources +Before making any changes to your existing source resources, create new `materialize_source_table_{source}` resources for each table that is currently defined within your sources. -Before making any changes to your existing source resources, create new `materialize_source_table_mysql` resources for each table that is currently defined within your sources. This ensures that the tables are preserved during the migration: +#### MySQL Example: ```hcl resource "materialize_source_table_mysql" "mysql_table_from_source" { @@ -68,15 +84,40 @@ resource "materialize_source_table_mysql" "mysql_table_from_source" { } ``` -### Step 2: Apply the Changes +#### Kafka Example: + +```hcl +resource "materialize_source_table_kafka" "kafka_table_from_source" { + name = "kafka_table_from_source" + schema_name = "public" + database_name = "materialize" + + source_name { + name = materialize_source_kafka.kafka_source.name + } + + key_format { + text = true + } -Run `terraform plan` and `terraform apply` to create the new `materialize_source_table_mysql` resources. This step ensures that the tables are defined separately from the source and are not removed from Materialize. + value_format { + text = true + } + +} +``` + +### Step 2: Apply the Changes -> **Note:** This will start an ingestion process for the newly created source tables. +Run `terraform plan` and `terraform apply` to create the new `materialize_source_table_{source}` resources. ### Step 3: Remove Table Blocks from Source Resources -Once the new `materialize_source_table_mysql` resources are successfully created, you can safely remove the `table` blocks from your existing source resources: +Once the new `materialize_source_table_{source}` resources are successfully created, remove all the deprecated and table-specific attributes from your source resources. + +#### MySQL Example: + +For MySQL sources, remove the `table` block and any table-specific attributes from the source resource: ```hcl resource "materialize_source_mysql" "mysql_source" { @@ -99,96 +140,88 @@ resource "materialize_source_mysql" "mysql_source" { } ``` -This will drop the old tables from the source resources. - -### Step 4: Update Terraform State - -After removing the `table` blocks from your source resources, run `terraform plan` and `terraform apply` again to update the Terraform state and apply the changes. - -### Step 5: Verify the Migration - -After applying the changes, verify that your tables are still correctly set up in Materialize by checking the table definitions using Materialize’s SQL commands. - -During the migration, you can use both the old `table` blocks and the new `materialize_source_table_{source}` resources simultaneously. This allows for a gradual transition until the old method is fully deprecated. - -The same approach can be used for other source types such as Postgres, eg. `materialize_source_table_postgres`. - -## Automated Migration Process (TBD) - -> **Note:** This will still not work as the previous source tables are considered subsources of the source and are missing from the `mz_tables` table in Materialize so we can't import them directly without recreating them. - -Once the migration on the Materialize side has been implemented, a more automated migration process will be available. The steps will include: - -### Step 1: Define `materialize_source_table_{source}` Resources +#### Kafka Example: -First, define the new `materialize_source_table_mysql` resources for each table: +For Kafka sources, remove the `format`, `include_key`, `include_headers`, and other table-specific attributes from the source resource: ```hcl -resource "materialize_source_table_mysql" "mysql_table_from_source" { - name = "mysql_table1_from_source" - schema_name = "public" - database_name = "materialize" +resource "materialize_source_kafka" "kafka_source" { + name = "kafka_source" + cluster_name = "cluster_name" - source { - name = materialize_source_mysql.mysql_source.name - // Define the schema and database for the source if needed + kafka_connection { + name = materialize_connection_kafka.kafka_connection.name } - upstream_name = "mysql_table1" - upstream_schema_name = "shop" + topic = "example_topic" - ignore_columns = ["about"] + lifecycle { + ignore_changes = [ + include_key, + include_headers, + format, + ... + ] + } + // Remove the format, include_key, include_headers, and other table-specific attributes } ``` -### Step 2: Modify the Existing Source Resource - -Next, modify the existing source resource by removing the `table` blocks and adding an `ignore_changes` directive for the `table` attribute. This prevents Terraform from trying to delete the tables: +In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source}` resources. -```hcl -resource "materialize_source_mysql" "mysql_source" { - name = "mysql_source" - cluster_name = "cluster_name" +### Step 4: Update Terraform State - mysql_connection { - name = materialize_connection_mysql.mysql_connection.name - } +After removing the `table` blocks and the table/topic specific attributes from your source resources, run `terraform plan` and `terraform apply` again to update the Terraform state and apply the changes. - lifecycle { - ignore_changes = [table] - } -} -``` +### Step 5: Verify the Migration -- **`lifecycle { ignore_changes = [table] }`**: This directive tells Terraform to ignore changes to the `table` attribute, preventing it from trying to delete tables that were previously defined in the source resource. +After applying the changes, verify that your tables are still correctly set up in Materialize by checking the table definitions using Materialize's SQL commands. -### Step 3: Import the Existing Tables +## Importing Existing Tables -You can then import the existing tables into the new `materialize_source_table_mysql` resources without disrupting your existing setup: +To import existing tables into your Terraform state, use the following command: ```bash -terraform import materialize_source_table_mysql.mysql_table_from_source : +terraform import materialize_source_table_{source}.table_name : ``` -Replace `` with the actual region and `` with the table ID. You can find the table ID by querying the `mz_tables` table. +Replace `{source}` with the appropriate source type (e.g., `mysql`, `kafka`), `` with the actual region, and `` with the table ID. -### Step 4: Run Terraform Plan and Apply +### Important Note on Importing -Finally, run `terraform plan` and `terraform apply` to ensure that everything is correctly set up without triggering any unwanted deletions. +Due to limitations in the current read function, not all properties of the source tables are available when importing. To work around this, you'll need to use the `ignore_changes` lifecycle meta-argument for certain attributes that can't be read back from the state. -This approach allows you to migrate your tables safely without disrupting your existing setup. +For example, for a Kafka source table: -## Importing Existing Tables +```hcl +resource "materialize_source_table_kafka" "kafka_table_from_source" { + name = "kafka_table_from_source" + schema_name = "public" + database_name = "materialize" -To import existing tables into your Terraform state using the manual migration process, use the following command: + source_name = materialize_source_kafka.kafka_source.name -```bash -terraform import materialize_source_table_mysql.table_name : + include_key = true + include_headers = true + + envelope { + upsert = true + } + + lifecycle { + ignore_changes = [ + include_key, + include_headers, + envelope + ... Add other attributes here as needed + ] + } +} ``` -Ensure you replace `` with the region where the table is located and `` with the ID of the table. +This `ignore_changes` block tells Terraform to ignore changes to these attributes during subsequent applies, preventing Terraform from trying to update these values based on incomplete information from the state. -> **Note:** The `upstream_name` and `upstream_schema_name` attributes are not yet implemented on the Materialize side, so the import process will not work until these changes are made. +After importing, you may need to manually update these ignored attributes in your Terraform configuration to match the actual state in Materialize. ## Future Improvements diff --git a/templates/guides/materialize_source_table.md.tmpl b/templates/guides/materialize_source_table.md.tmpl index 3ada7b22..119ab0e9 100644 --- a/templates/guides/materialize_source_table.md.tmpl +++ b/templates/guides/materialize_source_table.md.tmpl @@ -1,10 +1,3 @@ ---- -page_title: "Source versioning: migrating to `materialize_source_table` Resource" -subcategory: "" -description: |- - ---- - # Source versioning: migrating to `materialize_source_table_{source}` Resource In previous versions of the Materialize Terraform provider, source tables were defined within the source resource itself and were considered subsources of the source rather than separate entities. @@ -36,19 +29,42 @@ resource "materialize_source_mysql" "mysql_source" { } ``` -The same approach was used for other source types such as Postgres and the load generator sources. +### Example: Kafka Source + +```hcl +resource "materialize_source_kafka" "example_source_kafka_format_text" { + name = "source_kafka_text" + comment = "source kafka comment" + cluster_name = materialize_cluster.cluster_source.name + topic = "topic1" + + kafka_connection { + name = materialize_connection_kafka.kafka_connection.name + schema_name = materialize_connection_kafka.kafka_connection.schema_name + database_name = materialize_connection_kafka.kafka_connection.database_name + } + key_format { + text = true + } + value_format { + text = true + } +} +``` ## New Approach -The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table_mysql` resource. +The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table_{source}` resource. ## Manual Migration Process -This manual migration process requires users to create new source tables using the new `materialize_source_table_{source}` resource and then remove the old ones. In this example, we will use MySQL as the source type. +This manual migration process requires users to create new source tables using the new `materialize_source_table_{source}` resource and then remove the old ones. We'll cover examples for both MySQL and Kafka sources. + +### Step 1: Define `materialize_source_table_{source}` Resources -### Step 1: Define `materialize_source_table_mysql` Resources +Before making any changes to your existing source resources, create new `materialize_source_table_{source}` resources for each table that is currently defined within your sources. -Before making any changes to your existing source resources, create new `materialize_source_table_mysql` resources for each table that is currently defined within your sources. This ensures that the tables are preserved during the migration: +#### MySQL Example: ```hcl resource "materialize_source_table_mysql" "mysql_table_from_source" { @@ -68,15 +84,40 @@ resource "materialize_source_table_mysql" "mysql_table_from_source" { } ``` -### Step 2: Apply the Changes +#### Kafka Example: + +```hcl +resource "materialize_source_table_kafka" "kafka_table_from_source" { + name = "kafka_table_from_source" + schema_name = "public" + database_name = "materialize" + + source_name { + name = materialize_source_kafka.kafka_source.name + } + + key_format { + text = true + } -Run `terraform plan` and `terraform apply` to create the new `materialize_source_table_mysql` resources. This step ensures that the tables are defined separately from the source and are not removed from Materialize. + value_format { + text = true + } + +} +``` + +### Step 2: Apply the Changes -> **Note:** This will start an ingestion process for the newly created source tables. +Run `terraform plan` and `terraform apply` to create the new `materialize_source_table_{source}` resources. ### Step 3: Remove Table Blocks from Source Resources -Once the new `materialize_source_table_mysql` resources are successfully created, you can safely remove the `table` blocks from your existing source resources: +Once the new `materialize_source_table_{source}` resources are successfully created, remove all the deprecated and table-specific attributes from your source resources. + +#### MySQL Example: + +For MySQL sources, remove the `table` block and any table-specific attributes from the source resource: ```hcl resource "materialize_source_mysql" "mysql_source" { @@ -99,96 +140,88 @@ resource "materialize_source_mysql" "mysql_source" { } ``` -This will drop the old tables from the source resources. - -### Step 4: Update Terraform State - -After removing the `table` blocks from your source resources, run `terraform plan` and `terraform apply` again to update the Terraform state and apply the changes. - -### Step 5: Verify the Migration - -After applying the changes, verify that your tables are still correctly set up in Materialize by checking the table definitions using Materialize’s SQL commands. - -During the migration, you can use both the old `table` blocks and the new `materialize_source_table_{source}` resources simultaneously. This allows for a gradual transition until the old method is fully deprecated. - -The same approach can be used for other source types such as Postgres, eg. `materialize_source_table_postgres`. - -## Automated Migration Process (TBD) - -> **Note:** This will still not work as the previous source tables are considered subsources of the source and are missing from the `mz_tables` table in Materialize so we can't import them directly without recreating them. - -Once the migration on the Materialize side has been implemented, a more automated migration process will be available. The steps will include: - -### Step 1: Define `materialize_source_table_{source}` Resources +#### Kafka Example: -First, define the new `materialize_source_table_mysql` resources for each table: +For Kafka sources, remove the `format`, `include_key`, `include_headers`, and other table-specific attributes from the source resource: ```hcl -resource "materialize_source_table_mysql" "mysql_table_from_source" { - name = "mysql_table1_from_source" - schema_name = "public" - database_name = "materialize" +resource "materialize_source_kafka" "kafka_source" { + name = "kafka_source" + cluster_name = "cluster_name" - source { - name = materialize_source_mysql.mysql_source.name - // Define the schema and database for the source if needed + kafka_connection { + name = materialize_connection_kafka.kafka_connection.name } - upstream_name = "mysql_table1" - upstream_schema_name = "shop" + topic = "example_topic" - ignore_columns = ["about"] + lifecycle { + ignore_changes = [ + include_key, + include_headers, + format, + ... + ] + } + // Remove the format, include_key, include_headers, and other table-specific attributes } ``` -### Step 2: Modify the Existing Source Resource - -Next, modify the existing source resource by removing the `table` blocks and adding an `ignore_changes` directive for the `table` attribute. This prevents Terraform from trying to delete the tables: +In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source}` resources. -```hcl -resource "materialize_source_mysql" "mysql_source" { - name = "mysql_source" - cluster_name = "cluster_name" +### Step 4: Update Terraform State - mysql_connection { - name = materialize_connection_mysql.mysql_connection.name - } +After removing the `table` blocks and the table/topic specific attributes from your source resources, run `terraform plan` and `terraform apply` again to update the Terraform state and apply the changes. - lifecycle { - ignore_changes = [table] - } -} -``` +### Step 5: Verify the Migration -- **`lifecycle { ignore_changes = [table] }`**: This directive tells Terraform to ignore changes to the `table` attribute, preventing it from trying to delete tables that were previously defined in the source resource. +After applying the changes, verify that your tables are still correctly set up in Materialize by checking the table definitions using Materialize's SQL commands. -### Step 3: Import the Existing Tables +## Importing Existing Tables -You can then import the existing tables into the new `materialize_source_table_mysql` resources without disrupting your existing setup: +To import existing tables into your Terraform state, use the following command: ```bash -terraform import materialize_source_table_mysql.mysql_table_from_source : +terraform import materialize_source_table_{source}.table_name : ``` -Replace `` with the actual region and `` with the table ID. You can find the table ID by querying the `mz_tables` table. +Replace `{source}` with the appropriate source type (e.g., `mysql`, `kafka`), `` with the actual region, and `` with the table ID. -### Step 4: Run Terraform Plan and Apply +### Important Note on Importing -Finally, run `terraform plan` and `terraform apply` to ensure that everything is correctly set up without triggering any unwanted deletions. +Due to limitations in the current read function, not all properties of the source tables are available when importing. To work around this, you'll need to use the `ignore_changes` lifecycle meta-argument for certain attributes that can't be read back from the state. -This approach allows you to migrate your tables safely without disrupting your existing setup. +For example, for a Kafka source table: -## Importing Existing Tables +```hcl +resource "materialize_source_table_kafka" "kafka_table_from_source" { + name = "kafka_table_from_source" + schema_name = "public" + database_name = "materialize" -To import existing tables into your Terraform state using the manual migration process, use the following command: + source_name = materialize_source_kafka.kafka_source.name -```bash -terraform import materialize_source_table_mysql.table_name : + include_key = true + include_headers = true + + envelope { + upsert = true + } + + lifecycle { + ignore_changes = [ + include_key, + include_headers, + envelope + ... Add other attributes here as needed + ] + } +} ``` -Ensure you replace `` with the region where the table is located and `` with the ID of the table. +This `ignore_changes` block tells Terraform to ignore changes to these attributes during subsequent applies, preventing Terraform from trying to update these values based on incomplete information from the state. -> **Note:** The `upstream_name` and `upstream_schema_name` attributes are not yet implemented on the Materialize side, so the import process will not work until these changes are made. +After importing, you may need to manually update these ignored attributes in your Terraform configuration to match the actual state in Materialize. ## Future Improvements From ca3c823493b3e5325c6a60afdc9c5d23e4324567 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 16 Sep 2024 23:21:36 +0300 Subject: [PATCH 18/46] Update guide migration guide --- docs/guides/materialize_source_table.md | 2 ++ templates/guides/materialize_source_table.md.tmpl | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/guides/materialize_source_table.md b/docs/guides/materialize_source_table.md index 119ab0e9..1a269449 100644 --- a/docs/guides/materialize_source_table.md +++ b/docs/guides/materialize_source_table.md @@ -169,6 +169,8 @@ resource "materialize_source_kafka" "kafka_source" { In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source}` resources. +> Note: We will make the changes to those attributes a no-op, so the `ignore_changes` block will not be necessary. + ### Step 4: Update Terraform State After removing the `table` blocks and the table/topic specific attributes from your source resources, run `terraform plan` and `terraform apply` again to update the Terraform state and apply the changes. diff --git a/templates/guides/materialize_source_table.md.tmpl b/templates/guides/materialize_source_table.md.tmpl index 119ab0e9..1a269449 100644 --- a/templates/guides/materialize_source_table.md.tmpl +++ b/templates/guides/materialize_source_table.md.tmpl @@ -169,6 +169,8 @@ resource "materialize_source_kafka" "kafka_source" { In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source}` resources. +> Note: We will make the changes to those attributes a no-op, so the `ignore_changes` block will not be necessary. + ### Step 4: Update Terraform State After removing the `table` blocks and the table/topic specific attributes from your source resources, run `terraform plan` and `terraform apply` again to update the Terraform state and apply the changes. From f81f05b85dc4bed87eaed0938a857b387e5a052d Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 17 Sep 2024 16:10:15 +0300 Subject: [PATCH 19/46] Add import examples for Kafka source tables --- docs/resources/source_table_kafka.md | 63 ++++++++++++++++++- docs/resources/source_table_load_generator.md | 4 +- docs/resources/source_table_mysql.md | 4 +- docs/resources/source_table_postgres.md | 4 +- .../materialize_source_table_kafka/import.sh | 5 ++ .../resource.tf | 46 ++++++++++++++ .../import.sh | 4 +- .../materialize_source_table_mysql/import.sh | 4 +- .../import.sh | 4 +- 9 files changed, 125 insertions(+), 13 deletions(-) create mode 100644 examples/resources/materialize_source_table_kafka/import.sh create mode 100644 examples/resources/materialize_source_table_kafka/resource.tf diff --git a/docs/resources/source_table_kafka.md b/docs/resources/source_table_kafka.md index 63cc3081..806dc7ba 100644 --- a/docs/resources/source_table_kafka.md +++ b/docs/resources/source_table_kafka.md @@ -10,7 +10,56 @@ description: |- A Kafka source describes a Kafka cluster you want Materialize to read data from. - +## Example Usage + +```terraform +resource "materialize_source_table_kafka" "kafka_source_table" { + name = "kafka_source_table" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_kafka.test_source_kafka.name + schema_name = materialize_source_kafka.test_source_kafka.schema_name + database_name = materialize_source_kafka.test_source_kafka.database_name + } + + upstream_name = "terraform" # The kafka source topic name + include_key = true + include_key_alias = "message_key" + include_headers = true + include_headers_alias = "message_headers" + include_partition = true + include_partition_alias = "message_partition" + include_offset = true + include_offset_alias = "message_offset" + include_timestamp = true + include_timestamp_alias = "message_timestamp" + + + key_format { + text = true + } + value_format { + json = true + } + + envelope { + upsert = true + upsert_options { + value_decoding_errors { + inline { + enabled = true + alias = "decoding_error" + } + } + } + } + + ownership_role = "mz_system" + comment = "This is a test Kafka source table" +} +``` ## Schema @@ -320,3 +369,15 @@ Optional: - `database_name` (String) The schema_registry_connection database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `schema_name` (String) The schema_registry_connection schema name. Defaults to `public`. + +## Import + +Import is supported using the following syntax: + +```shell +# Source tables can be imported using the source table id: +terraform import materialize_source_table_kafka.example_source_table_kafka : + +# Source id and information be found in the `mz_catalog.mz_tables` table +# The region is the region where the database is located (e.g. aws/us-east-1) +``` diff --git a/docs/resources/source_table_load_generator.md b/docs/resources/source_table_load_generator.md index a8f1770e..aba578f2 100644 --- a/docs/resources/source_table_load_generator.md +++ b/docs/resources/source_table_load_generator.md @@ -70,8 +70,8 @@ Optional: Import is supported using the following syntax: ```shell -# Source tables can be imported using the source id: -terraform import materialize_source_table_load_generator.example_source_table_loadgen : +# Source tables can be imported using the source table id: +terraform import materialize_source_table_load_generator.example_source_table_loadgen : # Source id and information be found in the `mz_catalog.mz_tables` table # The region is the region where the database is located (e.g. aws/us-east-1) diff --git a/docs/resources/source_table_mysql.md b/docs/resources/source_table_mysql.md index 3ac79870..193204f8 100644 --- a/docs/resources/source_table_mysql.md +++ b/docs/resources/source_table_mysql.md @@ -77,8 +77,8 @@ Optional: Import is supported using the following syntax: ```shell -# Source tables can be imported using the source id: -terraform import materialize_source_table_mysql.example_source_table_mysql : +# Source tables can be imported using the source table id: +terraform import materialize_source_table_mysql.example_source_table_mysql : # Source id and information be found in the `mz_catalog.mz_tables` table # The region is the region where the database is located (e.g. aws/us-east-1) diff --git a/docs/resources/source_table_postgres.md b/docs/resources/source_table_postgres.md index 8eebc851..53ba1334 100644 --- a/docs/resources/source_table_postgres.md +++ b/docs/resources/source_table_postgres.md @@ -75,8 +75,8 @@ Optional: Import is supported using the following syntax: ```shell -# Source tables can be imported using the source id: -terraform import materialize_source_table_postgres.example_source_table_postgres : +# Source tables can be imported using the source table id: +terraform import materialize_source_table_postgres.example_source_table_postgres : # Source id and information be found in the `mz_catalog.mz_tables` table # The region is the region where the database is located (e.g. aws/us-east-1) diff --git a/examples/resources/materialize_source_table_kafka/import.sh b/examples/resources/materialize_source_table_kafka/import.sh new file mode 100644 index 00000000..b4d52540 --- /dev/null +++ b/examples/resources/materialize_source_table_kafka/import.sh @@ -0,0 +1,5 @@ +# Source tables can be imported using the source table id: +terraform import materialize_source_table_kafka.example_source_table_kafka : + +# Source id and information be found in the `mz_catalog.mz_tables` table +# The region is the region where the database is located (e.g. aws/us-east-1) diff --git a/examples/resources/materialize_source_table_kafka/resource.tf b/examples/resources/materialize_source_table_kafka/resource.tf new file mode 100644 index 00000000..2aede69c --- /dev/null +++ b/examples/resources/materialize_source_table_kafka/resource.tf @@ -0,0 +1,46 @@ +resource "materialize_source_table_kafka" "kafka_source_table" { + name = "kafka_source_table" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_kafka.test_source_kafka.name + schema_name = materialize_source_kafka.test_source_kafka.schema_name + database_name = materialize_source_kafka.test_source_kafka.database_name + } + + upstream_name = "terraform" # The kafka source topic name + include_key = true + include_key_alias = "message_key" + include_headers = true + include_headers_alias = "message_headers" + include_partition = true + include_partition_alias = "message_partition" + include_offset = true + include_offset_alias = "message_offset" + include_timestamp = true + include_timestamp_alias = "message_timestamp" + + + key_format { + text = true + } + value_format { + json = true + } + + envelope { + upsert = true + upsert_options { + value_decoding_errors { + inline { + enabled = true + alias = "decoding_error" + } + } + } + } + + ownership_role = "mz_system" + comment = "This is a test Kafka source table" +} diff --git a/examples/resources/materialize_source_table_load_generator/import.sh b/examples/resources/materialize_source_table_load_generator/import.sh index 50e09749..c673df14 100644 --- a/examples/resources/materialize_source_table_load_generator/import.sh +++ b/examples/resources/materialize_source_table_load_generator/import.sh @@ -1,5 +1,5 @@ -# Source tables can be imported using the source id: -terraform import materialize_source_table_load_generator.example_source_table_loadgen : +# Source tables can be imported using the source table id: +terraform import materialize_source_table_load_generator.example_source_table_loadgen : # Source id and information be found in the `mz_catalog.mz_tables` table # The region is the region where the database is located (e.g. aws/us-east-1) diff --git a/examples/resources/materialize_source_table_mysql/import.sh b/examples/resources/materialize_source_table_mysql/import.sh index 4ed92a32..1d910379 100644 --- a/examples/resources/materialize_source_table_mysql/import.sh +++ b/examples/resources/materialize_source_table_mysql/import.sh @@ -1,5 +1,5 @@ -# Source tables can be imported using the source id: -terraform import materialize_source_table_mysql.example_source_table_mysql : +# Source tables can be imported using the source table id: +terraform import materialize_source_table_mysql.example_source_table_mysql : # Source id and information be found in the `mz_catalog.mz_tables` table # The region is the region where the database is located (e.g. aws/us-east-1) diff --git a/examples/resources/materialize_source_table_postgres/import.sh b/examples/resources/materialize_source_table_postgres/import.sh index 1580dd37..91e794a4 100644 --- a/examples/resources/materialize_source_table_postgres/import.sh +++ b/examples/resources/materialize_source_table_postgres/import.sh @@ -1,5 +1,5 @@ -# Source tables can be imported using the source id: -terraform import materialize_source_table_postgres.example_source_table_postgres : +# Source tables can be imported using the source table id: +terraform import materialize_source_table_postgres.example_source_table_postgres : # Source id and information be found in the `mz_catalog.mz_tables` table # The region is the region where the database is located (e.g. aws/us-east-1) From 5f435352561ff84634b9c20192e5e4842ce69365 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 20 Sep 2024 16:11:48 +0300 Subject: [PATCH 20/46] Add upstream mysql and postgres table names --- pkg/materialize/source_table.go | 2 +- pkg/materialize/source_table_mysql.go | 5 +- pkg/materialize/source_table_postgres.go | 4 + .../acceptance_source_table_kafka_test.go | 232 ++++++++++++++++++ .../acceptance_source_table_mysql_test.go | 7 +- .../acceptance_source_table_postgres_test.go | 5 + pkg/resources/resource_source_table_mysql.go | 15 +- .../resource_source_table_postgres.go | 15 +- 8 files changed, 265 insertions(+), 20 deletions(-) create mode 100644 pkg/provider/acceptance_source_table_kafka_test.go diff --git a/pkg/materialize/source_table.go b/pkg/materialize/source_table.go index c24c3ec0..1b6fdc34 100644 --- a/pkg/materialize/source_table.go +++ b/pkg/materialize/source_table.go @@ -18,7 +18,7 @@ type SourceTableParams struct { SourceSchemaName sql.NullString `db:"source_schema_name"` SourceDatabaseName sql.NullString `db:"source_database_name"` SourceType sql.NullString `db:"source_type"` - UpstreamName sql.NullString `db:"upstream_name"` + UpstreamName sql.NullString `db:"upstream_table_name"` UpstreamSchemaName sql.NullString `db:"upstream_schema_name"` TextColumns pq.StringArray `db:"text_columns"` Comment sql.NullString `db:"comment"` diff --git a/pkg/materialize/source_table_mysql.go b/pkg/materialize/source_table_mysql.go index b55b01c9..c9aa375c 100644 --- a/pkg/materialize/source_table_mysql.go +++ b/pkg/materialize/source_table_mysql.go @@ -9,7 +9,6 @@ import ( ) // MySQL specific params and query -// TODO: Add upstream table name and schema name type SourceTableMySQLParams struct { SourceTableParams ExcludeColumns pq.StringArray `db:"exclude_columns"` @@ -25,6 +24,8 @@ var sourceTableMySQLQuery = ` mz_sources.name AS source_name, source_schemas.name AS source_schema_name, source_databases.name AS source_database_name, + mz_mysql_source_tables.table_name AS upstream_table_name, + mz_mysql_source_tables.schema_name AS upstream_schema_name, mz_sources.type AS source_type, comments.comment AS comment, mz_roles.name AS owner_name, @@ -40,6 +41,8 @@ var sourceTableMySQLQuery = ` ON mz_sources.schema_id = source_schemas.id JOIN mz_databases AS source_databases ON source_schemas.database_id = source_databases.id + LEFT JOIN mz_internal.mz_mysql_source_tables + ON mz_tables.id = mz_mysql_source_tables.id JOIN mz_roles ON mz_tables.owner_id = mz_roles.id LEFT JOIN ( diff --git a/pkg/materialize/source_table_postgres.go b/pkg/materialize/source_table_postgres.go index dc1f77db..ad658bbb 100644 --- a/pkg/materialize/source_table_postgres.go +++ b/pkg/materialize/source_table_postgres.go @@ -25,6 +25,8 @@ var sourceTablePostgresQuery = ` mz_sources.name AS source_name, source_schemas.name AS source_schema_name, source_databases.name AS source_database_name, + mz_postgres_source_tables.table_name AS upstream_table_name, + mz_postgres_source_tables.schema_name AS upstream_schema_name, mz_sources.type AS source_type, comments.comment AS comment, mz_roles.name AS owner_name, @@ -40,6 +42,8 @@ var sourceTablePostgresQuery = ` ON mz_sources.schema_id = source_schemas.id JOIN mz_databases AS source_databases ON source_schemas.database_id = source_databases.id + LEFT JOIN mz_internal.mz_postgres_source_tables + ON mz_tables.id = mz_postgres_source_tables.id JOIN mz_roles ON mz_tables.owner_id = mz_roles.id LEFT JOIN ( diff --git a/pkg/provider/acceptance_source_table_kafka_test.go b/pkg/provider/acceptance_source_table_kafka_test.go new file mode 100644 index 00000000..cc918674 --- /dev/null +++ b/pkg/provider/acceptance_source_table_kafka_test.go @@ -0,0 +1,232 @@ +package provider + +import ( + "fmt" + "testing" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/hashicorp/terraform-plugin-testing/helper/acctest" + "github.com/hashicorp/terraform-plugin-testing/helper/resource" +) + +func TestAccSourceTableKafka_basic(t *testing.T) { + addTestTopic() + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: nil, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableKafkaBasicResource(nameSpace), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table_kafka.test_kafka"), + resource.TestMatchResourceAttr("materialize_source_table_kafka.test_kafka", "id", terraformObjectIdRegex), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "name", nameSpace+"_table_kafka"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "database_name", "materialize"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "qualified_sql_name", fmt.Sprintf(`"materialize"."public"."%s_table_kafka"`, nameSpace)), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "upstream_name", "terraform"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_key", "true"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_key_alias", "message_key"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_headers", "true"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_headers_alias", "message_headers"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_partition", "true"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_partition_alias", "message_partition"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_offset", "true"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_offset_alias", "message_offset"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_timestamp", "true"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_timestamp_alias", "message_timestamp"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "format.0.json", "true"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "key_format.0.text", "true"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "value_format.0.json", "true"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "envelope.0.upsert", "true"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "envelope.0.upsert_options.0.value_decoding_errors.0.inline.0.enabled", "true"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "envelope.0.upsert_options.0.value_decoding_errors.0.inline.0.alias", "decoding_error"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "expose_progress.0.name", nameSpace+"_progress"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "ownership_role", "mz_system"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "comment", "This is a test Kafka source table"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "source.#", "1"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "source.0.name", nameSpace+"_source_kafka"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "source.0.schema_name", "public"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "source.0.database_name", "materialize"), + ), + }, + }, + }) +} + +func TestAccSourceTableKafka_update(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: nil, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableKafkaResource(nameSpace, "kafka_table2", "mz_system", ""), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table_kafka.test"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "name", nameSpace+"_table"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "upstream_name", "kafka_table2"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "ownership_role", "mz_system"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "comment", ""), + ), + }, + { + Config: testAccSourceTableKafkaResource(nameSpace, "terraform", nameSpace+"_role", "Updated comment"), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table_kafka.test"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "name", nameSpace+"_table"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "upstream_name", "terraform"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "ownership_role", nameSpace+"_role"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "comment", "Updated comment"), + ), + }, + }, + }) +} + +func TestAccSourceTableKafka_disappears(t *testing.T) { + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + CheckDestroy: testAccCheckAllSourceTableDestroyed, + Steps: []resource.TestStep{ + { + Config: testAccSourceTableKafkaResource(nameSpace, "kafka_table2", "mz_system", ""), + Check: resource.ComposeTestCheckFunc( + testAccCheckSourceTableExists("materialize_source_table_kafka.test"), + testAccCheckObjectDisappears( + materialize.MaterializeObject{ + ObjectType: "TABLE", + Name: nameSpace + "_table", + }, + ), + ), + PlanOnly: true, + ExpectNonEmptyPlan: true, + }, + }, + }) +} + +func testAccSourceTableKafkaBasicResource(nameSpace string) string { + return fmt.Sprintf(` + resource "materialize_connection_kafka" "kafka_connection" { + name = "%[1]s_connection_kafka" + kafka_broker { + broker = "redpanda:9092" + } + security_protocol = "PLAINTEXT" + } + + resource "materialize_source_kafka" "test_source_kafka" { + name = "%[1]s_source_kafka" + cluster_name = "quickstart" + topic = "terraform" + + kafka_connection { + name = materialize_connection_kafka.kafka_connection.name + } + } + + resource "materialize_source_table_kafka" "test_kafka" { + name = "%[1]s_table_kafka" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_kafka.test_source_kafka.name + } + + upstream_name = "terraform" + include_key = true + include_key_alias = "message_key" + include_headers = true + include_headers_alias = "message_headers" + include_partition = true + include_partition_alias = "message_partition" + include_offset = true + include_offset_alias = "message_offset" + include_timestamp = true + include_timestamp_alias = "message_timestamp" + + + key_format { + text = true + } + value_format { + json = true + } + + envelope { + upsert = true + upsert_options { + value_decoding_errors { + inline { + enabled = true + alias = "decoding_error" + } + } + } + } + + ownership_role = "mz_system" + comment = "This is a test Kafka source table" + } + `, nameSpace) +} + +func testAccSourceTableKafkaResource(nameSpace, upstreamName, ownershipRole, comment string) string { + return fmt.Sprintf(` + resource "materialize_connection_kafka" "kafka_connection" { + name = "%[1]s_connection_kafka" + kafka_broker { + broker = "redpanda:9092" + } + security_protocol = "PLAINTEXT" + } + + resource "materialize_source_kafka" "test_source_kafka" { + name = "%[1]s_source_kafka" + cluster_name = "quickstart" + topic = "terraform" + + kafka_connection { + name = materialize_connection_kafka.kafka_connection.name + } + + key_format { + json = true + } + value_format { + json = true + } + } + + resource "materialize_role" "test_role" { + name = "%[1]s_role" + } + + resource "materialize_source_table_kafka" "test" { + name = "%[1]s_table" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_kafka.test_source_kafka.name + schema_name = "public" + database_name = "materialize" + } + + upstream_name = "%[2]s" + + ownership_role = "%[3]s" + comment = "%[4]s" + + depends_on = [materialize_role.test_role] + } + `, nameSpace, upstreamName, ownershipRole, comment) +} diff --git a/pkg/provider/acceptance_source_table_mysql_test.go b/pkg/provider/acceptance_source_table_mysql_test.go index 835d4470..a78709b9 100644 --- a/pkg/provider/acceptance_source_table_mysql_test.go +++ b/pkg/provider/acceptance_source_table_mysql_test.go @@ -35,10 +35,13 @@ func TestAccSourceTableMySQL_basic(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "source.0.name", nameSpace+"_source_mysql"), resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "source.0.schema_name", "public"), resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "source.0.database_name", "materialize"), - // resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "text_columns.#", "1"), - // resource.TestCheckResourceAttr("materialize_source_table_mysql.test_mysql", "text_columns.0", "about"), ), }, + { + ResourceName: "materialize_source_table_mysql.test_mysql", + ImportState: true, + ImportStateVerify: false, + }, }, }) } diff --git a/pkg/provider/acceptance_source_table_postgres_test.go b/pkg/provider/acceptance_source_table_postgres_test.go index 9ffb34cc..9807ef15 100644 --- a/pkg/provider/acceptance_source_table_postgres_test.go +++ b/pkg/provider/acceptance_source_table_postgres_test.go @@ -37,6 +37,11 @@ func TestAccSourceTablePostgres_basic(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table_postgres.test_postgres", "source.0.database_name", "materialize"), ), }, + { + ResourceName: "materialize_source_table_postgres.test_postgres", + ImportState: true, + ImportStateVerify: false, + }, }, }) } diff --git a/pkg/resources/resource_source_table_mysql.go b/pkg/resources/resource_source_table_mysql.go index 17346dee..0ae89cf0 100644 --- a/pkg/resources/resource_source_table_mysql.go +++ b/pkg/resources/resource_source_table_mysql.go @@ -181,14 +181,13 @@ func sourceTableMySQLRead(ctx context.Context, d *schema.ResourceData, meta inte return diag.FromErr(err) } - // TODO: Set the upstream_name and upstream_schema_name once supported on the Materialize side - // if err := d.Set("upstream_name", t.UpstreamName.String); err != nil { - // return diag.FromErr(err) - // } - - // if err := d.Set("upstream_schema_name", t.UpstreamSchemaName.String); err != nil { - // return diag.FromErr(err) - // } + if err := d.Set("upstream_name", t.UpstreamName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("upstream_schema_name", t.UpstreamSchemaName.String); err != nil { + return diag.FromErr(err) + } if err := d.Set("ownership_role", t.OwnerName.String); err != nil { return diag.FromErr(err) diff --git a/pkg/resources/resource_source_table_postgres.go b/pkg/resources/resource_source_table_postgres.go index 391f788a..43a6564d 100644 --- a/pkg/resources/resource_source_table_postgres.go +++ b/pkg/resources/resource_source_table_postgres.go @@ -166,14 +166,13 @@ func sourceTablePostgresRead(ctx context.Context, d *schema.ResourceData, meta i return diag.FromErr(err) } - // TODO: Set the upstream_name and upstream_schema_name once supported on the Materialize side - // if err := d.Set("upstream_name", t.UpstreamName.String); err != nil { - // return diag.FromErr(err) - // } - - // if err := d.Set("upstream_schema_name", t.UpstreamSchemaName.String); err != nil { - // return diag.FromErr(err) - // } + if err := d.Set("upstream_name", t.UpstreamName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("upstream_schema_name", t.UpstreamSchemaName.String); err != nil { + return diag.FromErr(err) + } if err := d.Set("ownership_role", t.OwnerName.String); err != nil { return diag.FromErr(err) From d4b31b608c0c85f4cd9243ac5c56222b654db4f0 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 20 Sep 2024 16:37:33 +0300 Subject: [PATCH 21/46] Fix unit tests --- .../acceptance_source_table_kafka_test.go | 4 +- pkg/resources/resource_source_table_mysql.go | 46 +++++++++- .../resource_source_table_mysql_test.go | 12 +-- .../resource_source_table_postgres.go | 48 +++++++++- .../resource_source_table_postgres_test.go | 12 +-- pkg/testhelpers/mock_scans.go | 90 +++++++++++++++++++ 6 files changed, 194 insertions(+), 18 deletions(-) diff --git a/pkg/provider/acceptance_source_table_kafka_test.go b/pkg/provider/acceptance_source_table_kafka_test.go index cc918674..f0de1d3c 100644 --- a/pkg/provider/acceptance_source_table_kafka_test.go +++ b/pkg/provider/acceptance_source_table_kafka_test.go @@ -64,11 +64,11 @@ func TestAccSourceTableKafka_update(t *testing.T) { CheckDestroy: nil, Steps: []resource.TestStep{ { - Config: testAccSourceTableKafkaResource(nameSpace, "kafka_table2", "mz_system", ""), + Config: testAccSourceTableKafkaResource(nameSpace, "terraform", "mz_system", ""), Check: resource.ComposeTestCheckFunc( testAccCheckSourceTableExists("materialize_source_table_kafka.test"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "name", nameSpace+"_table"), - resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "upstream_name", "kafka_table2"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "upstream_name", "terraform"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "ownership_role", "mz_system"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "comment", ""), ), diff --git a/pkg/resources/resource_source_table_mysql.go b/pkg/resources/resource_source_table_mysql.go index 0ae89cf0..024cc67e 100644 --- a/pkg/resources/resource_source_table_mysql.go +++ b/pkg/resources/resource_source_table_mysql.go @@ -58,7 +58,7 @@ func SourceTableMySQL() *schema.Resource { return &schema.Resource{ CreateContext: sourceTableMySQLCreate, ReadContext: sourceTableMySQLRead, - UpdateContext: sourceTableUpdate, + UpdateContext: sourceTableMySQLUpdate, DeleteContext: sourceTableDelete, Importer: &schema.ResourceImporter{ @@ -131,7 +131,7 @@ func sourceTableMySQLCreate(ctx context.Context, d *schema.ResourceData, meta an } } - i, err := materialize.SourceTableId(metaDb, o) + i, err := materialize.SourceTableMySQLId(metaDb, o) if err != nil { return diag.FromErr(err) } @@ -140,6 +140,48 @@ func sourceTableMySQLCreate(ctx context.Context, d *schema.ResourceData, meta an return sourceTableMySQLRead(ctx, d, meta) } +func sourceTableMySQLUpdate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + tableName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, _, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} + + if d.HasChange("name") { + oldName, newName := d.GetChange("name") + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: oldName.(string), SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewSourceTableBuilder(metaDb, o) + if err := b.Rename(newName.(string)); err != nil { + return diag.FromErr(err) + } + } + + if d.HasChange("ownership_role") { + _, newRole := d.GetChange("ownership_role") + b := materialize.NewOwnershipBuilder(metaDb, o) + + if err := b.Alter(newRole.(string)); err != nil { + return diag.FromErr(err) + } + } + + if d.HasChange("comment") { + _, newComment := d.GetChange("comment") + b := materialize.NewCommentBuilder(metaDb, o) + + if err := b.Object(newComment.(string)); err != nil { + return diag.FromErr(err) + } + } + + return sourceTableMySQLRead(ctx, d, meta) +} + func sourceTableMySQLRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { i := d.Id() diff --git a/pkg/resources/resource_source_table_mysql_test.go b/pkg/resources/resource_source_table_mysql_test.go index a082899d..57f2bb1d 100644 --- a/pkg/resources/resource_source_table_mysql_test.go +++ b/pkg/resources/resource_source_table_mysql_test.go @@ -44,11 +44,11 @@ func TestResourceSourceTableMySQLCreate(t *testing.T) { // Query Id ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` - testhelpers.MockSourceTableScan(mock, ip) + testhelpers.MockSourceTableMySQLScan(mock, ip) // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableMySQLScan(mock, pp) if err := sourceTableMySQLCreate(context.TODO(), d, db); err != nil { t.Fatal(err) @@ -65,9 +65,9 @@ func TestResourceSourceTableMySQLRead(t *testing.T) { testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableMySQLScan(mock, pp) - if err := sourceTableRead(context.TODO(), d, db); err != nil { + if err := sourceTableMySQLRead(context.TODO(), d, db); err != nil { t.Fatal(err) } @@ -89,9 +89,9 @@ func TestResourceSourceTableMySQLUpdate(t *testing.T) { // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableMySQLScan(mock, pp) - if err := sourceTableUpdate(context.TODO(), d, db); err != nil { + if err := sourceTableMySQLUpdate(context.TODO(), d, db); err != nil { t.Fatal(err) } }) diff --git a/pkg/resources/resource_source_table_postgres.go b/pkg/resources/resource_source_table_postgres.go index 43a6564d..77d9ff07 100644 --- a/pkg/resources/resource_source_table_postgres.go +++ b/pkg/resources/resource_source_table_postgres.go @@ -51,7 +51,7 @@ func SourceTablePostgres() *schema.Resource { return &schema.Resource{ CreateContext: sourceTablePostgresCreate, ReadContext: sourceTablePostgresRead, - UpdateContext: sourceTableUpdate, + UpdateContext: sourceTablePostgresUpdate, DeleteContext: sourceTableDelete, Importer: &schema.ResourceImporter{ @@ -116,7 +116,7 @@ func sourceTablePostgresCreate(ctx context.Context, d *schema.ResourceData, meta } } - i, err := materialize.SourceTableId(metaDb, o) + i, err := materialize.SourceTablePostgresId(metaDb, o) if err != nil { return diag.FromErr(err) } @@ -125,6 +125,50 @@ func sourceTablePostgresCreate(ctx context.Context, d *schema.ResourceData, meta return sourceTablePostgresRead(ctx, d, meta) } +func sourceTablePostgresUpdate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + tableName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, _, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} + + if d.HasChange("name") { + oldName, newName := d.GetChange("name") + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: oldName.(string), SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewSourceTableBuilder(metaDb, o) + if err := b.Rename(newName.(string)); err != nil { + return diag.FromErr(err) + } + } + + // TODO: Handle source and text_columns changes once supported on the Materialize side + + if d.HasChange("ownership_role") { + _, newRole := d.GetChange("ownership_role") + b := materialize.NewOwnershipBuilder(metaDb, o) + + if err := b.Alter(newRole.(string)); err != nil { + return diag.FromErr(err) + } + } + + if d.HasChange("comment") { + _, newComment := d.GetChange("comment") + b := materialize.NewCommentBuilder(metaDb, o) + + if err := b.Object(newComment.(string)); err != nil { + return diag.FromErr(err) + } + } + + return sourceTablePostgresRead(ctx, d, meta) +} + func sourceTablePostgresRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { i := d.Id() diff --git a/pkg/resources/resource_source_table_postgres_test.go b/pkg/resources/resource_source_table_postgres_test.go index ec655d16..44747742 100644 --- a/pkg/resources/resource_source_table_postgres_test.go +++ b/pkg/resources/resource_source_table_postgres_test.go @@ -43,11 +43,11 @@ func TestResourceSourceTablePostgresCreate(t *testing.T) { // Query Id ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` - testhelpers.MockSourceTableScan(mock, ip) + testhelpers.MockSourceTablePostgresScan(mock, ip) // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTablePostgresScan(mock, pp) if err := sourceTablePostgresCreate(context.TODO(), d, db); err != nil { t.Fatal(err) @@ -64,9 +64,9 @@ func TestResourceSourceTablePostgresRead(t *testing.T) { testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTablePostgresScan(mock, pp) - if err := sourceTableRead(context.TODO(), d, db); err != nil { + if err := sourceTablePostgresRead(context.TODO(), d, db); err != nil { t.Fatal(err) } @@ -88,9 +88,9 @@ func TestResourceSourceTablePostgresUpdate(t *testing.T) { // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTablePostgresScan(mock, pp) - if err := sourceTableUpdate(context.TODO(), d, db); err != nil { + if err := sourceTablePostgresUpdate(context.TODO(), d, db); err != nil { t.Fatal(err) } }) diff --git a/pkg/testhelpers/mock_scans.go b/pkg/testhelpers/mock_scans.go index 4abed015..e64b412c 100644 --- a/pkg/testhelpers/mock_scans.go +++ b/pkg/testhelpers/mock_scans.go @@ -775,6 +775,96 @@ func MockTableScan(mock sqlmock.Sqlmock, predicate string) { mock.ExpectQuery(q).WillReturnRows(ir) } +func MockSourceTableMySQLScan(mock sqlmock.Sqlmock, predicate string) { + b := ` + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.name AS source_name, + source_schemas.name AS source_schema_name, + source_databases.name AS source_database_name, + mz_mysql_source_tables.table_name AS upstream_table_name, + mz_mysql_source_tables.schema_name AS upstream_schema_name, + mz_sources.type AS source_type, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_sources + ON mz_tables.source_id = mz_sources.id + JOIN mz_schemas AS source_schemas + ON mz_sources.schema_id = source_schemas.id + JOIN mz_databases AS source_databases + ON source_schemas.database_id = source_databases.id + LEFT JOIN mz_internal.mz_mysql_source_tables + ON mz_tables.id = mz_mysql_source_tables.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN \( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + \) comments + ON mz_tables.id = comments.id` + + q := mockQueryBuilder(b, predicate, "") + ir := mock.NewRows([]string{"id", "name", "schema_name", "database_name", "source_name", "source_schema_name", "source_database_name", "upstream_table_name", "upstream_schema_name", "source_type", "comment", "owner_name", "privileges"}). + AddRow("u1", "table", "schema", "database", "source", "public", "materialize", "upstream_table", "upstream_schema", "mysql", "comment", "materialize", defaultPrivilege) + mock.ExpectQuery(q).WillReturnRows(ir) +} + +func MockSourceTablePostgresScan(mock sqlmock.Sqlmock, predicate string) { + b := ` + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.name AS source_name, + source_schemas.name AS source_schema_name, + source_databases.name AS source_database_name, + mz_postgres_source_tables.table_name AS upstream_table_name, + mz_postgres_source_tables.schema_name AS upstream_schema_name, + mz_sources.type AS source_type, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_sources + ON mz_tables.source_id = mz_sources.id + JOIN mz_schemas AS source_schemas + ON mz_sources.schema_id = source_schemas.id + JOIN mz_databases AS source_databases + ON source_schemas.database_id = source_databases.id + LEFT JOIN mz_internal.mz_postgres_source_tables + ON mz_tables.id = mz_postgres_source_tables.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN \( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + \) comments + ON mz_tables.id = comments.id` + + q := mockQueryBuilder(b, predicate, "") + ir := mock.NewRows([]string{"id", "name", "schema_name", "database_name", "source_name", "source_schema_name", "source_database_name", "upstream_table_name", "upstream_schema_name", "source_type", "comment", "owner_name", "privileges"}). + AddRow("u1", "table", "schema", "database", "source", "public", "materialize", "upstream_table", "upstream_schema", "postgres", "comment", "materialize", defaultPrivilege) + mock.ExpectQuery(q).WillReturnRows(ir) +} + func MockSourceTableScan(mock sqlmock.Sqlmock, predicate string) { b := ` SELECT From 47bb160348d076a8cd614aae348df69e333bcfa0 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Sat, 21 Sep 2024 00:35:03 +0300 Subject: [PATCH 22/46] Add Kafka upstream references --- compose.yaml | 4 +- docs/resources/source_table_kafka.md | 10 +- pkg/materialize/source_table_kafka.go | 69 ++++++++++ pkg/materialize/source_table_kafka_test.go | 12 +- .../acceptance_source_table_kafka_test.go | 14 +- pkg/resources/resource_source_table_kafka.go | 121 ++++++++++++++++-- .../resource_source_table_kafka_test.go | 60 ++++----- pkg/testhelpers/mock_scans.go | 44 +++++++ 8 files changed, 273 insertions(+), 61 deletions(-) diff --git a/compose.yaml b/compose.yaml index f1e6d7d0..fa7ac5b6 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,7 +1,9 @@ services: materialized: - image: materialize/materialized:latest + # TODO: Revise the image tag to the latest stable release after testing + # image: materialize/materialized:latest + image: materialize/materialized:v0.118.0-dev.0--pr.g4c4d1ae4f815f25f8e2e29f59be7d3634a489b7f container_name: materialized command: - '--cluster-replica-sizes={"3xsmall": {"workers": 1, "scale": 1, "credits_per_hour": "1", "is_cc": false}, "2xsmall": {"workers": 1, "scale": 1, "credits_per_hour": "1", "is_cc": false}, "25cc": {"workers": 1, "scale": 1, "credits_per_hour": "1"}, "50cc": {"workers": 1, "scale": 1, "credits_per_hour": "1"}}' diff --git a/docs/resources/source_table_kafka.md b/docs/resources/source_table_kafka.md index 806dc7ba..80288bab 100644 --- a/docs/resources/source_table_kafka.md +++ b/docs/resources/source_table_kafka.md @@ -66,14 +66,14 @@ resource "materialize_source_table_kafka" "kafka_source_table" { ### Required -- `name` (String) The identifier for the source. +- `name` (String) The identifier for the source table. - `source` (Block List, Min: 1, Max: 1) The source this table is created from. (see [below for nested schema](#nestedblock--source)) -- `upstream_name` (String) The name of the Kafka topic in the upstream Kafka cluster. +- `topic` (String) The name of the Kafka topic in the upstream Kafka cluster. ### Optional - `comment` (String) **Public Preview** Comment on an object in the database. -- `database_name` (String) The identifier for the source database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `database_name` (String) The identifier for the source table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `envelope` (Block List, Max: 1) How Materialize should interpret records (e.g. append-only, upsert).. (see [below for nested schema](#nestedblock--envelope)) - `expose_progress` (Block List, Max: 1) The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`. (see [below for nested schema](#nestedblock--expose_progress)) - `format` (Block List, Max: 1) How to decode raw bytes from different formats into data structures Materialize can understand at runtime. (see [below for nested schema](#nestedblock--format)) @@ -90,13 +90,13 @@ resource "materialize_source_table_kafka" "kafka_source_table" { - `key_format` (Block List, Max: 1) Set the key format explicitly. (see [below for nested schema](#nestedblock--key_format)) - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. -- `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. +- `schema_name` (String) The identifier for the source table schema in Materialize. Defaults to `public`. - `value_format` (Block List, Max: 1) Set the value format explicitly. (see [below for nested schema](#nestedblock--value_format)) ### Read-Only - `id` (String) The ID of this resource. -- `qualified_sql_name` (String) The fully qualified name of the source. +- `qualified_sql_name` (String) The fully qualified name of the source table. ### Nested Schema for `source` diff --git a/pkg/materialize/source_table_kafka.go b/pkg/materialize/source_table_kafka.go index f06696d3..aa8c1ff5 100644 --- a/pkg/materialize/source_table_kafka.go +++ b/pkg/materialize/source_table_kafka.go @@ -7,6 +7,75 @@ import ( "github.com/jmoiron/sqlx" ) +type SourceTableKafkaParams struct { + SourceTableParams +} + +var sourceTableKafkaQuery = ` + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.name AS source_name, + source_schemas.name AS source_schema_name, + source_databases.name AS source_database_name, + mz_kafka_source_tables.topic AS upstream_table_name, + mz_sources.type AS source_type, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_sources + ON mz_tables.source_id = mz_sources.id + JOIN mz_schemas AS source_schemas + ON mz_sources.schema_id = source_schemas.id + JOIN mz_databases AS source_databases + ON source_schemas.database_id = source_databases.id + LEFT JOIN mz_internal.mz_kafka_source_tables + ON mz_tables.id = mz_kafka_source_tables.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN ( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + ) comments + ON mz_tables.id = comments.id +` + +func SourceTableKafkaId(conn *sqlx.DB, obj MaterializeObject) (string, error) { + p := map[string]string{ + "mz_tables.name": obj.Name, + "mz_schemas.name": obj.SchemaName, + "mz_databases.name": obj.DatabaseName, + } + q := NewBaseQuery(sourceTableKafkaQuery).QueryPredicate(p) + + var t SourceTableParams + if err := conn.Get(&t, q); err != nil { + return "", err + } + + return t.TableId.String, nil +} + +func ScanSourceTableKafka(conn *sqlx.DB, id string) (SourceTableKafkaParams, error) { + q := NewBaseQuery(sourceTableKafkaQuery).QueryPredicate(map[string]string{"mz_tables.id": id}) + + var params SourceTableKafkaParams + if err := conn.Get(¶ms, q); err != nil { + return params, err + } + + return params, nil +} + type SourceTableKafkaBuilder struct { *SourceTableBuilder includeKey bool diff --git a/pkg/materialize/source_table_kafka_test.go b/pkg/materialize/source_table_kafka_test.go index cbc85468..22f41703 100644 --- a/pkg/materialize/source_table_kafka_test.go +++ b/pkg/materialize/source_table_kafka_test.go @@ -13,7 +13,7 @@ func TestResourceSourceTableKafkaCreate(t *testing.T) { mock.ExpectExec( `CREATE TABLE "database"."schema"."source" FROM SOURCE "database"."schema"."kafka_source" - \(REFERENCE "upstream_table"\) + \(REFERENCE "topic"\) FORMAT JSON INCLUDE KEY AS message_key, HEADERS AS message_headers, PARTITION AS message_partition ENVELOPE UPSERT @@ -23,7 +23,7 @@ func TestResourceSourceTableKafkaCreate(t *testing.T) { o := MaterializeObject{Name: "source", SchemaName: "schema", DatabaseName: "database"} b := NewSourceTableKafkaBuilder(db, o) b.Source(IdentifierSchemaStruct{Name: "kafka_source", DatabaseName: "database", SchemaName: "schema"}) - b.UpstreamName("upstream_table") + b.UpstreamName("topic") b.Format(SourceFormatSpecStruct{Json: true}) b.IncludeKey() b.IncludeKeyAlias("message_key") @@ -45,7 +45,7 @@ func TestResourceSourceTableKafkaCreateWithAvroFormat(t *testing.T) { mock.ExpectExec( `CREATE TABLE "database"."schema"."source" FROM SOURCE "database"."schema"."kafka_source" - \(REFERENCE "upstream_table"\) + \(REFERENCE "topic"\) FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION "database"."schema"."schema_registry" KEY STRATEGY EXTRACT VALUE STRATEGY EXTRACT @@ -56,7 +56,7 @@ func TestResourceSourceTableKafkaCreateWithAvroFormat(t *testing.T) { o := MaterializeObject{Name: "source", SchemaName: "schema", DatabaseName: "database"} b := NewSourceTableKafkaBuilder(db, o) b.Source(IdentifierSchemaStruct{Name: "kafka_source", DatabaseName: "database", SchemaName: "schema"}) - b.UpstreamName("upstream_table") + b.UpstreamName("topic") b.Format(SourceFormatSpecStruct{ Avro: &AvroFormatSpec{ SchemaRegistryConnection: IdentifierSchemaStruct{Name: "schema_registry", DatabaseName: "database", SchemaName: "schema"}, @@ -78,7 +78,7 @@ func TestResourceSourceTableKafkaCreateWithUpsertOptions(t *testing.T) { mock.ExpectExec( `CREATE TABLE "database"."schema"."source" FROM SOURCE "database"."schema"."kafka_source" - \(REFERENCE "upstream_table"\) + \(REFERENCE "topic"\) FORMAT JSON INCLUDE KEY, HEADERS, PARTITION, OFFSET, TIMESTAMP ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS my_error_col\)\) @@ -88,7 +88,7 @@ func TestResourceSourceTableKafkaCreateWithUpsertOptions(t *testing.T) { o := MaterializeObject{Name: "source", SchemaName: "schema", DatabaseName: "database"} b := NewSourceTableKafkaBuilder(db, o) b.Source(IdentifierSchemaStruct{Name: "kafka_source", DatabaseName: "database", SchemaName: "schema"}) - b.UpstreamName("upstream_table") + b.UpstreamName("topic") b.Format(SourceFormatSpecStruct{Json: true}) b.IncludeKey() b.IncludeHeaders() diff --git a/pkg/provider/acceptance_source_table_kafka_test.go b/pkg/provider/acceptance_source_table_kafka_test.go index f0de1d3c..4ddfc6e6 100644 --- a/pkg/provider/acceptance_source_table_kafka_test.go +++ b/pkg/provider/acceptance_source_table_kafka_test.go @@ -25,8 +25,8 @@ func TestAccSourceTableKafka_basic(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "name", nameSpace+"_table_kafka"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "database_name", "materialize"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "schema_name", "public"), - resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "qualified_sql_name", fmt.Sprintf(`"materialize"."public"."%s_table_kafka"`, nameSpace)), - resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "upstream_name", "terraform"), + // resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "qualified_sql_name", fmt.Sprintf(`"materialize"."public"."%s_table_kafka"`, nameSpace)), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "topic", "terraform"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_key", "true"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_key_alias", "message_key"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_headers", "true"), @@ -37,13 +37,11 @@ func TestAccSourceTableKafka_basic(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_offset_alias", "message_offset"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_timestamp", "true"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_timestamp_alias", "message_timestamp"), - resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "format.0.json", "true"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "key_format.0.text", "true"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "value_format.0.json", "true"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "envelope.0.upsert", "true"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "envelope.0.upsert_options.0.value_decoding_errors.0.inline.0.enabled", "true"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "envelope.0.upsert_options.0.value_decoding_errors.0.inline.0.alias", "decoding_error"), - resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "expose_progress.0.name", nameSpace+"_progress"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "ownership_role", "mz_system"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "comment", "This is a test Kafka source table"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "source.#", "1"), @@ -68,7 +66,7 @@ func TestAccSourceTableKafka_update(t *testing.T) { Check: resource.ComposeTestCheckFunc( testAccCheckSourceTableExists("materialize_source_table_kafka.test"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "name", nameSpace+"_table"), - resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "upstream_name", "terraform"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "topic", "terraform"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "ownership_role", "mz_system"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "comment", ""), ), @@ -78,7 +76,7 @@ func TestAccSourceTableKafka_update(t *testing.T) { Check: resource.ComposeTestCheckFunc( testAccCheckSourceTableExists("materialize_source_table_kafka.test"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "name", nameSpace+"_table"), - resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "upstream_name", "terraform"), + resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "topic", "terraform"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "ownership_role", nameSpace+"_role"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test", "comment", "Updated comment"), ), @@ -141,7 +139,7 @@ func testAccSourceTableKafkaBasicResource(nameSpace string) string { name = materialize_source_kafka.test_source_kafka.name } - upstream_name = "terraform" + topic = "terraform" include_key = true include_key_alias = "message_key" include_headers = true @@ -221,7 +219,7 @@ func testAccSourceTableKafkaResource(nameSpace, upstreamName, ownershipRole, com database_name = "materialize" } - upstream_name = "%[2]s" + topic = "%[2]s" ownership_role = "%[3]s" comment = "%[4]s" diff --git a/pkg/resources/resource_source_table_kafka.go b/pkg/resources/resource_source_table_kafka.go index c03d63ab..2ee8c14d 100644 --- a/pkg/resources/resource_source_table_kafka.go +++ b/pkg/resources/resource_source_table_kafka.go @@ -2,6 +2,7 @@ package resources import ( "context" + "database/sql" "log" "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" @@ -12,10 +13,10 @@ import ( ) var sourceTableKafkaSchema = map[string]*schema.Schema{ - "name": ObjectNameSchema("source", true, false), - "schema_name": SchemaNameSchema("source", false), - "database_name": DatabaseNameSchema("source", false), - "qualified_sql_name": QualifiedNameSchema("source"), + "name": ObjectNameSchema("source table", true, false), + "schema_name": SchemaNameSchema("source table", false), + "database_name": DatabaseNameSchema("source table", false), + "qualified_sql_name": QualifiedNameSchema("source table"), "comment": CommentSchema(false), "source": IdentifierSchema(IdentifierSchemaParams{ Elem: "source", @@ -23,7 +24,7 @@ var sourceTableKafkaSchema = map[string]*schema.Schema{ Required: true, ForceNew: true, }), - "upstream_name": { + "topic": { Type: schema.TypeString, Required: true, ForceNew: true, @@ -183,8 +184,8 @@ func SourceTableKafka() *schema.Resource { Description: "A Kafka source describes a Kafka cluster you want Materialize to read data from.", CreateContext: sourceTableKafkaCreate, - ReadContext: sourceTableRead, - UpdateContext: sourceTableUpdate, + ReadContext: sourceTableKafkaRead, + UpdateContext: sourceTableKafkaUpdate, DeleteContext: sourceTableDelete, Importer: &schema.ResourceImporter{ @@ -204,13 +205,13 @@ func sourceTableKafkaCreate(ctx context.Context, d *schema.ResourceData, meta an if err != nil { return diag.FromErr(err) } - o := materialize.MaterializeObject{ObjectType: "SOURCE", Name: sourceName, SchemaName: schemaName, DatabaseName: databaseName} + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: sourceName, SchemaName: schemaName, DatabaseName: databaseName} b := materialize.NewSourceTableKafkaBuilder(metaDb, o) source := materialize.GetIdentifierSchemaStruct(d.Get("source")) b.Source(source) - b.UpstreamName(d.Get("upstream_name").(string)) + b.UpstreamName(d.Get("topic").(string)) if v, ok := d.GetOk("include_key"); ok && v.(bool) { if alias, ok := d.GetOk("include_key_alias"); ok { @@ -305,11 +306,109 @@ func sourceTableKafkaCreate(ctx context.Context, d *schema.ResourceData, meta an } // set id - i, err := materialize.SourceTableId(metaDb, o) + i, err := materialize.SourceTableKafkaId(metaDb, o) if err != nil { return diag.FromErr(err) } d.SetId(utils.TransformIdWithRegion(string(region), i)) - return sourceTableRead(ctx, d, meta) + return sourceTableKafkaRead(ctx, d, meta) +} + +func sourceTableKafkaRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + i := d.Id() + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + t, err := materialize.ScanSourceTableKafka(metaDb, utils.ExtractId(i)) + if err == sql.ErrNoRows { + d.SetId("") + return nil + } else if err != nil { + return diag.FromErr(err) + } + + d.SetId(utils.TransformIdWithRegion(string(region), i)) + + if err := d.Set("name", t.TableName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("schema_name", t.SchemaName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("database_name", t.DatabaseName.String); err != nil { + return diag.FromErr(err) + } + + source := []interface{}{ + map[string]interface{}{ + "name": t.SourceName.String, + "schema_name": t.SourceSchemaName.String, + "database_name": t.SourceDatabaseName.String, + }, + } + if err := d.Set("source", source); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("topic", t.UpstreamName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("ownership_role", t.OwnerName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("comment", t.Comment.String); err != nil { + return diag.FromErr(err) + } + + return nil +} + +func sourceTableKafkaUpdate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + tableName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, _, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} + + if d.HasChange("name") { + oldName, newName := d.GetChange("name") + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: oldName.(string), SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewSourceTableKafkaBuilder(metaDb, o) + if err := b.Rename(newName.(string)); err != nil { + return diag.FromErr(err) + } + } + + if d.HasChange("ownership_role") { + _, newRole := d.GetChange("ownership_role") + b := materialize.NewOwnershipBuilder(metaDb, o) + + if err := b.Alter(newRole.(string)); err != nil { + return diag.FromErr(err) + } + } + + if d.HasChange("comment") { + _, newComment := d.GetChange("comment") + b := materialize.NewCommentBuilder(metaDb, o) + + if err := b.Object(newComment.(string)); err != nil { + return diag.FromErr(err) + } + } + + return sourceTableKafkaRead(ctx, d, meta) } diff --git a/pkg/resources/resource_source_table_kafka_test.go b/pkg/resources/resource_source_table_kafka_test.go index b13e9d7c..c52fc097 100644 --- a/pkg/resources/resource_source_table_kafka_test.go +++ b/pkg/resources/resource_source_table_kafka_test.go @@ -22,7 +22,7 @@ var inSourceTableKafka = map[string]interface{}{ "database_name": "materialize", }, }, - "upstream_name": "upstream_table", + "topic": "topic", "include_key": true, "include_key_alias": "message_key", "include_headers": true, @@ -65,7 +65,7 @@ func TestResourceSourceTableKafkaCreate(t *testing.T) { mock.ExpectExec( `CREATE TABLE "database"."schema"."table" FROM SOURCE "materialize"."public"."kafka_source" - \(REFERENCE "upstream_table"\) + \(REFERENCE "topic"\) FORMAT JSON INCLUDE KEY AS message_key, HEADERS AS message_headers, PARTITION AS message_partition ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS decoding_error\)\);`, @@ -73,11 +73,11 @@ func TestResourceSourceTableKafkaCreate(t *testing.T) { // Query Id ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` - testhelpers.MockSourceTableScan(mock, ip) + testhelpers.MockSourceTableKafkaScan(mock, ip) // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableKafkaScan(mock, pp) if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { t.Fatal(err) @@ -94,9 +94,9 @@ func TestResourceSourceTableKafkaRead(t *testing.T) { testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableKafkaScan(mock, pp) - if err := sourceTableRead(context.TODO(), d, db); err != nil { + if err := sourceTableKafkaRead(context.TODO(), d, db); err != nil { t.Fatal(err) } @@ -118,9 +118,9 @@ func TestResourceSourceTableKafkaUpdate(t *testing.T) { // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableKafkaScan(mock, pp) - if err := sourceTableUpdate(context.TODO(), d, db); err != nil { + if err := sourceTableKafkaUpdate(context.TODO(), d, db); err != nil { t.Fatal(err) } }) @@ -154,7 +154,7 @@ func TestResourceSourceTableKafkaCreateWithAvroFormat(t *testing.T) { "database_name": "materialize", }, }, - "upstream_name": "upstream_table", + "topic": "topic", "format": []interface{}{ map[string]interface{}{ "avro": []interface{}{ @@ -184,18 +184,18 @@ func TestResourceSourceTableKafkaCreateWithAvroFormat(t *testing.T) { mock.ExpectExec( `CREATE TABLE "database"."schema"."table_avro" FROM SOURCE "materialize"."public"."kafka_source" - \(REFERENCE "upstream_table"\) + \(REFERENCE "topic"\) FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION "materialize"."public"."sr_conn" ENVELOPE DEBEZIUM;`, ).WillReturnResult(sqlmock.NewResult(1, 1)) // Query Id ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table_avro'` - testhelpers.MockSourceTableScan(mock, ip) + testhelpers.MockSourceTableKafkaScan(mock, ip) // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableKafkaScan(mock, pp) if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { t.Fatal(err) @@ -224,7 +224,7 @@ func TestResourceSourceTableKafkaCreateIncludeTrueNoAlias(t *testing.T) { mock.ExpectExec( `CREATE TABLE "database"."schema"."table" FROM SOURCE "materialize"."public"."kafka_source" - \(REFERENCE "upstream_table"\) + \(REFERENCE "topic"\) FORMAT JSON INCLUDE KEY, HEADERS, PARTITION, OFFSET, TIMESTAMP ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS decoding_error\)\);`, @@ -232,11 +232,11 @@ func TestResourceSourceTableKafkaCreateIncludeTrueNoAlias(t *testing.T) { // Query Id ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` - testhelpers.MockSourceTableScan(mock, ip) + testhelpers.MockSourceTableKafkaScan(mock, ip) // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableKafkaScan(mock, pp) if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { t.Fatal(err) @@ -262,18 +262,18 @@ func TestResourceSourceTableKafkaCreateIncludeFalseWithAlias(t *testing.T) { mock.ExpectExec( `CREATE TABLE "database"."schema"."table" FROM SOURCE "materialize"."public"."kafka_source" - \(REFERENCE "upstream_table"\) + \(REFERENCE "topic"\) FORMAT JSON ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS decoding_error\)\);`, ).WillReturnResult(sqlmock.NewResult(1, 1)) // Query Id ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table'` - testhelpers.MockSourceTableScan(mock, ip) + testhelpers.MockSourceTableKafkaScan(mock, ip) // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableKafkaScan(mock, pp) if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { t.Fatal(err) @@ -294,7 +294,7 @@ func TestResourceSourceTableKafkaCreateWithCSVFormat(t *testing.T) { "database_name": "materialize", }, }, - "upstream_name": "upstream_table", + "topic": "topic", "format": []interface{}{ map[string]interface{}{ "csv": []interface{}{ @@ -314,17 +314,17 @@ func TestResourceSourceTableKafkaCreateWithCSVFormat(t *testing.T) { mock.ExpectExec( `CREATE TABLE "database"."schema"."table_csv" FROM SOURCE "materialize"."public"."kafka_source" - \(REFERENCE "upstream_table"\) + \(REFERENCE "topic"\) FORMAT CSV WITH HEADER \( column1, column2, column3 \) DELIMITER ',';`, ).WillReturnResult(sqlmock.NewResult(1, 1)) // Query Id ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table_csv'` - testhelpers.MockSourceTableScan(mock, ip) + testhelpers.MockSourceTableKafkaScan(mock, ip) // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableKafkaScan(mock, pp) if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { t.Fatal(err) @@ -345,7 +345,7 @@ func TestResourceSourceTableKafkaCreateWithKeyAndValueFormat(t *testing.T) { "database_name": "materialize", }, }, - "upstream_name": "upstream_table", + "topic": "topic", "key_format": []interface{}{ map[string]interface{}{ "json": true, @@ -375,18 +375,18 @@ func TestResourceSourceTableKafkaCreateWithKeyAndValueFormat(t *testing.T) { mock.ExpectExec( `CREATE TABLE "database"."schema"."table_key_value" FROM SOURCE "materialize"."public"."kafka_source" - \(REFERENCE "upstream_table"\) + \(REFERENCE "topic"\) KEY FORMAT JSON VALUE FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION "materialize"."public"."sr_conn";`, ).WillReturnResult(sqlmock.NewResult(1, 1)) // Query Id ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table_key_value'` - testhelpers.MockSourceTableScan(mock, ip) + testhelpers.MockSourceTableKafkaScan(mock, ip) // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableKafkaScan(mock, pp) if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { t.Fatal(err) @@ -407,7 +407,7 @@ func TestResourceSourceTableKafkaCreateWithProtobufFormat(t *testing.T) { "database_name": "materialize", }, }, - "upstream_name": "upstream_table", + "topic": "topic", "format": []interface{}{ map[string]interface{}{ "protobuf": []interface{}{ @@ -438,18 +438,18 @@ func TestResourceSourceTableKafkaCreateWithProtobufFormat(t *testing.T) { mock.ExpectExec( `CREATE TABLE "database"."schema"."table_protobuf" FROM SOURCE "materialize"."public"."kafka_source" - \(REFERENCE "upstream_table"\) + \(REFERENCE "topic"\) FORMAT PROTOBUF MESSAGE 'MyMessage' USING CONFLUENT SCHEMA REGISTRY CONNECTION "materialize"."public"."sr_conn" ENVELOPE NONE;`, ).WillReturnResult(sqlmock.NewResult(1, 1)) // Query Id ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'table_protobuf'` - testhelpers.MockSourceTableScan(mock, ip) + testhelpers.MockSourceTableKafkaScan(mock, ip) // Query Params pp := `WHERE mz_tables.id = 'u1'` - testhelpers.MockSourceTableScan(mock, pp) + testhelpers.MockSourceTableKafkaScan(mock, pp) if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { t.Fatal(err) diff --git a/pkg/testhelpers/mock_scans.go b/pkg/testhelpers/mock_scans.go index e64b412c..06c31572 100644 --- a/pkg/testhelpers/mock_scans.go +++ b/pkg/testhelpers/mock_scans.go @@ -865,6 +865,50 @@ func MockSourceTablePostgresScan(mock sqlmock.Sqlmock, predicate string) { mock.ExpectQuery(q).WillReturnRows(ir) } +func MockSourceTableKafkaScan(mock sqlmock.Sqlmock, predicate string) { + b := ` + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.name AS source_name, + source_schemas.name AS source_schema_name, + source_databases.name AS source_database_name, + mz_kafka_source_tables.topic AS upstream_table_name, + mz_sources.type AS source_type, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_sources + ON mz_tables.source_id = mz_sources.id + JOIN mz_schemas AS source_schemas + ON mz_sources.schema_id = source_schemas.id + JOIN mz_databases AS source_databases + ON source_schemas.database_id = source_databases.id + LEFT JOIN mz_internal.mz_kafka_source_tables + ON mz_tables.id = mz_kafka_source_tables.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN \( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + \) comments + ON mz_tables.id = comments.id` + + q := mockQueryBuilder(b, predicate, "") + ir := mock.NewRows([]string{"id", "name", "schema_name", "database_name", "source_name", "source_schema_name", "source_database_name", "upstream_table_name", "source_type", "comment", "owner_name", "privileges"}). + AddRow("u1", "table", "schema", "database", "source", "public", "materialize", "topic", "kafka", "comment", "materialize", defaultPrivilege) + mock.ExpectQuery(q).WillReturnRows(ir) +} + func MockSourceTableScan(mock sqlmock.Sqlmock, predicate string) { b := ` SELECT From 3c3969463f4d5220450a2922d8e79fce5a4dacf5 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 24 Sep 2024 12:10:26 +0300 Subject: [PATCH 23/46] Add integration tests --- integration/source.tf | 176 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) diff --git a/integration/source.tf b/integration/source.tf index 4ff3e287..4d1bf9c0 100644 --- a/integration/source.tf +++ b/integration/source.tf @@ -70,6 +70,21 @@ resource "materialize_source_load_generator" "load_generator_auction" { } } +# Create source table from Auction load generator source +resource "materialize_source_table_load_generator" "load_generator_auction_table" { + name = "load_gen_auction_table" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_load_generator.load_generator_auction.name + } + + comment = "source table load generator comment" + + upstream_name = "bids" +} + resource "materialize_source_load_generator" "load_generator_marketing" { name = "load_gen_marketing" schema_name = materialize_schema.schema.name @@ -82,6 +97,21 @@ resource "materialize_source_load_generator" "load_generator_marketing" { } } +# Create source table from Marketing load generator source +resource "materialize_source_table_load_generator" "load_generator_marketing_table" { + name = "load_gen_marketing_table" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_load_generator.load_generator_marketing.name + } + + comment = "source table load generator comment" + + upstream_name = "leads" +} + resource "materialize_source_load_generator" "load_generator_tpch" { name = "load_gen_tpch" schema_name = materialize_schema.schema.name @@ -144,6 +174,26 @@ resource "materialize_source_postgres" "example_source_postgres" { } } +# Create source table from Postgres source +resource "materialize_source_table_postgres" "source_table_postgres" { + name = "source_table2_postgres" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_postgres.example_source_postgres.name + schema_name = materialize_source_postgres.example_source_postgres.schema_name + database_name = materialize_source_postgres.example_source_postgres.database_name + } + + upstream_name = "table2" + upstream_schema_name = "public" + + text_columns = [ + "updated_at" + ] +} + resource "materialize_source_kafka" "example_source_kafka_format_text" { name = "source_kafka_text" comment = "source kafka comment" @@ -168,6 +218,32 @@ resource "materialize_source_kafka" "example_source_kafka_format_text" { depends_on = [materialize_sink_kafka.sink_kafka] } +# Create source table from Kafka source +resource "materialize_source_table_kafka" "source_table_kafka" { + name = "source_table_kafka" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_kafka.example_source_kafka_format_text.name + schema_name = materialize_source_kafka.example_source_kafka_format_text.schema_name + database_name = materialize_source_kafka.example_source_kafka_format_text.database_name + } + + topic = "topic1" + key_format_text = true + value_format_text = true + envelope_none = true + include_timestamp = true + include_offset = true + include_partition = true + include_key = true + include_key_alias = "key_alias" + include_offset_alias = "offset_alias" + include_partition_alias = "partition_alias" + include_timestamp_alias = "timestamp_alias" +} + resource "materialize_source_kafka" "example_source_kafka_format_bytes" { name = "source_kafka_bytes" cluster_name = materialize_cluster.cluster_source.name @@ -185,6 +261,27 @@ resource "materialize_source_kafka" "example_source_kafka_format_bytes" { depends_on = [materialize_sink_kafka.sink_kafka] } +# Create source table from Kafka source with bytes format +resource "materialize_source_table_kafka" "source_table_kafka_bytes" { + name = "source_table_kafka_bytes" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_kafka.example_source_kafka_format_bytes.name + schema_name = materialize_source_kafka.example_source_kafka_format_bytes.schema_name + database_name = materialize_source_kafka.example_source_kafka_format_bytes.database_name + } + + topic = "topic1" + + format { + bytes = true + } + + depends_on = [materialize_sink_kafka.sink_kafka] +} + resource "materialize_source_kafka" "example_source_kafka_format_avro" { name = "source_kafka_avro" cluster_name = materialize_cluster.cluster_source.name @@ -211,6 +308,33 @@ resource "materialize_source_kafka" "example_source_kafka_format_avro" { depends_on = [materialize_sink_kafka.sink_kafka] } +# Source table from Kafka source with Avro format +resource "materialize_source_table_kafka" "source_table_kafka_avro" { + name = "source_table_kafka_avro" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_kafka.example_source_kafka_format_avro.name + schema_name = materialize_source_kafka.example_source_kafka_format_avro.schema_name + database_name = materialize_source_kafka.example_source_kafka_format_avro.database_name + } + + topic = "topic1" + + format { + avro { + schema_registry_connection { + name = materialize_connection_confluent_schema_registry.schema_registry.name + schema_name = materialize_connection_confluent_schema_registry.schema_registry.schema_name + database_name = materialize_connection_confluent_schema_registry.schema_registry.database_name + } + } + } + + depends_on = [materialize_sink_kafka.sink_kafka] +} + resource "materialize_source_webhook" "example_webhook_source" { name = "example_webhook_source" comment = "source webhook comment" @@ -271,6 +395,22 @@ resource "materialize_source_mysql" "test" { } } +# Create source table from MySQL source +resource "materialize_source_table_mysql" "source_table_mysql" { + name = "source_table1_mysql" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_mysql.test.name + schema_name = materialize_source_mysql.test.schema_name + database_name = materialize_source_mysql.test.database_name + } + + upstream_name = "mysql_table1" + upstream_schema_name = "shop" +} + resource "materialize_source_grant" "source_grant_select" { role_name = materialize_role.role_1.name privilege = "SELECT" @@ -317,6 +457,42 @@ resource "materialize_source_kafka" "kafka_upsert_options_source" { include_key_alias = "key_alias" } +# Create source table from Kafka source with upsert options +resource "materialize_source_table_kafka" "source_table_kafka_upsert_options" { + name = "source_table_kafka_upsert_options" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_kafka.kafka_upsert_options_source.name + schema_name = materialize_source_kafka.kafka_upsert_options_source.schema_name + database_name = materialize_source_kafka.kafka_upsert_options_source.database_name + } + + topic = "topic1" + + key_format_text = true + value_format_text = true + + envelope_upsert = true + + upsert_options { + value_decoding_errors { + inline { + enabled = true + alias = "my_error_col" + } + } + } + + include_timestamp_alias = "timestamp_alias" + include_offset = true + include_offset_alias = "offset_alias" + include_partition = true + include_partition_alias = "partition_alias" + include_key_alias = "key_alias" +} + output "qualified_load_generator" { value = materialize_source_load_generator.load_generator.qualified_sql_name } From 76e22c23cda85d5154c2b53e7f70754ea885c5c4 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 24 Sep 2024 12:22:31 +0300 Subject: [PATCH 24/46] Fix failing test --- integration/source.tf | 60 ++++++++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/integration/source.tf b/integration/source.tf index 4d1bf9c0..1975da7e 100644 --- a/integration/source.tf +++ b/integration/source.tf @@ -77,7 +77,9 @@ resource "materialize_source_table_load_generator" "load_generator_auction_table database_name = "materialize" source { - name = materialize_source_load_generator.load_generator_auction.name + name = materialize_source_load_generator.load_generator_auction.name + schema_name = materialize_source_load_generator.load_generator_auction.schema_name + database_name = materialize_source_load_generator.load_generator_auction.database_name } comment = "source table load generator comment" @@ -104,7 +106,9 @@ resource "materialize_source_table_load_generator" "load_generator_marketing_tab database_name = "materialize" source { - name = materialize_source_load_generator.load_generator_marketing.name + name = materialize_source_load_generator.load_generator_marketing.name + schema_name = materialize_source_load_generator.load_generator_marketing.schema_name + database_name = materialize_source_load_generator.load_generator_marketing.database_name } comment = "source table load generator comment" @@ -230,18 +234,26 @@ resource "materialize_source_table_kafka" "source_table_kafka" { database_name = materialize_source_kafka.example_source_kafka_format_text.database_name } - topic = "topic1" - key_format_text = true - value_format_text = true - envelope_none = true - include_timestamp = true - include_offset = true - include_partition = true + topic = "topic1" + + key_format { + text = true + } + value_format { + json = true + } + include_key = true - include_key_alias = "key_alias" - include_offset_alias = "offset_alias" - include_partition_alias = "partition_alias" - include_timestamp_alias = "timestamp_alias" + include_key_alias = "message_key" + include_headers = true + include_headers_alias = "message_headers" + include_partition = true + include_partition_alias = "message_partition" + include_offset = true + include_offset_alias = "message_offset" + include_timestamp = true + include_timestamp_alias = "message_timestamp" + } resource "materialize_source_kafka" "example_source_kafka_format_bytes" { @@ -471,16 +483,22 @@ resource "materialize_source_table_kafka" "source_table_kafka_upsert_options" { topic = "topic1" - key_format_text = true - value_format_text = true + key_format { + text = true + } + value_format { + text = true + } - envelope_upsert = true - upsert_options { - value_decoding_errors { - inline { - enabled = true - alias = "my_error_col" + envelope { + upsert = true + upsert_options { + value_decoding_errors { + inline { + enabled = true + alias = "decoding_error" + } } } } From 14995ed757062968f772aae5fd0460d3a27ccbaa Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 24 Sep 2024 13:02:27 +0300 Subject: [PATCH 25/46] Extend data source to include upstream names --- .../datasource_source_table_test.go | 5 +- pkg/materialize/source_table.go | 75 ++++++++++-------- pkg/testhelpers/mock_scans.go | 77 +++++++++++-------- 3 files changed, 89 insertions(+), 68 deletions(-) diff --git a/pkg/datasources/datasource_source_table_test.go b/pkg/datasources/datasource_source_table_test.go index 7b084a08..59413f73 100644 --- a/pkg/datasources/datasource_source_table_test.go +++ b/pkg/datasources/datasource_source_table_test.go @@ -40,9 +40,8 @@ func TestSourceTableDatasource(t *testing.T) { r.Equal("schema", table["schema_name"]) r.Equal("database", table["database_name"]) r.Equal("KAFKA", table["source_type"]) - // TODO: Update once upstream_name and upstream_schema_name are supported - r.Equal("", table["upstream_name"]) - r.Equal("", table["upstream_schema_name"]) + r.Equal("table", table["upstream_name"]) + r.Equal("schema", table["upstream_schema_name"]) r.Equal("comment", table["comment"]) r.Equal("materialize", table["owner_name"]) diff --git a/pkg/materialize/source_table.go b/pkg/materialize/source_table.go index 1b6fdc34..e73ba519 100644 --- a/pkg/materialize/source_table.go +++ b/pkg/materialize/source_table.go @@ -27,38 +27,49 @@ type SourceTableParams struct { } var sourceTableQuery = NewBaseQuery(` - SELECT - mz_tables.id, - mz_tables.name, - mz_schemas.name AS schema_name, - mz_databases.name AS database_name, - mz_sources.name AS source_name, - source_schemas.name AS source_schema_name, - source_databases.name AS source_database_name, - mz_sources.type AS source_type, - comments.comment AS comment, - mz_roles.name AS owner_name, - mz_tables.privileges - FROM mz_tables - JOIN mz_schemas - ON mz_tables.schema_id = mz_schemas.id - JOIN mz_databases - ON mz_schemas.database_id = mz_databases.id - JOIN mz_sources - ON mz_tables.source_id = mz_sources.id - JOIN mz_schemas AS source_schemas - ON mz_sources.schema_id = source_schemas.id - JOIN mz_databases AS source_databases - ON source_schemas.database_id = source_databases.id - JOIN mz_roles - ON mz_tables.owner_id = mz_roles.id - LEFT JOIN ( - SELECT id, comment - FROM mz_internal.mz_comments - WHERE object_type = 'table' - AND object_sub_id IS NULL - ) comments - ON mz_tables.id = comments.id + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.name AS source_name, + source_schemas.name AS source_schema_name, + source_databases.name AS source_database_name, + mz_sources.type AS source_type, + COALESCE(mz_kafka_source_tables.topic, + mz_mysql_source_tables.table_name, + mz_postgres_source_tables.table_name) AS upstream_table_name, + COALESCE(mz_mysql_source_tables.schema_name, + mz_postgres_source_tables.schema_name) AS upstream_schema_name, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_sources + ON mz_tables.source_id = mz_sources.id + JOIN mz_schemas AS source_schemas + ON mz_sources.schema_id = source_schemas.id + JOIN mz_databases AS source_databases + ON source_schemas.database_id = source_databases.id + LEFT JOIN mz_internal.mz_kafka_source_tables + ON mz_tables.id = mz_kafka_source_tables.id + LEFT JOIN mz_internal.mz_mysql_source_tables + ON mz_tables.id = mz_mysql_source_tables.id + LEFT JOIN mz_internal.mz_postgres_source_tables + ON mz_tables.id = mz_postgres_source_tables.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN ( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + ) comments + ON mz_tables.id = comments.id `) func SourceTableId(conn *sqlx.DB, obj MaterializeObject) (string, error) { diff --git a/pkg/testhelpers/mock_scans.go b/pkg/testhelpers/mock_scans.go index 06c31572..ff1ecd23 100644 --- a/pkg/testhelpers/mock_scans.go +++ b/pkg/testhelpers/mock_scans.go @@ -912,41 +912,52 @@ func MockSourceTableKafkaScan(mock sqlmock.Sqlmock, predicate string) { func MockSourceTableScan(mock sqlmock.Sqlmock, predicate string) { b := ` SELECT - mz_tables.id, - mz_tables.name, - mz_schemas.name AS schema_name, - mz_databases.name AS database_name, - mz_sources.name AS source_name, - source_schemas.name AS source_schema_name, - source_databases.name AS source_database_name, - mz_sources.type AS source_type, - comments.comment AS comment, - mz_roles.name AS owner_name, - mz_tables.privileges - FROM mz_tables - JOIN mz_schemas - ON mz_tables.schema_id = mz_schemas.id - JOIN mz_databases - ON mz_schemas.database_id = mz_databases.id - JOIN mz_sources - ON mz_tables.source_id = mz_sources.id - JOIN mz_schemas AS source_schemas - ON mz_sources.schema_id = source_schemas.id - JOIN mz_databases AS source_databases - ON source_schemas.database_id = source_databases.id - JOIN mz_roles - ON mz_tables.owner_id = mz_roles.id - LEFT JOIN \( - SELECT id, comment - FROM mz_internal.mz_comments - WHERE object_type = 'table' - AND object_sub_id IS NULL - \) comments - ON mz_tables.id = comments.id` + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.name AS source_name, + source_schemas.name AS source_schema_name, + source_databases.name AS source_database_name, + mz_sources.type AS source_type, + COALESCE\(mz_kafka_source_tables.topic, + mz_mysql_source_tables.table_name, + mz_postgres_source_tables.table_name\) AS upstream_table_name, + COALESCE\(mz_mysql_source_tables.schema_name, + mz_postgres_source_tables.schema_name\) AS upstream_schema_name, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_sources + ON mz_tables.source_id = mz_sources.id + JOIN mz_schemas AS source_schemas + ON mz_sources.schema_id = source_schemas.id + JOIN mz_databases AS source_databases + ON source_schemas.database_id = source_databases.id + LEFT JOIN mz_internal.mz_kafka_source_tables + ON mz_tables.id = mz_kafka_source_tables.id + LEFT JOIN mz_internal.mz_mysql_source_tables + ON mz_tables.id = mz_mysql_source_tables.id + LEFT JOIN mz_internal.mz_postgres_source_tables + ON mz_tables.id = mz_postgres_source_tables.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN \( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + \) comments + ON mz_tables.id = comments.id` q := mockQueryBuilder(b, predicate, "") - ir := mock.NewRows([]string{"id", "name", "schema_name", "database_name", "source_name", "source_schema_name", "source_database_name", "source_type", "comment", "owner_name", "privileges"}). - AddRow("u1", "table", "schema", "database", "source", "public", "materialize", "KAFKA", "comment", "materialize", defaultPrivilege) + ir := mock.NewRows([]string{"id", "name", "schema_name", "database_name", "source_name", "source_schema_name", "source_database_name", "upstream_table_name", "upstream_schema_name", "source_type", "comment", "owner_name", "privileges"}). + AddRow("u1", "table", "schema", "database", "source", "public", "materialize", "table", "schema", "KAFKA", "comment", "materialize", defaultPrivilege) mock.ExpectQuery(q).WillReturnRows(ir) } From acf11904841dcafe094f8e5e6372619201f20483 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 27 Sep 2024 14:04:05 +0300 Subject: [PATCH 26/46] Small updates --- docs/resources/source_table_kafka.md | 4 ++-- .../resource.tf | 2 +- pkg/datasources/datasource_table.go | 20 +++++++++++-------- pkg/resources/resource_source_table.go | 11 ---------- pkg/resources/resource_source_table_kafka.go | 2 +- .../resource_source_table_postgres.go | 2 -- 6 files changed, 16 insertions(+), 25 deletions(-) diff --git a/docs/resources/source_table_kafka.md b/docs/resources/source_table_kafka.md index 80288bab..e25ddc84 100644 --- a/docs/resources/source_table_kafka.md +++ b/docs/resources/source_table_kafka.md @@ -24,7 +24,7 @@ resource "materialize_source_table_kafka" "kafka_source_table" { database_name = materialize_source_kafka.test_source_kafka.database_name } - upstream_name = "terraform" # The kafka source topic name + topic = "terraform" include_key = true include_key_alias = "message_key" include_headers = true @@ -68,7 +68,7 @@ resource "materialize_source_table_kafka" "kafka_source_table" { - `name` (String) The identifier for the source table. - `source` (Block List, Min: 1, Max: 1) The source this table is created from. (see [below for nested schema](#nestedblock--source)) -- `topic` (String) The name of the Kafka topic in the upstream Kafka cluster. +- `topic` (String) The name of the Kafka topic in the Kafka cluster. ### Optional diff --git a/examples/resources/materialize_source_table_kafka/resource.tf b/examples/resources/materialize_source_table_kafka/resource.tf index 2aede69c..ec2ceffb 100644 --- a/examples/resources/materialize_source_table_kafka/resource.tf +++ b/examples/resources/materialize_source_table_kafka/resource.tf @@ -9,7 +9,7 @@ resource "materialize_source_table_kafka" "kafka_source_table" { database_name = materialize_source_kafka.test_source_kafka.database_name } - upstream_name = "terraform" # The kafka source topic name + topic = "terraform" include_key = true include_key_alias = "message_key" include_headers = true diff --git a/pkg/datasources/datasource_table.go b/pkg/datasources/datasource_table.go index 699ab894..e2ffddf8 100644 --- a/pkg/datasources/datasource_table.go +++ b/pkg/datasources/datasource_table.go @@ -32,20 +32,24 @@ func Table() *schema.Resource { Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "id": { - Type: schema.TypeString, - Computed: true, + Description: "The unique identifier for the table", + Type: schema.TypeString, + Computed: true, }, "name": { - Type: schema.TypeString, - Computed: true, + Description: "The name of the table", + Type: schema.TypeString, + Computed: true, }, "schema_name": { - Type: schema.TypeString, - Computed: true, + Description: "The schema of the table", + Type: schema.TypeString, + Computed: true, }, "database_name": { - Type: schema.TypeString, - Computed: true, + Description: "The database of the table", + Type: schema.TypeString, + Computed: true, }, }, }, diff --git a/pkg/resources/resource_source_table.go b/pkg/resources/resource_source_table.go index 9c42ccf7..91788358 100644 --- a/pkg/resources/resource_source_table.go +++ b/pkg/resources/resource_source_table.go @@ -52,15 +52,6 @@ func sourceTableRead(ctx context.Context, d *schema.ResourceData, meta interface return diag.FromErr(err) } - // TODO: Set the upstream_name and upstream_schema_name once supported on the Materialize side - // if err := d.Set("upstream_name", t.UpstreamName.String); err != nil { - // return diag.FromErr(err) - // } - - // if err := d.Set("upstream_schema_name", t.UpstreamSchemaName.String); err != nil { - // return diag.FromErr(err) - // } - if err := d.Set("ownership_role", t.OwnerName.String); err != nil { return diag.FromErr(err) } @@ -93,8 +84,6 @@ func sourceTableUpdate(ctx context.Context, d *schema.ResourceData, meta any) di } } - // TODO: Handle source and text_columns changes once supported on the Materialize side - if d.HasChange("ownership_role") { _, newRole := d.GetChange("ownership_role") b := materialize.NewOwnershipBuilder(metaDb, o) diff --git a/pkg/resources/resource_source_table_kafka.go b/pkg/resources/resource_source_table_kafka.go index 2ee8c14d..23ea5ed2 100644 --- a/pkg/resources/resource_source_table_kafka.go +++ b/pkg/resources/resource_source_table_kafka.go @@ -28,7 +28,7 @@ var sourceTableKafkaSchema = map[string]*schema.Schema{ Type: schema.TypeString, Required: true, ForceNew: true, - Description: "The name of the Kafka topic in the upstream Kafka cluster.", + Description: "The name of the Kafka topic in the Kafka cluster.", }, "include_key": { Description: "Include a column containing the Kafka message key.", diff --git a/pkg/resources/resource_source_table_postgres.go b/pkg/resources/resource_source_table_postgres.go index 77d9ff07..47079777 100644 --- a/pkg/resources/resource_source_table_postgres.go +++ b/pkg/resources/resource_source_table_postgres.go @@ -146,8 +146,6 @@ func sourceTablePostgresUpdate(ctx context.Context, d *schema.ResourceData, meta } } - // TODO: Handle source and text_columns changes once supported on the Materialize side - if d.HasChange("ownership_role") { _, newRole := d.GetChange("ownership_role") b := materialize.NewOwnershipBuilder(metaDb, o) From 82c03049c138d7a953758eec42998ae7e939085a Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 30 Sep 2024 10:06:16 +0300 Subject: [PATCH 27/46] Switch back to latest image --- compose.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compose.yaml b/compose.yaml index fa7ac5b6..f1e6d7d0 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,9 +1,7 @@ services: materialized: - # TODO: Revise the image tag to the latest stable release after testing - # image: materialize/materialized:latest - image: materialize/materialized:v0.118.0-dev.0--pr.g4c4d1ae4f815f25f8e2e29f59be7d3634a489b7f + image: materialize/materialized:latest container_name: materialized command: - '--cluster-replica-sizes={"3xsmall": {"workers": 1, "scale": 1, "credits_per_hour": "1", "is_cc": false}, "2xsmall": {"workers": 1, "scale": 1, "credits_per_hour": "1", "is_cc": false}, "25cc": {"workers": 1, "scale": 1, "credits_per_hour": "1"}, "50cc": {"workers": 1, "scale": 1, "credits_per_hour": "1"}}' From c87e98481bb250c26b7ae12c0f566d61b1f568da Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Sun, 6 Oct 2024 14:30:52 +0300 Subject: [PATCH 28/46] FromAsCasing: 'as' and 'FROM' keywords' casing do not match --- mocks/cloud/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mocks/cloud/Dockerfile b/mocks/cloud/Dockerfile index adbd4af3..53a2198b 100644 --- a/mocks/cloud/Dockerfile +++ b/mocks/cloud/Dockerfile @@ -1,5 +1,5 @@ # Start from the official Golang base image -FROM golang:1.22 as builder +FROM golang:1.22 AS builder # Set the Current Working Directory inside the container WORKDIR /app From 48a6d314238aba9383b5a0998fda7b553525a692 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Sun, 6 Oct 2024 14:55:38 +0300 Subject: [PATCH 29/46] Add source reference data source --- docs/data-sources/source_reference.md | 43 +++++++ .../datasource_source_reference.go | 116 ++++++++++++++++++ pkg/materialize/source_reference.go | 76 ++++++++++++ pkg/provider/provider.go | 1 + 4 files changed, 236 insertions(+) create mode 100644 docs/data-sources/source_reference.md create mode 100644 pkg/datasources/datasource_source_reference.go create mode 100644 pkg/materialize/source_reference.go diff --git a/docs/data-sources/source_reference.md b/docs/data-sources/source_reference.md new file mode 100644 index 00000000..59883eff --- /dev/null +++ b/docs/data-sources/source_reference.md @@ -0,0 +1,43 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "materialize_source_reference Data Source - terraform-provider-materialize" +subcategory: "" +description: |- + +--- + +# materialize_source_reference (Data Source) + + + + + + +## Schema + +### Required + +- `source_id` (String) The ID of the source to get references for + +### Optional + +- `region` (String) The region in which the resource is located. + +### Read-Only + +- `id` (String) The ID of this resource. +- `references` (List of Object) The source references (see [below for nested schema](#nestedatt--references)) + + +### Nested Schema for `references` + +Read-Only: + +- `columns` (List of String) +- `name` (String) +- `namespace` (String) +- `source_database_name` (String) +- `source_name` (String) +- `source_schema_name` (String) +- `source_type` (String) +- `updated_at` (String) diff --git a/pkg/datasources/datasource_source_reference.go b/pkg/datasources/datasource_source_reference.go new file mode 100644 index 00000000..1f4c8652 --- /dev/null +++ b/pkg/datasources/datasource_source_reference.go @@ -0,0 +1,116 @@ +package datasources + +import ( + "context" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" +) + +func SourceReference() *schema.Resource { + return &schema.Resource{ + ReadContext: sourceReferenceRead, + Schema: map[string]*schema.Schema{ + "source_id": { + Type: schema.TypeString, + Required: true, + Description: "The ID of the source to get references for", + }, + "references": { + Type: schema.TypeList, + Computed: true, + Description: "The source references", + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "namespace": { + Type: schema.TypeString, + Computed: true, + Description: "The namespace of the reference", + }, + "name": { + Type: schema.TypeString, + Computed: true, + Description: "The name of the reference", + }, + "updated_at": { + Type: schema.TypeString, + Computed: true, + Description: "The last update timestamp of the reference", + }, + "columns": { + Type: schema.TypeList, + Computed: true, + Description: "The columns of the reference", + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + "source_name": { + Type: schema.TypeString, + Computed: true, + Description: "The name of the source", + }, + "source_schema_name": { + Type: schema.TypeString, + Computed: true, + Description: "The schema name of the source", + }, + "source_database_name": { + Type: schema.TypeString, + Computed: true, + Description: "The database name of the source", + }, + "source_type": { + Type: schema.TypeString, + Computed: true, + Description: "The type of the source", + }, + }, + }, + }, + "region": RegionSchema(), + }, + } +} + +func sourceReferenceRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + sourceID := d.Get("source_id").(string) + + var diags diag.Diagnostics + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + sourceReference, err := materialize.ListSourceReferences(metaDb, sourceID) + if err != nil { + return diag.FromErr(err) + } + + referenceFormats := []map[string]interface{}{} + for _, sr := range sourceReference { + referenceMap := map[string]interface{}{ + "namespace": sr.Namespace.String, + "name": sr.Name.String, + "updated_at": sr.UpdatedAt.Time.String(), + "columns": sr.Columns, + "source_name": sr.SourceName.String, + "source_schema_name": sr.SourceSchemaName.String, + "source_database_name": sr.SourceDBName.String, + "source_type": sr.SourceType.String, + } + referenceFormats = append(referenceFormats, referenceMap) + } + + if err := d.Set("references", referenceFormats); err != nil { + return diag.FromErr(err) + } + + d.SetId(utils.TransformIdWithRegion(string(region), sourceID)) + + return diags +} diff --git a/pkg/materialize/source_reference.go b/pkg/materialize/source_reference.go new file mode 100644 index 00000000..eb934f7a --- /dev/null +++ b/pkg/materialize/source_reference.go @@ -0,0 +1,76 @@ +package materialize + +import ( + "database/sql" + + "github.com/jmoiron/sqlx" + "github.com/lib/pq" +) + +type SourceReferenceParams struct { + SourceId sql.NullString `db:"source_id"` + Namespace sql.NullString `db:"namespace"` + Name sql.NullString `db:"name"` + UpdatedAt sql.NullTime `db:"updated_at"` + Columns pq.StringArray `db:"columns"` + SourceName sql.NullString `db:"source_name"` + SourceSchemaName sql.NullString `db:"source_schema_name"` + SourceDBName sql.NullString `db:"source_database_name"` + SourceType sql.NullString `db:"source_type"` +} + +var sourceReferenceQuery = NewBaseQuery(` + SELECT + sr.source_id, + sr.namespace, + sr.name, + sr.updated_at, + sr.columns, + s.name AS source_name, + ss.name AS source_schema_name, + sd.name AS source_database_name, + s.type AS source_type + FROM mz_internal.mz_source_references sr + JOIN mz_sources s ON sr.source_id = s.id + JOIN mz_schemas ss ON s.schema_id = ss.id + JOIN mz_databases sd ON ss.database_id = sd.id +`) + +func SourceReferenceId(conn *sqlx.DB, sourceId string) (string, error) { + p := map[string]string{ + "sr.source_id": sourceId, + } + q := sourceReferenceQuery.QueryPredicate(p) + + var s SourceReferenceParams + if err := conn.Get(&s, q); err != nil { + return "", err + } + + return s.SourceId.String, nil +} + +func ScanSourceReference(conn *sqlx.DB, id string) (SourceReferenceParams, error) { + q := sourceReferenceQuery.QueryPredicate(map[string]string{"sr.source_id": id}) + + var s SourceReferenceParams + if err := conn.Get(&s, q); err != nil { + return s, err + } + + return s, nil +} + +func ListSourceReferences(conn *sqlx.DB, sourceId string) ([]SourceReferenceParams, error) { + p := map[string]string{ + "sr.source_id": sourceId, + } + q := sourceReferenceQuery.QueryPredicate(p) + + var references []SourceReferenceParams + if err := conn.Select(&references, q); err != nil { + return references, err + } + + return references, nil +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 09132ad9..c3e9474c 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -162,6 +162,7 @@ func Provider(version string) *schema.Provider { "materialize_secret": datasources.Secret(), "materialize_sink": datasources.Sink(), "materialize_source": datasources.Source(), + "materialize_source_reference": datasources.SourceReference(), "materialize_source_table": datasources.SourceTable(), "materialize_scim_groups": datasources.SCIMGroups(), "materialize_scim_configs": datasources.SCIMConfigs(), From 7d05413450b8a478ef4b2560c4aa8dad39d0d0e3 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Sun, 6 Oct 2024 14:56:54 +0300 Subject: [PATCH 30/46] Add source reference data source example --- docs/data-sources/source_reference.md | 10 ++++++++++ .../materialize_source_reference/data-source.tf | 7 +++++++ 2 files changed, 17 insertions(+) create mode 100644 examples/data-sources/materialize_source_reference/data-source.tf diff --git a/docs/data-sources/source_reference.md b/docs/data-sources/source_reference.md index 59883eff..b3aac8ce 100644 --- a/docs/data-sources/source_reference.md +++ b/docs/data-sources/source_reference.md @@ -10,7 +10,17 @@ description: |- +## Example Usage +```terraform +data "materialize_source_reference" "source_references" { + source_id = materialize_source_mysql.test.id +} + +output "source_references" { + value = data.materialize_source_reference.my_source_references.references +} +``` ## Schema diff --git a/examples/data-sources/materialize_source_reference/data-source.tf b/examples/data-sources/materialize_source_reference/data-source.tf new file mode 100644 index 00000000..b4c430ee --- /dev/null +++ b/examples/data-sources/materialize_source_reference/data-source.tf @@ -0,0 +1,7 @@ +data "materialize_source_reference" "source_references" { + source_id = materialize_source_mysql.test.id +} + +output "source_references" { + value = data.materialize_source_reference.my_source_references.references +} From c94efd041f17530b16570b800d88c0cc073c64b7 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 14 Oct 2024 21:15:56 +0300 Subject: [PATCH 31/46] First round of the initial PR change requests --- docs/guides/materialize_source_table.md | 22 +++++----- docs/resources/source_kafka.md | 22 +++++----- docs/resources/source_mysql.md | 6 +-- docs/resources/source_table_mysql.md | 2 +- pkg/materialize/source_table_kafka.go | 20 ++++----- pkg/materialize/source_table_kafka_test.go | 2 +- pkg/resources/resource_source_kafka.go | 44 +++++++++---------- pkg/resources/resource_source_mysql.go | 14 +++--- .../resource_source_table_kafka_test.go | 6 +-- pkg/resources/resource_source_table_mysql.go | 2 +- .../guides/materialize_source_table.md.tmpl | 22 +++++----- 11 files changed, 81 insertions(+), 81 deletions(-) diff --git a/docs/guides/materialize_source_table.md b/docs/guides/materialize_source_table.md index 1a269449..fb374454 100644 --- a/docs/guides/materialize_source_table.md +++ b/docs/guides/materialize_source_table.md @@ -1,10 +1,10 @@ -# Source versioning: migrating to `materialize_source_table_{source}` Resource +# Source versioning: migrating to `materialize_source_table_{source_type}` Resource In previous versions of the Materialize Terraform provider, source tables were defined within the source resource itself and were considered subsources of the source rather than separate entities. -This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table_{source}` resource. +This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table_{source_type}` resource. -For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. For Kafka sources, you will need to create at least one `materialize_source_table_kafka` table to hold data for the kafka topic. +For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source_type}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. For Kafka sources, you will need to create at least one `materialize_source_table_kafka` table to hold data for the kafka topic. ## Old Approach @@ -54,15 +54,15 @@ resource "materialize_source_kafka" "example_source_kafka_format_text" { ## New Approach -The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table_{source}` resource. +The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table_{source_type}` resource. ## Manual Migration Process -This manual migration process requires users to create new source tables using the new `materialize_source_table_{source}` resource and then remove the old ones. We'll cover examples for both MySQL and Kafka sources. +This manual migration process requires users to create new source tables using the new `materialize_source_table_{source_type}` resource and then remove the old ones. We'll cover examples for both MySQL and Kafka sources. -### Step 1: Define `materialize_source_table_{source}` Resources +### Step 1: Define `materialize_source_table_{source_type}` Resources -Before making any changes to your existing source resources, create new `materialize_source_table_{source}` resources for each table that is currently defined within your sources. +Before making any changes to your existing source resources, create new `materialize_source_table_{source_type}` resources for each table that is currently defined within your sources. #### MySQL Example: @@ -109,11 +109,11 @@ resource "materialize_source_table_kafka" "kafka_table_from_source" { ### Step 2: Apply the Changes -Run `terraform plan` and `terraform apply` to create the new `materialize_source_table_{source}` resources. +Run `terraform plan` and `terraform apply` to create the new `materialize_source_table_{source_type}` resources. ### Step 3: Remove Table Blocks from Source Resources -Once the new `materialize_source_table_{source}` resources are successfully created, remove all the deprecated and table-specific attributes from your source resources. +Once the new `materialize_source_table_{source_type}` resources are successfully created, remove all the deprecated and table-specific attributes from your source resources. #### MySQL Example: @@ -167,7 +167,7 @@ resource "materialize_source_kafka" "kafka_source" { } ``` -In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source}` resources. +In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source_type}` resources. > Note: We will make the changes to those attributes a no-op, so the `ignore_changes` block will not be necessary. @@ -184,7 +184,7 @@ After applying the changes, verify that your tables are still correctly set up i To import existing tables into your Terraform state, use the following command: ```bash -terraform import materialize_source_table_{source}.table_name : +terraform import materialize_source_table_{source_type}.table_name : ``` Replace `{source}` with the appropriate source type (e.g., `mysql`, `kafka`), `` with the actual region, and `` with the table ID. diff --git a/docs/resources/source_kafka.md b/docs/resources/source_kafka.md index 593f5511..3df0f869 100644 --- a/docs/resources/source_kafka.md +++ b/docs/resources/source_kafka.md @@ -56,19 +56,19 @@ resource "materialize_source_kafka" "example_source_kafka" { - `cluster_name` (String) The cluster to maintain this source. - `comment` (String) Comment on an object in the database. - `database_name` (String) The identifier for the source database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. -- `envelope` (Block List, Max: 1, Deprecated) How Materialize should interpret records (e.g. append-only, upsert). Deprecated: Use the new materialize_source_table_kafka resource instead. (see [below for nested schema](#nestedblock--envelope)) +- `envelope` (Block List, Max: 1, Deprecated) How Materialize should interpret records (e.g. append-only, upsert). Deprecated: Use the new `materialize_source_table_kafka` resource instead. (see [below for nested schema](#nestedblock--envelope)) - `expose_progress` (Block List, Max: 1) The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`. (see [below for nested schema](#nestedblock--expose_progress)) - `format` (Block List, Max: 1) How to decode raw bytes from different formats into data structures Materialize can understand at runtime. (see [below for nested schema](#nestedblock--format)) -- `include_headers` (Boolean, Deprecated) Include message headers. Deprecated: Use the new materialize_source_table_kafka resource instead. -- `include_headers_alias` (String, Deprecated) Provide an alias for the headers column. Deprecated: Use the new materialize_source_table_kafka resource instead. -- `include_key` (Boolean, Deprecated) Include a column containing the Kafka message key. Deprecated: Use the new materialize_source_table_kafka resource instead. -- `include_key_alias` (String, Deprecated) Provide an alias for the key column. Deprecated: Use the new materialize_source_table_kafka resource instead. -- `include_offset` (Boolean, Deprecated) Include an offset column containing the Kafka message offset. Deprecated: Use the new materialize_source_table_kafka resource instead. -- `include_offset_alias` (String, Deprecated) Provide an alias for the offset column. Deprecated: Use the new materialize_source_table_kafka resource instead. -- `include_partition` (Boolean, Deprecated) Include a partition column containing the Kafka message partition. Deprecated: Use the new materialize_source_table_kafka resource instead. -- `include_partition_alias` (String, Deprecated) Provide an alias for the partition column. Deprecated: Use the new materialize_source_table_kafka resource instead. -- `include_timestamp` (Boolean, Deprecated) Include a timestamp column containing the Kafka message timestamp. Deprecated: Use the new materialize_source_table_kafka resource instead. -- `include_timestamp_alias` (String, Deprecated) Provide an alias for the timestamp column. Deprecated: Use the new materialize_source_table_kafka resource instead. +- `include_headers` (Boolean, Deprecated) Include message headers. Deprecated: Use the new `materialize_source_table_kafka` resource instead. +- `include_headers_alias` (String, Deprecated) Provide an alias for the headers column. Deprecated: Use the new `materialize_source_table_kafka` resource instead. +- `include_key` (Boolean, Deprecated) Include a column containing the Kafka message key. Deprecated: Use the new `materialize_source_table_kafka` resource instead. +- `include_key_alias` (String, Deprecated) Provide an alias for the key column. Deprecated: Use the new `materialize_source_table_kafka` resource instead. +- `include_offset` (Boolean, Deprecated) Include an offset column containing the Kafka message offset. Deprecated: Use the new `materialize_source_table_kafka` resource instead. +- `include_offset_alias` (String, Deprecated) Provide an alias for the offset column. Deprecated: Use the new `materialize_source_table_kafka` resource instead. +- `include_partition` (Boolean, Deprecated) Include a partition column containing the Kafka message partition. Deprecated: Use the new `materialize_source_table_kafka` resource instead. +- `include_partition_alias` (String, Deprecated) Provide an alias for the partition column. Deprecated: Use the new `materialize_source_table_kafka` resource instead. +- `include_timestamp` (Boolean, Deprecated) Include a timestamp column containing the Kafka message timestamp. Deprecated: Use the new `materialize_source_table_kafka` resource instead. +- `include_timestamp_alias` (String, Deprecated) Provide an alias for the timestamp column. Deprecated: Use the new `materialize_source_table_kafka` resource instead. - `key_format` (Block List, Max: 1) Set the key format explicitly. (see [below for nested schema](#nestedblock--key_format)) - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. diff --git a/docs/resources/source_mysql.md b/docs/resources/source_mysql.md index 01773461..41246367 100644 --- a/docs/resources/source_mysql.md +++ b/docs/resources/source_mysql.md @@ -53,12 +53,12 @@ resource "materialize_source_mysql" "test" { - `comment` (String) Comment on an object in the database. - `database_name` (String) The identifier for the source database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `expose_progress` (Block List, Max: 1) The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`. (see [below for nested schema](#nestedblock--expose_progress)) -- `ignore_columns` (List of String, Deprecated) Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_mysql resource instead. +- `ignore_columns` (List of String, Deprecated) Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new `materialize_source_table_mysql` resource instead. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `table` (Block Set, Deprecated) Specify the tables to be included in the source. Deprecated: Use the new materialize_source_table_mysql resource instead. (see [below for nested schema](#nestedblock--table)) -- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_mysql resource instead. +- `table` (Block Set, Deprecated) Specify the tables to be included in the source. Deprecated: Use the new `materialize_source_table_mysql` resource instead. (see [below for nested schema](#nestedblock--table)) +- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new `materialize_source_table_mysql` resource instead. ### Read-Only diff --git a/docs/resources/source_table_mysql.md b/docs/resources/source_table_mysql.md index 193204f8..e77c85ad 100644 --- a/docs/resources/source_table_mysql.md +++ b/docs/resources/source_table_mysql.md @@ -48,7 +48,7 @@ resource "materialize_source_table_mysql" "mysql_table_from_source" { - `comment` (String) **Public Preview** Comment on an object in the database. - `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. -- `exclude_columns` (List of String) Exclude specific columns when reading data from MySQL. The option used to be called `ignore_columns`. +- `exclude_columns` (List of String) Exclude specific columns when reading data from MySQL. This option used to be called `ignore_columns`. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the table schema in Materialize. Defaults to `public`. diff --git a/pkg/materialize/source_table_kafka.go b/pkg/materialize/source_table_kafka.go index aa8c1ff5..580183dd 100644 --- a/pkg/materialize/source_table_kafka.go +++ b/pkg/materialize/source_table_kafka.go @@ -215,7 +215,7 @@ func (b *SourceTableKafkaBuilder) Create() error { options = append(options, fmt.Sprintf(`FORMAT CSV WITH HEADER ( %s )`, strings.Join(b.format.Csv.Header, ", "))) } if b.format.Csv.DelimitedBy != "" { - options = append(options, fmt.Sprintf(`DELIMITER '%s'`, b.format.Csv.DelimitedBy)) + options = append(options, fmt.Sprintf(`DELIMITER %s`, QuoteString(b.format.Csv.DelimitedBy))) } } @@ -244,7 +244,7 @@ func (b *SourceTableKafkaBuilder) Create() error { if b.keyFormat.Protobuf != nil { if b.keyFormat.Protobuf.SchemaRegistryConnection.Name != "" && b.keyFormat.Protobuf.MessageName != "" { - options = append(options, fmt.Sprintf(`KEY FORMAT PROTOBUF MESSAGE '%s' USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, b.keyFormat.Protobuf.MessageName, QualifiedName(b.keyFormat.Protobuf.SchemaRegistryConnection.DatabaseName, b.keyFormat.Protobuf.SchemaRegistryConnection.SchemaName, b.keyFormat.Protobuf.SchemaRegistryConnection.Name))) + options = append(options, fmt.Sprintf(`KEY FORMAT PROTOBUF MESSAGE %s USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QuoteString(b.keyFormat.Protobuf.MessageName), QualifiedName(b.keyFormat.Protobuf.SchemaRegistryConnection.DatabaseName, b.keyFormat.Protobuf.SchemaRegistryConnection.SchemaName, b.keyFormat.Protobuf.SchemaRegistryConnection.Name))) } else if b.keyFormat.Protobuf.SchemaRegistryConnection.Name != "" { options = append(options, fmt.Sprintf(`KEY FORMAT PROTOBUF USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QualifiedName(b.keyFormat.Protobuf.SchemaRegistryConnection.DatabaseName, b.keyFormat.Protobuf.SchemaRegistryConnection.SchemaName, b.keyFormat.Protobuf.SchemaRegistryConnection.Name))) } @@ -258,7 +258,7 @@ func (b *SourceTableKafkaBuilder) Create() error { options = append(options, fmt.Sprintf(`KEY FORMAT CSV WITH HEADER ( %s )`, strings.Join(b.keyFormat.Csv.Header, ", "))) } if b.keyFormat.Csv.DelimitedBy != "" { - options = append(options, fmt.Sprintf(`KEY DELIMITER '%s'`, b.keyFormat.Csv.DelimitedBy)) + options = append(options, fmt.Sprintf(`KEY DELIMITER %s`, QuoteString(b.keyFormat.Csv.DelimitedBy))) } } @@ -287,7 +287,7 @@ func (b *SourceTableKafkaBuilder) Create() error { if b.valueFormat.Protobuf != nil { if b.valueFormat.Protobuf.SchemaRegistryConnection.Name != "" && b.valueFormat.Protobuf.MessageName != "" { - options = append(options, fmt.Sprintf(`VALUE FORMAT PROTOBUF MESSAGE '%s' USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, b.valueFormat.Protobuf.MessageName, QualifiedName(b.valueFormat.Protobuf.SchemaRegistryConnection.DatabaseName, b.valueFormat.Protobuf.SchemaRegistryConnection.SchemaName, b.valueFormat.Protobuf.SchemaRegistryConnection.Name))) + options = append(options, fmt.Sprintf(`VALUE FORMAT PROTOBUF MESSAGE %s USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QuoteString(b.valueFormat.Protobuf.MessageName), QualifiedName(b.valueFormat.Protobuf.SchemaRegistryConnection.DatabaseName, b.valueFormat.Protobuf.SchemaRegistryConnection.SchemaName, b.valueFormat.Protobuf.SchemaRegistryConnection.Name))) } else if b.valueFormat.Protobuf.SchemaRegistryConnection.Name != "" { options = append(options, fmt.Sprintf(`VALUE FORMAT PROTOBUF USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QualifiedName(b.valueFormat.Protobuf.SchemaRegistryConnection.DatabaseName, b.valueFormat.Protobuf.SchemaRegistryConnection.SchemaName, b.valueFormat.Protobuf.SchemaRegistryConnection.Name))) } @@ -301,7 +301,7 @@ func (b *SourceTableKafkaBuilder) Create() error { options = append(options, fmt.Sprintf(`VALUE FORMAT CSV WITH HEADER ( %s )`, strings.Join(b.valueFormat.Csv.Header, ", "))) } if b.valueFormat.Csv.DelimitedBy != "" { - options = append(options, fmt.Sprintf(`VALUE DELIMITER '%s'`, b.valueFormat.Csv.DelimitedBy)) + options = append(options, fmt.Sprintf(`VALUE DELIMITER %s`, QuoteString(b.valueFormat.Csv.DelimitedBy))) } } @@ -319,35 +319,35 @@ func (b *SourceTableKafkaBuilder) Create() error { var metadataOptions []string if b.includeKey { if b.keyAlias != "" { - metadataOptions = append(metadataOptions, fmt.Sprintf("KEY AS %s", b.keyAlias)) + metadataOptions = append(metadataOptions, fmt.Sprintf("KEY AS %s", QuoteIdentifier(b.keyAlias))) } else { metadataOptions = append(metadataOptions, "KEY") } } if b.includeHeaders { if b.headersAlias != "" { - metadataOptions = append(metadataOptions, fmt.Sprintf("HEADERS AS %s", b.headersAlias)) + metadataOptions = append(metadataOptions, fmt.Sprintf("HEADERS AS %s", QuoteIdentifier(b.headersAlias))) } else { metadataOptions = append(metadataOptions, "HEADERS") } } if b.includePartition { if b.partitionAlias != "" { - metadataOptions = append(metadataOptions, fmt.Sprintf("PARTITION AS %s", b.partitionAlias)) + metadataOptions = append(metadataOptions, fmt.Sprintf("PARTITION AS %s", QuoteIdentifier(b.partitionAlias))) } else { metadataOptions = append(metadataOptions, "PARTITION") } } if b.includeOffset { if b.offsetAlias != "" { - metadataOptions = append(metadataOptions, fmt.Sprintf("OFFSET AS %s", b.offsetAlias)) + metadataOptions = append(metadataOptions, fmt.Sprintf("OFFSET AS %s", QuoteIdentifier(b.offsetAlias))) } else { metadataOptions = append(metadataOptions, "OFFSET") } } if b.includeTimestamp { if b.timestampAlias != "" { - metadataOptions = append(metadataOptions, fmt.Sprintf("TIMESTAMP AS %s", b.timestampAlias)) + metadataOptions = append(metadataOptions, fmt.Sprintf("TIMESTAMP AS %s", QuoteIdentifier(b.timestampAlias))) } else { metadataOptions = append(metadataOptions, "TIMESTAMP") } diff --git a/pkg/materialize/source_table_kafka_test.go b/pkg/materialize/source_table_kafka_test.go index 22f41703..9682b904 100644 --- a/pkg/materialize/source_table_kafka_test.go +++ b/pkg/materialize/source_table_kafka_test.go @@ -15,7 +15,7 @@ func TestResourceSourceTableKafkaCreate(t *testing.T) { FROM SOURCE "database"."schema"."kafka_source" \(REFERENCE "topic"\) FORMAT JSON - INCLUDE KEY AS message_key, HEADERS AS message_headers, PARTITION AS message_partition + INCLUDE KEY AS "message_key", HEADERS AS "message_headers", PARTITION AS "message_partition" ENVELOPE UPSERT EXPOSE PROGRESS AS "database"."schema"."progress";`, ).WillReturnResult(sqlmock.NewResult(1, 1)) diff --git a/pkg/resources/resource_source_kafka.go b/pkg/resources/resource_source_kafka.go index 7641e9ac..6cc7ce1f 100644 --- a/pkg/resources/resource_source_kafka.go +++ b/pkg/resources/resource_source_kafka.go @@ -32,72 +32,72 @@ var sourceKafkaSchema = map[string]*schema.Schema{ ForceNew: true, }, "include_key": { - Description: "Include a column containing the Kafka message key. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Include a column containing the Kafka message key. Deprecated: Use the new `materialize_source_table_kafka` resource instead.", + Deprecated: "Use the new `materialize_source_table_kafka` resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, }, "include_key_alias": { - Description: "Provide an alias for the key column. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Provide an alias for the key column. Deprecated: Use the new `materialize_source_table_kafka` resource instead.", + Deprecated: "Use the new `materialize_source_table_kafka` resource instead.", Type: schema.TypeString, Optional: true, ForceNew: true, }, "include_headers": { - Description: "Include message headers. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Include message headers. Deprecated: Use the new `materialize_source_table_kafka` resource instead.", + Deprecated: "Use the new `materialize_source_table_kafka` resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, Default: false, }, "include_headers_alias": { - Description: "Provide an alias for the headers column. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Provide an alias for the headers column. Deprecated: Use the new `materialize_source_table_kafka` resource instead.", + Deprecated: "Use the new `materialize_source_table_kafka` resource instead.", Type: schema.TypeString, Optional: true, ForceNew: true, }, "include_partition": { - Description: "Include a partition column containing the Kafka message partition. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Include a partition column containing the Kafka message partition. Deprecated: Use the new `materialize_source_table_kafka` resource instead.", + Deprecated: "Use the new `materialize_source_table_kafka` resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, }, "include_partition_alias": { - Description: "Provide an alias for the partition column. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Provide an alias for the partition column. Deprecated: Use the new `materialize_source_table_kafka` resource instead.", + Deprecated: "Use the new `materialize_source_table_kafka` resource instead.", Type: schema.TypeString, Optional: true, ForceNew: true, }, "include_offset": { - Description: "Include an offset column containing the Kafka message offset. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Include an offset column containing the Kafka message offset. Deprecated: Use the new `materialize_source_table_kafka` resource instead.", + Deprecated: "Use the new `materialize_source_table_kafka` resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, }, "include_offset_alias": { - Description: "Provide an alias for the offset column. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Provide an alias for the offset column. Deprecated: Use the new `materialize_source_table_kafka` resource instead.", + Deprecated: "Use the new `materialize_source_table_kafka` resource instead.", Type: schema.TypeString, Optional: true, ForceNew: true, }, "include_timestamp": { - Description: "Include a timestamp column containing the Kafka message timestamp. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Include a timestamp column containing the Kafka message timestamp. Deprecated: Use the new `materialize_source_table_kafka` resource instead.", + Deprecated: "Use the new `materialize_source_table_kafka` resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, }, "include_timestamp_alias": { - Description: "Provide an alias for the timestamp column. Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "Provide an alias for the timestamp column. Deprecated: Use the new `materialize_source_table_kafka` resource instead.", + Deprecated: "Use the new `materialize_source_table_kafka` resource instead.", Type: schema.TypeString, Optional: true, ForceNew: true, @@ -106,8 +106,8 @@ var sourceKafkaSchema = map[string]*schema.Schema{ "key_format": FormatSpecSchema("key_format", "Set the key format explicitly.", false), "value_format": FormatSpecSchema("value_format", "Set the value format explicitly.", false), "envelope": { - Description: "How Materialize should interpret records (e.g. append-only, upsert). Deprecated: Use the new materialize_source_table_kafka resource instead.", - Deprecated: "Use the new materialize_source_table_kafka resource instead.", + Description: "How Materialize should interpret records (e.g. append-only, upsert). Deprecated: Use the new `materialize_source_table_kafka` resource instead.", + Deprecated: "Use the new `materialize_source_table_kafka` resource instead.", Type: schema.TypeList, MaxItems: 1, Elem: &schema.Resource{ diff --git a/pkg/resources/resource_source_mysql.go b/pkg/resources/resource_source_mysql.go index 40ef1681..6950818f 100644 --- a/pkg/resources/resource_source_mysql.go +++ b/pkg/resources/resource_source_mysql.go @@ -27,22 +27,22 @@ var sourceMySQLSchema = map[string]*schema.Schema{ ForceNew: true, }), "ignore_columns": { - Description: "Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_mysql resource instead.", - Deprecated: "Use the new materialize_source_table_mysql resource instead.", + Description: "Ignore specific columns when reading data from MySQL. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new `materialize_source_table_mysql` resource instead.", + Deprecated: "Use the new `materialize_source_table_mysql` resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "text_columns": { - Description: "Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_mysql resource instead.", - Deprecated: "Use the new materialize_source_table_mysql resource instead.", + Description: "Decode data as text for specific columns that contain MySQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new `materialize_source_table_mysql` resource instead.", + Deprecated: "Use the new `materialize_source_table_mysql` resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "table": { - Description: "Specify the tables to be included in the source. Deprecated: Use the new materialize_source_table_mysql resource instead.", - Deprecated: "Use the new materialize_source_table_mysql resource instead.", + Description: "Specify the tables to be included in the source. Deprecated: Use the new `materialize_source_table_mysql` resource instead.", + Deprecated: "Use the new `materialize_source_table_mysql` resource instead.", Type: schema.TypeSet, Optional: true, Elem: &schema.Resource{ @@ -81,7 +81,7 @@ var sourceMySQLSchema = map[string]*schema.Schema{ }, "all_tables": { Description: "Include all tables in the source. If `table` is specified, this will be ignored.", - Deprecated: "Use the new materialize_source_table_mysql resource instead.", + Deprecated: "Use the new `materialize_source_table_mysql` resource instead.", Type: schema.TypeBool, Optional: true, ForceNew: true, diff --git a/pkg/resources/resource_source_table_kafka_test.go b/pkg/resources/resource_source_table_kafka_test.go index c52fc097..ea4d8c87 100644 --- a/pkg/resources/resource_source_table_kafka_test.go +++ b/pkg/resources/resource_source_table_kafka_test.go @@ -67,8 +67,8 @@ func TestResourceSourceTableKafkaCreate(t *testing.T) { FROM SOURCE "materialize"."public"."kafka_source" \(REFERENCE "topic"\) FORMAT JSON - INCLUDE KEY AS message_key, HEADERS AS message_headers, PARTITION AS message_partition - ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS decoding_error\)\);`, + INCLUDE KEY AS "message_key", HEADERS AS "message_headers", PARTITION AS "message_partition" + ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS "decoding_error"\)\);`, ).WillReturnResult(sqlmock.NewResult(1, 1)) // Query Id @@ -227,7 +227,7 @@ func TestResourceSourceTableKafkaCreateIncludeTrueNoAlias(t *testing.T) { \(REFERENCE "topic"\) FORMAT JSON INCLUDE KEY, HEADERS, PARTITION, OFFSET, TIMESTAMP - ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS decoding_error\)\);`, + ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS "decoding_error"\)\);`, ).WillReturnResult(sqlmock.NewResult(1, 1)) // Query Id diff --git a/pkg/resources/resource_source_table_mysql.go b/pkg/resources/resource_source_table_mysql.go index 024cc67e..78b321e6 100644 --- a/pkg/resources/resource_source_table_mysql.go +++ b/pkg/resources/resource_source_table_mysql.go @@ -43,7 +43,7 @@ var sourceTableMySQLSchema = map[string]*schema.Schema{ ForceNew: true, }, "exclude_columns": { - Description: "Exclude specific columns when reading data from MySQL. The option used to be called `ignore_columns`.", + Description: "Exclude specific columns when reading data from MySQL. This option used to be called `ignore_columns`.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, diff --git a/templates/guides/materialize_source_table.md.tmpl b/templates/guides/materialize_source_table.md.tmpl index 1a269449..fb374454 100644 --- a/templates/guides/materialize_source_table.md.tmpl +++ b/templates/guides/materialize_source_table.md.tmpl @@ -1,10 +1,10 @@ -# Source versioning: migrating to `materialize_source_table_{source}` Resource +# Source versioning: migrating to `materialize_source_table_{source_type}` Resource In previous versions of the Materialize Terraform provider, source tables were defined within the source resource itself and were considered subsources of the source rather than separate entities. -This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table_{source}` resource. +This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table_{source_type}` resource. -For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. For Kafka sources, you will need to create at least one `materialize_source_table_kafka` table to hold data for the kafka topic. +For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source_type}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. For Kafka sources, you will need to create at least one `materialize_source_table_kafka` table to hold data for the kafka topic. ## Old Approach @@ -54,15 +54,15 @@ resource "materialize_source_kafka" "example_source_kafka_format_text" { ## New Approach -The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table_{source}` resource. +The new approach separates source definitions and table definitions. You will now create the source without specifying the tables, and then define each table using the `materialize_source_table_{source_type}` resource. ## Manual Migration Process -This manual migration process requires users to create new source tables using the new `materialize_source_table_{source}` resource and then remove the old ones. We'll cover examples for both MySQL and Kafka sources. +This manual migration process requires users to create new source tables using the new `materialize_source_table_{source_type}` resource and then remove the old ones. We'll cover examples for both MySQL and Kafka sources. -### Step 1: Define `materialize_source_table_{source}` Resources +### Step 1: Define `materialize_source_table_{source_type}` Resources -Before making any changes to your existing source resources, create new `materialize_source_table_{source}` resources for each table that is currently defined within your sources. +Before making any changes to your existing source resources, create new `materialize_source_table_{source_type}` resources for each table that is currently defined within your sources. #### MySQL Example: @@ -109,11 +109,11 @@ resource "materialize_source_table_kafka" "kafka_table_from_source" { ### Step 2: Apply the Changes -Run `terraform plan` and `terraform apply` to create the new `materialize_source_table_{source}` resources. +Run `terraform plan` and `terraform apply` to create the new `materialize_source_table_{source_type}` resources. ### Step 3: Remove Table Blocks from Source Resources -Once the new `materialize_source_table_{source}` resources are successfully created, remove all the deprecated and table-specific attributes from your source resources. +Once the new `materialize_source_table_{source_type}` resources are successfully created, remove all the deprecated and table-specific attributes from your source resources. #### MySQL Example: @@ -167,7 +167,7 @@ resource "materialize_source_kafka" "kafka_source" { } ``` -In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source}` resources. +In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source_type}` resources. > Note: We will make the changes to those attributes a no-op, so the `ignore_changes` block will not be necessary. @@ -184,7 +184,7 @@ After applying the changes, verify that your tables are still correctly set up i To import existing tables into your Terraform state, use the following command: ```bash -terraform import materialize_source_table_{source}.table_name : +terraform import materialize_source_table_{source_type}.table_name : ``` Replace `{source}` with the appropriate source type (e.g., `mysql`, `kafka`), `` with the actual region, and `` with the table ID. From 811d160471a88c9f7d05ba2ad55f82dafef293b1 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 14 Oct 2024 22:18:30 +0300 Subject: [PATCH 32/46] Fix failing tests --- pkg/materialize/source_table_kafka.go | 2 +- pkg/materialize/source_table_kafka_test.go | 2 +- pkg/resources/resource_source_table_kafka_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/materialize/source_table_kafka.go b/pkg/materialize/source_table_kafka.go index 580183dd..1bbe3567 100644 --- a/pkg/materialize/source_table_kafka.go +++ b/pkg/materialize/source_table_kafka.go @@ -367,7 +367,7 @@ func (b *SourceTableKafkaBuilder) Create() error { if inlineOptions.Enabled { upsertOption += " (VALUE DECODING ERRORS = (INLINE" if inlineOptions.Alias != "" { - upsertOption += fmt.Sprintf(" AS %s", inlineOptions.Alias) + upsertOption += fmt.Sprintf(" AS %s", QuoteIdentifier(inlineOptions.Alias)) } upsertOption += "))" } diff --git a/pkg/materialize/source_table_kafka_test.go b/pkg/materialize/source_table_kafka_test.go index 9682b904..85ca8a8c 100644 --- a/pkg/materialize/source_table_kafka_test.go +++ b/pkg/materialize/source_table_kafka_test.go @@ -81,7 +81,7 @@ func TestResourceSourceTableKafkaCreateWithUpsertOptions(t *testing.T) { \(REFERENCE "topic"\) FORMAT JSON INCLUDE KEY, HEADERS, PARTITION, OFFSET, TIMESTAMP - ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS my_error_col\)\) + ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS "my_error_col"\)\) EXPOSE PROGRESS AS "database"."schema"."progress";`, ).WillReturnResult(sqlmock.NewResult(1, 1)) diff --git a/pkg/resources/resource_source_table_kafka_test.go b/pkg/resources/resource_source_table_kafka_test.go index ea4d8c87..6105dee6 100644 --- a/pkg/resources/resource_source_table_kafka_test.go +++ b/pkg/resources/resource_source_table_kafka_test.go @@ -264,7 +264,7 @@ func TestResourceSourceTableKafkaCreateIncludeFalseWithAlias(t *testing.T) { FROM SOURCE "materialize"."public"."kafka_source" \(REFERENCE "topic"\) FORMAT JSON - ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS decoding_error\)\);`, + ENVELOPE UPSERT \(VALUE DECODING ERRORS = \(INLINE AS "decoding_error"\)\);`, ).WillReturnResult(sqlmock.NewResult(1, 1)) // Query Id From 0f7a9217118b6904c19f3054b6c7c5f03c0666e8 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 15 Oct 2024 00:13:30 +0300 Subject: [PATCH 33/46] Fix failing tests --- docs/resources/source_postgres.md | 4 ++-- pkg/resources/resource_source_postgres.go | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/resources/source_postgres.md b/docs/resources/source_postgres.md index 829ca9cf..fad5965b 100644 --- a/docs/resources/source_postgres.md +++ b/docs/resources/source_postgres.md @@ -62,8 +62,8 @@ resource "materialize_source_postgres" "example_source_postgres" { - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source schema in Materialize. Defaults to `public`. -- `table` (Block Set, Deprecated) Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new materialize_source_table_postgres resource instead. (see [below for nested schema](#nestedblock--table)) -- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_postgres resource instead. +- `table` (Block Set, Deprecated) Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new `materialize_source_table_postgres` resource instead. (see [below for nested schema](#nestedblock--table)) +- `text_columns` (List of String, Deprecated) Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new `materialize_source_table_postgres` resource instead. ### Read-Only diff --git a/pkg/resources/resource_source_postgres.go b/pkg/resources/resource_source_postgres.go index 040eb16a..ead886d1 100644 --- a/pkg/resources/resource_source_postgres.go +++ b/pkg/resources/resource_source_postgres.go @@ -33,15 +33,15 @@ var sourcePostgresSchema = map[string]*schema.Schema{ ForceNew: true, }, "text_columns": { - Description: "Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new materialize_source_table_postgres resource instead.", - Deprecated: "Use the new materialize_source_table_postgres resource instead.", + Description: "Decode data as text for specific columns that contain PostgreSQL types that are unsupported in Materialize. Can only be updated in place when also updating a corresponding `table` attribute. Deprecated: Use the new `materialize_source_table_postgres` resource instead.", + Deprecated: "Use the new `materialize_source_table_postgres` resource instead.", Type: schema.TypeList, Elem: &schema.Schema{Type: schema.TypeString}, Optional: true, }, "table": { - Description: "Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new materialize_source_table_postgres resource instead.", - Deprecated: "Use the new materialize_source_table_postgres resource instead.", + Description: "Creates subsources for specific tables in the Postgres connection. Deprecated: Use the new `materialize_source_table_postgres` resource instead.", + Deprecated: "Use the new `materialize_source_table_postgres` resource instead.", Type: schema.TypeSet, Optional: true, Elem: &schema.Resource{ @@ -204,7 +204,7 @@ func sourcePostgresCreate(ctx context.Context, d *schema.ResourceData, meta any) } if v, ok := d.GetOk("table"); ok { - log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_source_table_postgres resource instead.") + log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new `materialize_source_table_postgres` resource instead.") tables := v.(*schema.Set).List() t := materialize.GetTableStruct(tables) b.Table(t) @@ -291,7 +291,7 @@ func sourcePostgresUpdate(ctx context.Context, d *schema.ResourceData, meta any) } if d.HasChange("table") { - log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new materialize_source_table_postgres resource instead.") + log.Printf("[WARN] The 'table' field in materialize_source_postgres is deprecated. Use the new `materialize_source_table_postgres` resource instead.") ot, nt := d.GetChange("table") addTables := materialize.DiffTableStructs(nt.(*schema.Set).List(), ot.(*schema.Set).List()) dropTables := materialize.DiffTableStructs(ot.(*schema.Set).List(), nt.(*schema.Set).List()) From 009b3c297e41cfdfc1ad641728d445c5b85230b1 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 15 Oct 2024 13:52:48 +0300 Subject: [PATCH 34/46] Second round of the initial PR change requests --- docs/guides/materialize_source_table.md | 17 ++++++++++++++++- docs/resources/table_grant.md | 2 +- pkg/materialize/source_table_kafka.go | 2 +- .../guides/materialize_source_table.md.tmpl | 17 ++++++++++++++++- templates/resources/table_grant.md.tmpl | 2 +- 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/docs/guides/materialize_source_table.md b/docs/guides/materialize_source_table.md index fb374454..ddcc4d9d 100644 --- a/docs/guides/materialize_source_table.md +++ b/docs/guides/materialize_source_table.md @@ -1,3 +1,12 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +# template file: templates/guides/materialize_source_table.md.tmpl +page_title: "Source Table Migration Guide" +subcategory: "" +description: |- + Guide for migrating to the new materialize_source_table_{source_type} resources. +--- + # Source versioning: migrating to `materialize_source_table_{source_type}` Resource In previous versions of the Materialize Terraform provider, source tables were defined within the source resource itself and were considered subsources of the source rather than separate entities. @@ -169,7 +178,7 @@ resource "materialize_source_kafka" "kafka_source" { In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source_type}` resources. -> Note: We will make the changes to those attributes a no-op, so the `ignore_changes` block will not be necessary. +> Note: Once the migration process is fully implemented on the Materialize side and the attributes will have to be updated as no-op in future versions of the provider. That way the `ignore_changes` block will no longer be required. At that point, Terraform will correctly handle these attributes without needing the extra lifecycle configuration. Keep an eye on upcoming releases for this change. ### Step 4: Update Terraform State @@ -179,6 +188,12 @@ After removing the `table` blocks and the table/topic specific attributes from y After applying the changes, verify that your tables are still correctly set up in Materialize by checking the table definitions using Materialize's SQL commands. +For a more detailed view of a specific table, you can use the `SHOW CREATE TABLE` command: + +```sql +SHOW CREATE TABLE materialize.public.mysql_table1_from_source; +``` + ## Importing Existing Tables To import existing tables into your Terraform state, use the following command: diff --git a/docs/resources/table_grant.md b/docs/resources/table_grant.md index 9ed3f62a..9efdb4e3 100644 --- a/docs/resources/table_grant.md +++ b/docs/resources/table_grant.md @@ -53,4 +53,4 @@ Import is supported using the following syntax: terraform import materialize_table_grant.example :GRANT|TABLE||| # The region is the region where the database is located (e.g. aws/us-east-1) -``` \ No newline at end of file +``` diff --git a/pkg/materialize/source_table_kafka.go b/pkg/materialize/source_table_kafka.go index 1bbe3567..233d832d 100644 --- a/pkg/materialize/source_table_kafka.go +++ b/pkg/materialize/source_table_kafka.go @@ -201,7 +201,7 @@ func (b *SourceTableKafkaBuilder) Create() error { if b.format.Protobuf != nil { if b.format.Protobuf.SchemaRegistryConnection.Name != "" && b.format.Protobuf.MessageName != "" { - options = append(options, fmt.Sprintf(`FORMAT PROTOBUF MESSAGE '%s' USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, b.format.Protobuf.MessageName, QualifiedName(b.format.Protobuf.SchemaRegistryConnection.DatabaseName, b.format.Protobuf.SchemaRegistryConnection.SchemaName, b.format.Protobuf.SchemaRegistryConnection.Name))) + options = append(options, fmt.Sprintf(`FORMAT PROTOBUF MESSAGE %s USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QuoteString(b.format.Protobuf.MessageName), QualifiedName(b.format.Protobuf.SchemaRegistryConnection.DatabaseName, b.format.Protobuf.SchemaRegistryConnection.SchemaName, b.format.Protobuf.SchemaRegistryConnection.Name))) } else if b.format.Protobuf.SchemaRegistryConnection.Name != "" { options = append(options, fmt.Sprintf(`FORMAT PROTOBUF USING CONFLUENT SCHEMA REGISTRY CONNECTION %s`, QualifiedName(b.format.Protobuf.SchemaRegistryConnection.DatabaseName, b.format.Protobuf.SchemaRegistryConnection.SchemaName, b.format.Protobuf.SchemaRegistryConnection.Name))) } diff --git a/templates/guides/materialize_source_table.md.tmpl b/templates/guides/materialize_source_table.md.tmpl index fb374454..656a7cda 100644 --- a/templates/guides/materialize_source_table.md.tmpl +++ b/templates/guides/materialize_source_table.md.tmpl @@ -1,3 +1,12 @@ +--- +{{ printf "# generated by https://github.com/hashicorp/terraform-plugin-docs" }} +{{ printf "# template file: templates/guides/materialize_source_table.md.tmpl" }} +page_title: "Source Table Migration Guide" +subcategory: "" +description: |- + Guide for migrating to the new materialize_source_table_{source_type} resources. +--- + # Source versioning: migrating to `materialize_source_table_{source_type}` Resource In previous versions of the Materialize Terraform provider, source tables were defined within the source resource itself and were considered subsources of the source rather than separate entities. @@ -169,7 +178,7 @@ resource "materialize_source_kafka" "kafka_source" { In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source_type}` resources. -> Note: We will make the changes to those attributes a no-op, so the `ignore_changes` block will not be necessary. +> Note: Once the migration process is fully implemented on the Materialize side and the attributes will have to be updated as no-op in future versions of the provider. That way the `ignore_changes` block will no longer be required. At that point, Terraform will correctly handle these attributes without needing the extra lifecycle configuration. Keep an eye on upcoming releases for this change. ### Step 4: Update Terraform State @@ -179,6 +188,12 @@ After removing the `table` blocks and the table/topic specific attributes from y After applying the changes, verify that your tables are still correctly set up in Materialize by checking the table definitions using Materialize's SQL commands. +For a more detailed view of a specific table, you can use the `SHOW CREATE TABLE` command: + +```sql +SHOW CREATE TABLE materialize.public.mysql_table1_from_source; +``` + ## Importing Existing Tables To import existing tables into your Terraform state, use the following command: diff --git a/templates/resources/table_grant.md.tmpl b/templates/resources/table_grant.md.tmpl index bebb5df8..e2cf0934 100644 --- a/templates/resources/table_grant.md.tmpl +++ b/templates/resources/table_grant.md.tmpl @@ -21,4 +21,4 @@ description: |- Import is supported using the following syntax: -{{ codefile "shell" (printf "%s%s%s" "examples/resources/" .Name "/import.sh") }} \ No newline at end of file +{{ codefile "shell" (printf "%s%s%s" "examples/resources/" .Name "/import.sh") }} From ea4f8dda71f5831ebe05b9bec4cd81b80aef91cb Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 15 Oct 2024 15:45:35 +0300 Subject: [PATCH 35/46] Add unit tests to data source source reference --- .../datasource_source_reference.go | 2 +- .../datasource_source_reference_test.go | 46 +++++++++++++++++++ pkg/materialize/source_reference.go | 2 +- pkg/testhelpers/mock_scans.go | 30 ++++++++++++ 4 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 pkg/datasources/datasource_source_reference_test.go diff --git a/pkg/datasources/datasource_source_reference.go b/pkg/datasources/datasource_source_reference.go index 1f4c8652..46d6bf20 100644 --- a/pkg/datasources/datasource_source_reference.go +++ b/pkg/datasources/datasource_source_reference.go @@ -96,7 +96,7 @@ func sourceReferenceRead(ctx context.Context, d *schema.ResourceData, meta inter referenceMap := map[string]interface{}{ "namespace": sr.Namespace.String, "name": sr.Name.String, - "updated_at": sr.UpdatedAt.Time.String(), + "updated_at": sr.UpdatedAt.String, "columns": sr.Columns, "source_name": sr.SourceName.String, "source_schema_name": sr.SourceSchemaName.String, diff --git a/pkg/datasources/datasource_source_reference_test.go b/pkg/datasources/datasource_source_reference_test.go new file mode 100644 index 00000000..8dfa0774 --- /dev/null +++ b/pkg/datasources/datasource_source_reference_test.go @@ -0,0 +1,46 @@ +package datasources + +import ( + "context" + "testing" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/stretchr/testify/require" +) + +func TestSourceReferenceDatasource(t *testing.T) { + r := require.New(t) + + in := map[string]interface{}{ + "source_id": "source-id", + } + d := schema.TestResourceDataRaw(t, SourceReference().Schema, in) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + predicate := `WHERE sr.source_id = 'source-id'` + testhelpers.MockSourceReferenceScan(mock, predicate) + + if err := sourceReferenceRead(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + + // Verify the results + references := d.Get("references").([]interface{}) + r.Equal(1, len(references)) + + reference := references[0].(map[string]interface{}) + r.Equal("namespace", reference["namespace"]) + r.Equal("reference_name", reference["name"]) + r.Equal("2023-10-01T12:34:56Z", reference["updated_at"]) + r.Equal([]interface{}{"column1", "column2"}, reference["columns"]) + r.Equal("source_name", reference["source_name"]) + r.Equal("source_schema_name", reference["source_schema_name"]) + r.Equal("source_database_name", reference["source_database_name"]) + r.Equal("source_type", reference["source_type"]) + }) +} diff --git a/pkg/materialize/source_reference.go b/pkg/materialize/source_reference.go index eb934f7a..49eadf9e 100644 --- a/pkg/materialize/source_reference.go +++ b/pkg/materialize/source_reference.go @@ -11,7 +11,7 @@ type SourceReferenceParams struct { SourceId sql.NullString `db:"source_id"` Namespace sql.NullString `db:"namespace"` Name sql.NullString `db:"name"` - UpdatedAt sql.NullTime `db:"updated_at"` + UpdatedAt sql.NullString `db:"updated_at"` Columns pq.StringArray `db:"columns"` SourceName sql.NullString `db:"source_name"` SourceSchemaName sql.NullString `db:"source_schema_name"` diff --git a/pkg/testhelpers/mock_scans.go b/pkg/testhelpers/mock_scans.go index ff1ecd23..4ca40e4b 100644 --- a/pkg/testhelpers/mock_scans.go +++ b/pkg/testhelpers/mock_scans.go @@ -961,6 +961,36 @@ func MockSourceTableScan(mock sqlmock.Sqlmock, predicate string) { mock.ExpectQuery(q).WillReturnRows(ir) } +func MockSourceReferenceScan(mock sqlmock.Sqlmock, predicate string) { + b := ` + SELECT + sr.source_id, + sr.namespace, + sr.name, + sr.updated_at, + sr.columns, + s.name AS source_name, + ss.name AS source_schema_name, + sd.name AS source_database_name, + s.type AS source_type + FROM mz_internal.mz_source_references sr + JOIN mz_sources s ON sr.source_id = s.id + JOIN mz_schemas ss ON s.schema_id = ss.id + JOIN mz_databases sd ON ss.database_id = sd.id` + + q := mockQueryBuilder(b, predicate, "") + ir := mock.NewRows([]string{ + "source_id", "namespace", "name", "updated_at", "columns", + "source_name", "source_schema_name", "source_database_name", "source_type", + }).AddRow( + "source-id", "namespace", "reference_name", "2023-10-01T12:34:56Z", + pq.StringArray{"column1", "column2"}, + "source_name", "source_schema_name", "source_database_name", "source_type", + ) + + mock.ExpectQuery(q).WillReturnRows(ir) +} + func MockTypeScan(mock sqlmock.Sqlmock, predicate string) { b := ` SELECT From 41eb5938e707aaa6a3108c2b5ac6a877f1f494f3 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 21 Oct 2024 19:17:06 +0300 Subject: [PATCH 36/46] PR change requests --- docs/data-sources/source_reference.md | 4 +- docs/guides/materialize_source_table.md | 4 +- docs/resources/source_table_kafka.md | 2 +- integration/source.tf | 23 +++++++++ .../datasource_source_reference.go | 1 + pkg/materialize/source_table.go | 16 +++--- pkg/resources/resource_source_table_kafka.go | 2 +- .../resource_source_table_kafka_test.go | 50 +++++++++++++++++++ .../guides/materialize_source_table.md.tmpl | 4 +- 9 files changed, 92 insertions(+), 14 deletions(-) diff --git a/docs/data-sources/source_reference.md b/docs/data-sources/source_reference.md index b3aac8ce..8535ace1 100644 --- a/docs/data-sources/source_reference.md +++ b/docs/data-sources/source_reference.md @@ -3,12 +3,12 @@ page_title: "materialize_source_reference Data Source - terraform-provider-materialize" subcategory: "" description: |- - + The materialize_source_reference data source retrieves information about a Materialize source's references, including details about namespaces, columns, and last update times. --- # materialize_source_reference (Data Source) - +The `materialize_source_reference` data source retrieves information about a Materialize source's references, including details about namespaces, columns, and last update times. ## Example Usage diff --git a/docs/guides/materialize_source_table.md b/docs/guides/materialize_source_table.md index ddcc4d9d..3e5301af 100644 --- a/docs/guides/materialize_source_table.md +++ b/docs/guides/materialize_source_table.md @@ -13,7 +13,7 @@ In previous versions of the Materialize Terraform provider, source tables were d This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table_{source_type}` resource. -For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source_type}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. For Kafka sources, you will need to create at least one `materialize_source_table_kafka` table to hold data for the kafka topic. +For each MySQL and Postgres source, you will need to create a new `materialize_source_table_{source_type}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. For Kafka sources, you will need to create a `materialize_source_table_kafka` table with the same name as the kafka source to contain the data for the kafka topic. ## Old Approach @@ -242,4 +242,4 @@ After importing, you may need to manually update these ignored attributes in you ## Future Improvements -The Kafka and Webhooks sources are currently being implemented. Once these changes, the migration process will be updated to include them. +Webhooks sources have not yet been migrated to the new model. Once this changes, the migration process will be updated to include them. diff --git a/docs/resources/source_table_kafka.md b/docs/resources/source_table_kafka.md index e25ddc84..73bee1b4 100644 --- a/docs/resources/source_table_kafka.md +++ b/docs/resources/source_table_kafka.md @@ -68,7 +68,6 @@ resource "materialize_source_table_kafka" "kafka_source_table" { - `name` (String) The identifier for the source table. - `source` (Block List, Min: 1, Max: 1) The source this table is created from. (see [below for nested schema](#nestedblock--source)) -- `topic` (String) The name of the Kafka topic in the Kafka cluster. ### Optional @@ -91,6 +90,7 @@ resource "materialize_source_table_kafka" "kafka_source_table" { - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. - `schema_name` (String) The identifier for the source table schema in Materialize. Defaults to `public`. +- `topic` (String) The name of the Kafka topic in the Kafka cluster. - `value_format` (Block List, Max: 1) Set the value format explicitly. (see [below for nested schema](#nestedblock--value_format)) ### Read-Only diff --git a/integration/source.tf b/integration/source.tf index 1975da7e..c3273fd4 100644 --- a/integration/source.tf +++ b/integration/source.tf @@ -222,6 +222,29 @@ resource "materialize_source_kafka" "example_source_kafka_format_text" { depends_on = [materialize_sink_kafka.sink_kafka] } +resource "materialize_source_kafka" "example_source_kafka_no_topic" { + name = "source_kafka_no_topic" + comment = "source kafka comment no topic" + cluster_name = materialize_cluster.cluster_source.name + + kafka_connection { + name = materialize_connection_kafka.kafka_connection.name + schema_name = materialize_connection_kafka.kafka_connection.schema_name + database_name = materialize_connection_kafka.kafka_connection.database_name + } + key_format { + text = true + } + value_format { + text = true + } + expose_progress { + name = "expose_kafka" + } + + depends_on = [materialize_sink_kafka.sink_kafka] +} + # Create source table from Kafka source resource "materialize_source_table_kafka" "source_table_kafka" { name = "source_table_kafka" diff --git a/pkg/datasources/datasource_source_reference.go b/pkg/datasources/datasource_source_reference.go index 46d6bf20..17e16750 100644 --- a/pkg/datasources/datasource_source_reference.go +++ b/pkg/datasources/datasource_source_reference.go @@ -13,6 +13,7 @@ import ( func SourceReference() *schema.Resource { return &schema.Resource{ ReadContext: sourceReferenceRead, + Description: "The `materialize_source_reference` data source retrieves information about a Materialize source's references, including details about namespaces, columns, and last update times.", Schema: map[string]*schema.Schema{ "source_id": { Type: schema.TypeString, diff --git a/pkg/materialize/source_table.go b/pkg/materialize/source_table.go index e73ba519..c1932cce 100644 --- a/pkg/materialize/source_table.go +++ b/pkg/materialize/source_table.go @@ -156,14 +156,18 @@ func (b *SourceTableBuilder) BaseCreate(sourceType string, additionalOptions fun q := strings.Builder{} q.WriteString(fmt.Sprintf(`CREATE TABLE %s`, b.QualifiedName())) q.WriteString(fmt.Sprintf(` FROM SOURCE %s`, b.source.QualifiedName())) - q.WriteString(` (REFERENCE `) - if b.upstreamSchemaName != "" { - q.WriteString(fmt.Sprintf(`%s.`, QuoteIdentifier(b.upstreamSchemaName))) - } - q.WriteString(QuoteIdentifier(b.upstreamName)) + // Reference is not required for Kafka sources and single-output load generator sources + if b.upstreamName != "" { + q.WriteString(` (REFERENCE `) + + if b.upstreamSchemaName != "" { + q.WriteString(fmt.Sprintf(`%s.`, QuoteIdentifier(b.upstreamSchemaName))) + } + q.WriteString(QuoteIdentifier(b.upstreamName)) - q.WriteString(")") + q.WriteString(")") + } if additionalOptions != nil { options := additionalOptions() diff --git a/pkg/resources/resource_source_table_kafka.go b/pkg/resources/resource_source_table_kafka.go index 23ea5ed2..b60b7bae 100644 --- a/pkg/resources/resource_source_table_kafka.go +++ b/pkg/resources/resource_source_table_kafka.go @@ -26,7 +26,7 @@ var sourceTableKafkaSchema = map[string]*schema.Schema{ }), "topic": { Type: schema.TypeString, - Required: true, + Optional: true, ForceNew: true, Description: "The name of the Kafka topic in the Kafka cluster.", }, diff --git a/pkg/resources/resource_source_table_kafka_test.go b/pkg/resources/resource_source_table_kafka_test.go index 6105dee6..0088b876 100644 --- a/pkg/resources/resource_source_table_kafka_test.go +++ b/pkg/resources/resource_source_table_kafka_test.go @@ -456,3 +456,53 @@ func TestResourceSourceTableKafkaCreateWithProtobufFormat(t *testing.T) { } }) } + +func TestResourceSourceTableKafkaCreateWithNoTopic(t *testing.T) { + r := require.New(t) + inSourceTableKafkaNoTopic := map[string]interface{}{ + "name": "no_topic", + "schema_name": "schema", + "database_name": "database", + "source": []interface{}{ + map[string]interface{}{ + "name": "kafka_source", + "schema_name": "public", + "database_name": "materialize", + }, + }, + "format": []interface{}{ + map[string]interface{}{ + "json": true, + }, + }, + "envelope": []interface{}{ + map[string]interface{}{ + "none": true, + }, + }, + } + d := schema.TestResourceDataRaw(t, SourceTableKafka().Schema, inSourceTableKafkaNoTopic) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec( + `CREATE TABLE "database"."schema"."no_topic" + FROM SOURCE "materialize"."public"."kafka_source" + FORMAT JSON + ENVELOPE NONE;`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'no_topic'` + testhelpers.MockSourceTableKafkaScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableKafkaScan(mock, pp) + + if err := sourceTableKafkaCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} diff --git a/templates/guides/materialize_source_table.md.tmpl b/templates/guides/materialize_source_table.md.tmpl index 656a7cda..ac51670c 100644 --- a/templates/guides/materialize_source_table.md.tmpl +++ b/templates/guides/materialize_source_table.md.tmpl @@ -13,7 +13,7 @@ In previous versions of the Materialize Terraform provider, source tables were d This guide will walk you through the process of migrating your existing source table definitions to the new `materialize_source_table_{source_type}` resource. -For each source type (e.g., MySQL, Postgres, etc.), you will need to create a new `materialize_source_table_{source_type}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. For Kafka sources, you will need to create at least one `materialize_source_table_kafka` table to hold data for the kafka topic. +For each MySQL and Postgres source, you will need to create a new `materialize_source_table_{source_type}` resource for each table that was previously defined within the source resource. This ensures that the tables are preserved during the migration process. For Kafka sources, you will need to create a `materialize_source_table_kafka` table with the same name as the kafka source to contain the data for the kafka topic. ## Old Approach @@ -242,4 +242,4 @@ After importing, you may need to manually update these ignored attributes in you ## Future Improvements -The Kafka and Webhooks sources are currently being implemented. Once these changes, the migration process will be updated to include them. +Webhooks sources have not yet been migrated to the new model. Once this changes, the migration process will be updated to include them. From 9ba091e886fcb6dd5339b6bc57c99eb86554128c Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 21 Oct 2024 19:23:30 +0300 Subject: [PATCH 37/46] Fix failing tests --- integration/source.tf | 43 ++++++++++++++++++++----------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/integration/source.tf b/integration/source.tf index c3273fd4..798c8938 100644 --- a/integration/source.tf +++ b/integration/source.tf @@ -222,29 +222,6 @@ resource "materialize_source_kafka" "example_source_kafka_format_text" { depends_on = [materialize_sink_kafka.sink_kafka] } -resource "materialize_source_kafka" "example_source_kafka_no_topic" { - name = "source_kafka_no_topic" - comment = "source kafka comment no topic" - cluster_name = materialize_cluster.cluster_source.name - - kafka_connection { - name = materialize_connection_kafka.kafka_connection.name - schema_name = materialize_connection_kafka.kafka_connection.schema_name - database_name = materialize_connection_kafka.kafka_connection.database_name - } - key_format { - text = true - } - value_format { - text = true - } - expose_progress { - name = "expose_kafka" - } - - depends_on = [materialize_sink_kafka.sink_kafka] -} - # Create source table from Kafka source resource "materialize_source_table_kafka" "source_table_kafka" { name = "source_table_kafka" @@ -279,6 +256,26 @@ resource "materialize_source_table_kafka" "source_table_kafka" { } +resource "materialize_source_table_kafka" "source_table_kafka_no_topic" { + name = "source_table_kafka_no_topic" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_kafka.example_source_kafka_format_text.name + schema_name = materialize_source_kafka.example_source_kafka_format_text.schema_name + database_name = materialize_source_kafka.example_source_kafka_format_text.database_name + } + + key_format { + text = true + } + value_format { + json = true + } + +} + resource "materialize_source_kafka" "example_source_kafka_format_bytes" { name = "source_kafka_bytes" cluster_name = materialize_cluster.cluster_source.name From 3b62b79e1fadb47e1b52e1d69715e7bba077d8b7 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 21 Oct 2024 19:48:17 +0300 Subject: [PATCH 38/46] Fix failing tests --- pkg/resources/resource_source_table_kafka.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/resources/resource_source_table_kafka.go b/pkg/resources/resource_source_table_kafka.go index b60b7bae..f7b146c9 100644 --- a/pkg/resources/resource_source_table_kafka.go +++ b/pkg/resources/resource_source_table_kafka.go @@ -28,6 +28,7 @@ var sourceTableKafkaSchema = map[string]*schema.Schema{ Type: schema.TypeString, Optional: true, ForceNew: true, + Computed: true, Description: "The name of the Kafka topic in the Kafka cluster.", }, "include_key": { From 30b0db5f5cf88120a02042296a1cfef1c530d0ef Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 28 Oct 2024 19:38:10 +0200 Subject: [PATCH 39/46] Add alter source refresh to data source --- docs/data-sources/source_reference.md | 4 +- .../datasource_source_reference.go | 3 +- pkg/materialize/source_postgres.go | 38 ++-- pkg/materialize/source_reference.go | 18 +- pkg/materialize/source_reference_test.go | 95 ++++++++++ pkg/provider/acceptance_cluster_test.go | 1 - ...ptance_datasource_source_reference_test.go | 172 ++++++++++++++++++ 7 files changed, 307 insertions(+), 24 deletions(-) create mode 100644 pkg/materialize/source_reference_test.go create mode 100644 pkg/provider/acceptance_datasource_source_reference_test.go diff --git a/docs/data-sources/source_reference.md b/docs/data-sources/source_reference.md index 8535ace1..eefa72c9 100644 --- a/docs/data-sources/source_reference.md +++ b/docs/data-sources/source_reference.md @@ -3,12 +3,12 @@ page_title: "materialize_source_reference Data Source - terraform-provider-materialize" subcategory: "" description: |- - The materialize_source_reference data source retrieves information about a Materialize source's references, including details about namespaces, columns, and last update times. + The materialize_source_reference data source retrieves a list of available upstream references for a given Materialize source. These references represent potential tables that can be created based on the source, but they do not necessarily indicate references the source is already ingesting. This allows users to see all upstream data that could be materialized into tables. --- # materialize_source_reference (Data Source) -The `materialize_source_reference` data source retrieves information about a Materialize source's references, including details about namespaces, columns, and last update times. +The `materialize_source_reference` data source retrieves a list of *available* upstream references for a given Materialize source. These references represent potential tables that can be created based on the source, but they do not necessarily indicate references the source is already ingesting. This allows users to see all upstream data that could be materialized into tables. ## Example Usage diff --git a/pkg/datasources/datasource_source_reference.go b/pkg/datasources/datasource_source_reference.go index 17e16750..2b6e4572 100644 --- a/pkg/datasources/datasource_source_reference.go +++ b/pkg/datasources/datasource_source_reference.go @@ -13,7 +13,7 @@ import ( func SourceReference() *schema.Resource { return &schema.Resource{ ReadContext: sourceReferenceRead, - Description: "The `materialize_source_reference` data source retrieves information about a Materialize source's references, including details about namespaces, columns, and last update times.", + Description: "The `materialize_source_reference` data source retrieves a list of *available* upstream references for a given Materialize source. These references represent potential tables that can be created based on the source, but they do not necessarily indicate references the source is already ingesting. This allows users to see all upstream data that could be materialized into tables.", Schema: map[string]*schema.Schema{ "source_id": { Type: schema.TypeString, @@ -79,6 +79,7 @@ func SourceReference() *schema.Resource { func sourceReferenceRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { sourceID := d.Get("source_id").(string) + sourceID = utils.ExtractId(sourceID) var diags diag.Diagnostics diff --git a/pkg/materialize/source_postgres.go b/pkg/materialize/source_postgres.go index d744af9b..2c18633f 100644 --- a/pkg/materialize/source_postgres.go +++ b/pkg/materialize/source_postgres.go @@ -80,26 +80,28 @@ func (b *SourcePostgresBuilder) Create() error { q.WriteString(fmt.Sprintf(` (%s)`, p)) - q.WriteString(` FOR TABLES (`) - for i, t := range b.table { - if t.UpstreamSchemaName == "" { - t.UpstreamSchemaName = b.SchemaName - } - if t.Name == "" { - t.Name = t.UpstreamName - } - if t.SchemaName == "" { - t.SchemaName = b.SchemaName - } - if t.DatabaseName == "" { - t.DatabaseName = b.DatabaseName - } - q.WriteString(fmt.Sprintf(`%s.%s AS %s.%s.%s`, QuoteIdentifier(t.UpstreamSchemaName), QuoteIdentifier(t.UpstreamName), QuoteIdentifier(t.DatabaseName), QuoteIdentifier(t.SchemaName), QuoteIdentifier(t.Name))) - if i < len(b.table)-1 { - q.WriteString(`, `) + if b.table != nil && len(b.table) > 0 { + q.WriteString(` FOR TABLES (`) + for i, t := range b.table { + if t.UpstreamSchemaName == "" { + t.UpstreamSchemaName = b.SchemaName + } + if t.Name == "" { + t.Name = t.UpstreamName + } + if t.SchemaName == "" { + t.SchemaName = b.SchemaName + } + if t.DatabaseName == "" { + t.DatabaseName = b.DatabaseName + } + q.WriteString(fmt.Sprintf(`%s.%s AS %s.%s.%s`, QuoteIdentifier(t.UpstreamSchemaName), QuoteIdentifier(t.UpstreamName), QuoteIdentifier(t.DatabaseName), QuoteIdentifier(t.SchemaName), QuoteIdentifier(t.Name))) + if i < len(b.table)-1 { + q.WriteString(`, `) + } } + q.WriteString(`)`) } - q.WriteString(`)`) if b.exposeProgress.Name != "" { q.WriteString(fmt.Sprintf(` EXPOSE PROGRESS AS %s`, b.exposeProgress.QualifiedName())) diff --git a/pkg/materialize/source_reference.go b/pkg/materialize/source_reference.go index 49eadf9e..408d657e 100644 --- a/pkg/materialize/source_reference.go +++ b/pkg/materialize/source_reference.go @@ -2,6 +2,7 @@ package materialize import ( "database/sql" + "fmt" "github.com/jmoiron/sqlx" "github.com/lib/pq" @@ -61,9 +62,22 @@ func ScanSourceReference(conn *sqlx.DB, id string) (SourceReferenceParams, error return s, nil } -func ListSourceReferences(conn *sqlx.DB, sourceId string) ([]SourceReferenceParams, error) { +func refreshSourceReferences(conn *sqlx.DB, sourceName, schemaName, databaseName string) error { + query := fmt.Sprintf(`ALTER SOURCE %s REFRESH REFERENCES`, QualifiedName(databaseName, schemaName, sourceName)) + _, err := conn.Exec(query) + return err +} + +func ListSourceReferences(conn *sqlx.DB, id string) ([]SourceReferenceParams, error) { + source, err := ScanSource(conn, id) + if err == nil { + if err := refreshSourceReferences(conn, source.SourceName.String, source.SchemaName.String, source.DatabaseName.String); err != nil { + return nil, fmt.Errorf("error refreshing source references: %v", err) + } + } + p := map[string]string{ - "sr.source_id": sourceId, + "sr.source_id": id, } q := sourceReferenceQuery.QueryPredicate(p) diff --git a/pkg/materialize/source_reference_test.go b/pkg/materialize/source_reference_test.go new file mode 100644 index 00000000..b34bfbe4 --- /dev/null +++ b/pkg/materialize/source_reference_test.go @@ -0,0 +1,95 @@ +package materialize + +import ( + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/jmoiron/sqlx" + "github.com/lib/pq" +) + +func TestSourceReferenceId(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectQuery( + `SELECT sr\.source_id, sr\.namespace, sr\.name, sr\.updated_at, sr\.columns, s\.name AS source_name, ss\.name AS source_schema_name, sd\.name AS source_database_name, s\.type AS source_type + FROM mz_internal\.mz_source_references sr + JOIN mz_sources s ON sr\.source_id = s\.id + JOIN mz_schemas ss ON s\.schema_id = ss\.id + JOIN mz_databases sd ON ss\.database_id = sd\.id + WHERE sr\.source_id = 'test-source-id'`, + ). + WillReturnRows(sqlmock.NewRows([]string{"source_id"}).AddRow("test-source-id")) + + result, err := SourceReferenceId(db, "test-source-id") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result != "test-source-id" { + t.Errorf("expected source id to be 'test-source-id', got %v", result) + } + }) +} + +func TestScanSourceReference(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectQuery( + `SELECT sr\.source_id, sr\.namespace, sr\.name, sr\.updated_at, sr\.columns, s\.name AS source_name, ss\.name AS source_schema_name, sd\.name AS source_database_name, s\.type AS source_type + FROM mz_internal\.mz_source_references sr + JOIN mz_sources s ON sr\.source_id = s\.id + JOIN mz_schemas ss ON s\.schema_id = ss\.id + JOIN mz_databases sd ON ss\.database_id = sd\.id + WHERE sr\.source_id = 'test-source-id'`, + ). + WillReturnRows(sqlmock.NewRows([]string{"source_id", "namespace", "name", "updated_at", "columns", "source_name", "source_schema_name", "source_database_name", "source_type"}). + AddRow("test-source-id", "test-namespace", "test-name", "2024-10-28", pq.StringArray{"col1", "col2"}, "source-name", "source-schema-name", "source-database-name", "source-type")) + + result, err := ScanSourceReference(db, "test-source-id") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if result.SourceId.String != "test-source-id" { + t.Errorf("expected source id to be 'test-source-id', got %v", result.SourceId.String) + } + }) +} + +func TestRefreshSourceReferences(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `ALTER SOURCE "test-database"\."test-schema"\."test-source" REFRESH REFERENCES`, + ). + WillReturnResult(sqlmock.NewResult(1, 1)) + + err := refreshSourceReferences(db, "test-source", "test-schema", "test-database") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) +} + +func TestListSourceReferences(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectQuery( + `SELECT sr\.source_id, sr\.namespace, sr\.name, sr\.updated_at, sr\.columns, s\.name AS source_name, ss\.name AS source_schema_name, sd\.name AS source_database_name, s\.type AS source_type + FROM mz_internal\.mz_source_references sr + JOIN mz_sources s ON sr\.source_id = s\.id + JOIN mz_schemas ss ON s\.schema_id = ss\.id + JOIN mz_databases sd ON ss\.database_id = sd\.id + WHERE sr\.source_id = 'test-source-id'`, + ). + WillReturnRows(sqlmock.NewRows([]string{"source_id", "namespace", "name", "updated_at", "columns", "source_name", "source_schema_name", "source_database_name", "source_type"}). + AddRow("test-source-id", "test-namespace", "test-name", "2024-10-28", pq.StringArray{"col1", "col2"}, "source-name", "source-schema-name", "source-database-name", "source-type")) + + result, err := ListSourceReferences(db, "test-source-id") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result) != 1 { + t.Errorf("expected 1 result, got %d", len(result)) + } + if result[0].SourceId.String != "test-source-id" { + t.Errorf("expected source id to be 'test-source-id', got %v", result[0].SourceId.String) + } + }) +} diff --git a/pkg/provider/acceptance_cluster_test.go b/pkg/provider/acceptance_cluster_test.go index 0ab70632..83182933 100644 --- a/pkg/provider/acceptance_cluster_test.go +++ b/pkg/provider/acceptance_cluster_test.go @@ -459,7 +459,6 @@ func testAccManagedClusterResourceAlterGraceful(clusterName, clusterSize string, enabled = true timeout = "10m" on_timeout = "%[4]s" - } } `, diff --git a/pkg/provider/acceptance_datasource_source_reference_test.go b/pkg/provider/acceptance_datasource_source_reference_test.go new file mode 100644 index 00000000..c20a44f2 --- /dev/null +++ b/pkg/provider/acceptance_datasource_source_reference_test.go @@ -0,0 +1,172 @@ +package provider + +import ( + "fmt" + "testing" + + "github.com/hashicorp/terraform-plugin-testing/helper/acctest" + "github.com/hashicorp/terraform-plugin-testing/helper/resource" +) + +func TestAccDataSourceSourceReference_basic(t *testing.T) { + addTestTopic() + nameSpace := acctest.RandStringFromCharSet(10, acctest.CharSetAlpha) + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactories, + Steps: []resource.TestStep{ + { + Config: testAccDataSourceSourceReferenceConfig(nameSpace), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttrSet("data.materialize_source_reference.kafka", "source_id"), + resource.TestCheckResourceAttrSet("data.materialize_source_reference.postgres", "source_id"), + resource.TestCheckResourceAttrSet("data.materialize_source_reference.mysql", "source_id"), + + // Check total references + resource.TestCheckResourceAttr("data.materialize_source_reference.kafka", "references.#", "1"), + resource.TestCheckResourceAttr("data.materialize_source_reference.postgres", "references.#", "3"), + resource.TestCheckResourceAttr("data.materialize_source_reference.mysql", "references.#", "4"), + + // Check Postgres reference attributes + resource.TestCheckResourceAttr("data.materialize_source_reference.postgres", "references.0.namespace", "public"), + resource.TestCheckResourceAttrSet("data.materialize_source_reference.postgres", "references.0.name"), + resource.TestCheckResourceAttr("data.materialize_source_reference.postgres", "references.0.source_name", fmt.Sprintf("%s_source_postgres", nameSpace)), + resource.TestCheckResourceAttr("data.materialize_source_reference.postgres", "references.0.source_type", "postgres"), + resource.TestCheckResourceAttrSet("data.materialize_source_reference.postgres", "references.0.updated_at"), + + // Check MySQL reference attributes + resource.TestCheckResourceAttr("data.materialize_source_reference.mysql", "references.0.namespace", "shop"), + resource.TestCheckResourceAttrSet("data.materialize_source_reference.mysql", "references.0.name"), + resource.TestCheckResourceAttr("data.materialize_source_reference.mysql", "references.0.source_name", fmt.Sprintf("%s_source_mysql", nameSpace)), + resource.TestCheckResourceAttr("data.materialize_source_reference.mysql", "references.1.source_type", "mysql"), + resource.TestCheckResourceAttrSet("data.materialize_source_reference.mysql", "references.1.updated_at"), + + // Check Kafka reference attributes + resource.TestCheckResourceAttr("data.materialize_source_reference.kafka", "references.0.name", "terraform"), + resource.TestCheckResourceAttr("data.materialize_source_reference.kafka", "references.0.source_name", fmt.Sprintf("%s_source_kafka", nameSpace)), + resource.TestCheckResourceAttr("data.materialize_source_reference.kafka", "references.0.source_type", "kafka"), + resource.TestCheckResourceAttrSet("data.materialize_source_reference.kafka", "references.0.updated_at"), + ), + }, + }, + }) +} + +func testAccDataSourceSourceReferenceConfig(nameSpace string) string { + return fmt.Sprintf(` + // Postgres setup + resource "materialize_secret" "postgres_password" { + name = "%[1]s_secret_postgres" + value = "c2VjcmV0Cg==" + } + + resource "materialize_connection_postgres" "postgres_connection" { + name = "%[1]s_connection_postgres" + host = "postgres" + port = 5432 + user { + text = "postgres" + } + password { + name = materialize_secret.postgres_password.name + } + database = "postgres" + } + + resource "materialize_source_postgres" "test_source_postgres" { + name = "%[1]s_source_postgres" + cluster_name = "quickstart" + + postgres_connection { + name = materialize_connection_postgres.postgres_connection.name + } + publication = "mz_source" + } + + resource "materialize_source_table_postgres" "table_from_source_pg" { + name = "%[1]s_table" + schema_name = "public" + database_name = "materialize" + + source { + name = materialize_source_postgres.test_source_postgres.name + } + + upstream_name = "table2" + upstream_schema_name = "public" + } + + // MySQL setup + resource "materialize_secret" "mysql_password" { + name = "%[1]s_secret_mysql" + value = "c2VjcmV0Cg==" + } + + resource "materialize_connection_mysql" "mysql_connection" { + name = "%[1]s_connection_mysql" + host = "mysql" + port = 3306 + user { + text = "repluser" + } + password { + name = materialize_secret.mysql_password.name + } + } + + resource "materialize_source_mysql" "test_source_mysql" { + name = "%[1]s_source_mysql" + cluster_name = "quickstart" + + mysql_connection { + name = materialize_connection_mysql.mysql_connection.name + } + } + + // Kafka setup + resource "materialize_connection_kafka" "kafka_connection" { + name = "%[1]s_connection_kafka" + kafka_broker { + broker = "redpanda:9092" + } + security_protocol = "PLAINTEXT" + } + + resource "materialize_source_kafka" "test_source_kafka" { + name = "%[1]s_source_kafka" + cluster_name = "quickstart" + topic = "terraform" + + kafka_connection { + name = materialize_connection_kafka.kafka_connection.name + } + value_format { + json = true + } + key_format { + json = true + } + } + + data "materialize_source_reference" "kafka" { + source_id = materialize_source_kafka.test_source_kafka.id + depends_on = [ + materialize_source_kafka.test_source_kafka + ] + } + + data "materialize_source_reference" "postgres" { + source_id = materialize_source_postgres.test_source_postgres.id + depends_on = [ + materialize_source_postgres.test_source_postgres + ] + } + + data "materialize_source_reference" "mysql" { + source_id = materialize_source_mysql.test_source_mysql.id + depends_on = [ + materialize_source_mysql.test_source_mysql + ] + } + `, nameSpace) +} From e9573b392262a4880b0c293a06cb115a8d17f689 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 28 Oct 2024 21:31:11 +0200 Subject: [PATCH 40/46] Add new kafk acolumns from mz_kafka_source_tables --- pkg/materialize/source_table_kafka.go | 76 +++++++++++--------- pkg/resources/resource_source_table_kafka.go | 2 + 2 files changed, 43 insertions(+), 35 deletions(-) diff --git a/pkg/materialize/source_table_kafka.go b/pkg/materialize/source_table_kafka.go index 233d832d..ce933225 100644 --- a/pkg/materialize/source_table_kafka.go +++ b/pkg/materialize/source_table_kafka.go @@ -9,44 +9,50 @@ import ( type SourceTableKafkaParams struct { SourceTableParams + EnvelopeType string `db:"envelope_type"` + KeyFormat string `db:"key_format"` + ValueFormat string `db:"value_format"` } var sourceTableKafkaQuery = ` - SELECT - mz_tables.id, - mz_tables.name, - mz_schemas.name AS schema_name, - mz_databases.name AS database_name, - mz_sources.name AS source_name, - source_schemas.name AS source_schema_name, - source_databases.name AS source_database_name, - mz_kafka_source_tables.topic AS upstream_table_name, - mz_sources.type AS source_type, - comments.comment AS comment, - mz_roles.name AS owner_name, - mz_tables.privileges - FROM mz_tables - JOIN mz_schemas - ON mz_tables.schema_id = mz_schemas.id - JOIN mz_databases - ON mz_schemas.database_id = mz_databases.id - JOIN mz_sources - ON mz_tables.source_id = mz_sources.id - JOIN mz_schemas AS source_schemas - ON mz_sources.schema_id = source_schemas.id - JOIN mz_databases AS source_databases - ON source_schemas.database_id = source_databases.id - LEFT JOIN mz_internal.mz_kafka_source_tables - ON mz_tables.id = mz_kafka_source_tables.id - JOIN mz_roles - ON mz_tables.owner_id = mz_roles.id - LEFT JOIN ( - SELECT id, comment - FROM mz_internal.mz_comments - WHERE object_type = 'table' - AND object_sub_id IS NULL - ) comments - ON mz_tables.id = comments.id + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.name AS source_name, + source_schemas.name AS source_schema_name, + source_databases.name AS source_database_name, + mz_kafka_source_tables.topic AS upstream_table_name, + mz_kafka_source_tables.envelope_type, + mz_kafka_source_tables.key_format, + mz_kafka_source_tables.value_format, + mz_sources.type AS source_type, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_sources + ON mz_tables.source_id = mz_sources.id + JOIN mz_schemas AS source_schemas + ON mz_sources.schema_id = source_schemas.id + JOIN mz_databases AS source_databases + ON source_schemas.database_id = source_databases.id + LEFT JOIN mz_internal.mz_kafka_source_tables + ON mz_tables.id = mz_kafka_source_tables.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN ( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + ) comments + ON mz_tables.id = comments.id ` func SourceTableKafkaId(conn *sqlx.DB, obj MaterializeObject) (string, error) { diff --git a/pkg/resources/resource_source_table_kafka.go b/pkg/resources/resource_source_table_kafka.go index f7b146c9..d1adf4a5 100644 --- a/pkg/resources/resource_source_table_kafka.go +++ b/pkg/resources/resource_source_table_kafka.go @@ -361,6 +361,8 @@ func sourceTableKafkaRead(ctx context.Context, d *schema.ResourceData, meta inte return diag.FromErr(err) } + // TODO: include envelope_type, key_format and value_format from mz_internal.mz_kafka_source_tables + if err := d.Set("ownership_role", t.OwnerName.String); err != nil { return diag.FromErr(err) } From 74410c0e540d6e0eca8a2380199855f975b53ada Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 28 Oct 2024 21:36:37 +0200 Subject: [PATCH 41/46] Fix failing MockSourceTableKafkaScan test --- pkg/materialize/source_table_kafka.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pkg/materialize/source_table_kafka.go b/pkg/materialize/source_table_kafka.go index ce933225..a6d7b41d 100644 --- a/pkg/materialize/source_table_kafka.go +++ b/pkg/materialize/source_table_kafka.go @@ -9,9 +9,6 @@ import ( type SourceTableKafkaParams struct { SourceTableParams - EnvelopeType string `db:"envelope_type"` - KeyFormat string `db:"key_format"` - ValueFormat string `db:"value_format"` } var sourceTableKafkaQuery = ` @@ -24,9 +21,6 @@ var sourceTableKafkaQuery = ` source_schemas.name AS source_schema_name, source_databases.name AS source_database_name, mz_kafka_source_tables.topic AS upstream_table_name, - mz_kafka_source_tables.envelope_type, - mz_kafka_source_tables.key_format, - mz_kafka_source_tables.value_format, mz_sources.type AS source_type, comments.comment AS comment, mz_roles.name AS owner_name, @@ -63,7 +57,7 @@ func SourceTableKafkaId(conn *sqlx.DB, obj MaterializeObject) (string, error) { } q := NewBaseQuery(sourceTableKafkaQuery).QueryPredicate(p) - var t SourceTableParams + var t SourceTableKafkaParams if err := conn.Get(&t, q); err != nil { return "", err } From 6b3af39508c54eff18f6cd3dc0126ad76d3aa60d Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Tue, 29 Oct 2024 18:48:09 +0200 Subject: [PATCH 42/46] Remove confusing line from migration guide --- docs/guides/materialize_source_table.md | 2 -- templates/guides/materialize_source_table.md.tmpl | 2 -- 2 files changed, 4 deletions(-) diff --git a/docs/guides/materialize_source_table.md b/docs/guides/materialize_source_table.md index 3e5301af..1cb57137 100644 --- a/docs/guides/materialize_source_table.md +++ b/docs/guides/materialize_source_table.md @@ -178,8 +178,6 @@ resource "materialize_source_kafka" "kafka_source" { In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source_type}` resources. -> Note: Once the migration process is fully implemented on the Materialize side and the attributes will have to be updated as no-op in future versions of the provider. That way the `ignore_changes` block will no longer be required. At that point, Terraform will correctly handle these attributes without needing the extra lifecycle configuration. Keep an eye on upcoming releases for this change. - ### Step 4: Update Terraform State After removing the `table` blocks and the table/topic specific attributes from your source resources, run `terraform plan` and `terraform apply` again to update the Terraform state and apply the changes. diff --git a/templates/guides/materialize_source_table.md.tmpl b/templates/guides/materialize_source_table.md.tmpl index ac51670c..a41d6bea 100644 --- a/templates/guides/materialize_source_table.md.tmpl +++ b/templates/guides/materialize_source_table.md.tmpl @@ -178,8 +178,6 @@ resource "materialize_source_kafka" "kafka_source" { In the `lifecycle` block, add the `ignore_changes` meta-argument to prevent Terraform from trying to update these attributes during subsequent applies, that way Terraform won't try to update these values based on incomplete information from the state as they will no longer be defined in the source resource itself but in the new `materialize_source_table_{source_type}` resources. -> Note: Once the migration process is fully implemented on the Materialize side and the attributes will have to be updated as no-op in future versions of the provider. That way the `ignore_changes` block will no longer be required. At that point, Terraform will correctly handle these attributes without needing the extra lifecycle configuration. Keep an eye on upcoming releases for this change. - ### Step 4: Update Terraform State After removing the `table` blocks and the table/topic specific attributes from your source resources, run `terraform plan` and `terraform apply` again to update the Terraform state and apply the changes. From 001988a55c4ce362fd754fc50ea3613786142902 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 18 Nov 2024 10:37:05 +0200 Subject: [PATCH 43/46] Remove a left behind comment --- pkg/provider/acceptance_source_table_kafka_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/provider/acceptance_source_table_kafka_test.go b/pkg/provider/acceptance_source_table_kafka_test.go index 4ddfc6e6..37c6d8d9 100644 --- a/pkg/provider/acceptance_source_table_kafka_test.go +++ b/pkg/provider/acceptance_source_table_kafka_test.go @@ -25,7 +25,6 @@ func TestAccSourceTableKafka_basic(t *testing.T) { resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "name", nameSpace+"_table_kafka"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "database_name", "materialize"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "schema_name", "public"), - // resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "qualified_sql_name", fmt.Sprintf(`"materialize"."public"."%s_table_kafka"`, nameSpace)), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "topic", "terraform"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_key", "true"), resource.TestCheckResourceAttr("materialize_source_table_kafka.test_kafka", "include_key_alias", "message_key"), From 285933f46b76dd3f5c11eba4716efad3ac37c6ac Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Mon, 18 Nov 2024 11:01:52 +0200 Subject: [PATCH 44/46] explicitly enable create table from source as --all-features is not working --- compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose.yaml b/compose.yaml index f1e6d7d0..a63691d8 100644 --- a/compose.yaml +++ b/compose.yaml @@ -19,6 +19,7 @@ services: - --system-parameter-default=max_clusters=100 - --system-parameter-default=max_sources=100 - --system-parameter-default=max_aws_privatelink_connections=10 + - --system-parameter-default=enable_create_table_from_source=on - --all-features environment: MZ_NO_TELEMETRY: 1 @@ -48,6 +49,7 @@ services: - --system-parameter-default=max_sources=100 - --system-parameter-default=max_aws_privatelink_connections=10 - --system-parameter-default=transaction_isolation=serializable + - --system-parameter-default=enable_create_table_from_source=on - --all-features environment: MZ_NO_TELEMETRY: 1 From ef608741a18902ebbfec997f22d6cfe954fcb009 Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 3 Jan 2025 16:51:49 +0200 Subject: [PATCH 45/46] Generate docs --- docs/resources/source_table_kafka.md | 2 +- docs/resources/source_table_load_generator.md | 2 +- docs/resources/source_table_mysql.md | 2 +- docs/resources/source_table_postgres.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/resources/source_table_kafka.md b/docs/resources/source_table_kafka.md index 73bee1b4..3c5e1859 100644 --- a/docs/resources/source_table_kafka.md +++ b/docs/resources/source_table_kafka.md @@ -71,7 +71,7 @@ resource "materialize_source_table_kafka" "kafka_source_table" { ### Optional -- `comment` (String) **Public Preview** Comment on an object in the database. +- `comment` (String) Comment on an object in the database. - `database_name` (String) The identifier for the source table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `envelope` (Block List, Max: 1) How Materialize should interpret records (e.g. append-only, upsert).. (see [below for nested schema](#nestedblock--envelope)) - `expose_progress` (Block List, Max: 1) The name of the progress collection for the source. If this is not specified, the collection will be named `_progress`. (see [below for nested schema](#nestedblock--expose_progress)) diff --git a/docs/resources/source_table_load_generator.md b/docs/resources/source_table_load_generator.md index aba578f2..c422cc5f 100644 --- a/docs/resources/source_table_load_generator.md +++ b/docs/resources/source_table_load_generator.md @@ -41,7 +41,7 @@ resource "materialize_source_table_load_generator" "load_generator_table_from_so ### Optional -- `comment` (String) **Public Preview** Comment on an object in the database. +- `comment` (String) Comment on an object in the database. - `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. diff --git a/docs/resources/source_table_mysql.md b/docs/resources/source_table_mysql.md index e77c85ad..163fd51b 100644 --- a/docs/resources/source_table_mysql.md +++ b/docs/resources/source_table_mysql.md @@ -46,7 +46,7 @@ resource "materialize_source_table_mysql" "mysql_table_from_source" { ### Optional -- `comment` (String) **Public Preview** Comment on an object in the database. +- `comment` (String) Comment on an object in the database. - `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `exclude_columns` (List of String) Exclude specific columns when reading data from MySQL. This option used to be called `ignore_columns`. - `ownership_role` (String) The owernship role of the object. diff --git a/docs/resources/source_table_postgres.md b/docs/resources/source_table_postgres.md index 53ba1334..ffe107c3 100644 --- a/docs/resources/source_table_postgres.md +++ b/docs/resources/source_table_postgres.md @@ -45,7 +45,7 @@ resource "materialize_source_table_postgres" "postgres_table_from_source" { ### Optional -- `comment` (String) **Public Preview** Comment on an object in the database. +- `comment` (String) Comment on an object in the database. - `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. - `ownership_role` (String) The owernship role of the object. - `region` (String) The region to use for the resource connection. If not set, the default region is used. From efc1ae7e663ac445fbaae276e3936456771ee83e Mon Sep 17 00:00:00 2001 From: Bobby Iliev Date: Fri, 17 Jan 2025 14:42:57 +0200 Subject: [PATCH 46/46] add initial support for CREATE TABLE ... FROM WEBHOOK --- docs/resources/source_table_webhook.md | 144 +++++++ docs/resources/source_webhook.md | 4 +- .../import.sh | 5 + .../resource.tf | 33 ++ pkg/materialize/source_table_webhook.go | 231 +++++++++++ pkg/materialize/source_table_webhook_test.go | 209 ++++++++++ pkg/provider/provider.go | 1 + .../resource_source_table_webhook.go | 367 ++++++++++++++++++ .../resource_source_table_webhook_test.go | 116 ++++++ pkg/resources/resource_source_webhook.go | 7 +- pkg/testhelpers/mock_scans.go | 32 ++ 11 files changed, 1146 insertions(+), 3 deletions(-) create mode 100644 docs/resources/source_table_webhook.md create mode 100644 examples/resources/materialize_source_table_webhook/import.sh create mode 100644 examples/resources/materialize_source_table_webhook/resource.tf create mode 100644 pkg/materialize/source_table_webhook.go create mode 100644 pkg/materialize/source_table_webhook_test.go create mode 100644 pkg/resources/resource_source_table_webhook.go create mode 100644 pkg/resources/resource_source_table_webhook_test.go diff --git a/docs/resources/source_table_webhook.md b/docs/resources/source_table_webhook.md new file mode 100644 index 00000000..cc7e1414 --- /dev/null +++ b/docs/resources/source_table_webhook.md @@ -0,0 +1,144 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "materialize_source_table_webhook Resource - terraform-provider-materialize" +subcategory: "" +description: |- + A webhook source table allows reading data directly from webhooks. +--- + +# materialize_source_table_webhook (Resource) + +A webhook source table allows reading data directly from webhooks. + +## Example Usage + +```terraform +resource "materialize_source_table_webhook" "example_webhook" { + name = "example_webhook" + body_format = "json" + check_expression = "headers->'x-mz-api-key' = secret" + include_headers { + not = ["x-mz-api-key"] + } + + check_options { + field { + headers = true + } + } + + check_options { + field { + secret { + name = materialize_secret.password.name + database_name = materialize_secret.password.database_name + schema_name = materialize_secret.password.schema_name + } + } + alias = "secret" + } +} + +# CREATE TABLE example_webhook FROM WEBHOOK +# BODY FORMAT json +# INCLUDE HEADERS ( NOT 'x-mz-api-key' ) +# CHECK ( +# WITH ( HEADERS, SECRET materialize.public.password AS secret) +# headers->'x-mz-api-key' = secret +# ); +``` + + +## Schema + +### Required + +- `body_format` (String) The body format of the webhook. +- `name` (String) The identifier for the table. + +### Optional + +- `check_expression` (String) The check expression for the webhook. +- `check_options` (Block List) The check options for the webhook. (see [below for nested schema](#nestedblock--check_options)) +- `comment` (String) Comment on an object in the database. +- `database_name` (String) The identifier for the table database in Materialize. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `include_header` (Block List) Map a header value from a request into a column. (see [below for nested schema](#nestedblock--include_header)) +- `include_headers` (Block List, Max: 1) Include headers in the webhook. (see [below for nested schema](#nestedblock--include_headers)) +- `ownership_role` (String) The owernship role of the object. +- `region` (String) The region to use for the resource connection. If not set, the default region is used. +- `schema_name` (String) The identifier for the table schema in Materialize. Defaults to `public`. + +### Read-Only + +- `id` (String) The ID of this resource. +- `qualified_sql_name` (String) The fully qualified name of the table. + + +### Nested Schema for `check_options` + +Required: + +- `field` (Block List, Min: 1, Max: 1) The field for the check options. (see [below for nested schema](#nestedblock--check_options--field)) + +Optional: + +- `alias` (String) The alias for the check options. +- `bytes` (Boolean) Change type to `bytea`. + + +### Nested Schema for `check_options.field` + +Optional: + +- `body` (Boolean) The body for the check options. +- `headers` (Boolean) The headers for the check options. +- `secret` (Block List, Max: 1) The secret for the check options. (see [below for nested schema](#nestedblock--check_options--field--secret)) + + +### Nested Schema for `check_options.field.secret` + +Required: + +- `name` (String) The secret name. + +Optional: + +- `database_name` (String) The secret database name. Defaults to `MZ_DATABASE` environment variable if set or `materialize` if environment variable is not set. +- `schema_name` (String) The secret schema name. Defaults to `public`. + + + + + +### Nested Schema for `include_header` + +Required: + +- `header` (String) The name for the header. + +Optional: + +- `alias` (String) The alias for the header. +- `bytes` (Boolean) Change type to `bytea`. + + + +### Nested Schema for `include_headers` + +Optional: + +- `all` (Boolean) Include all headers. +- `not` (List of String) Headers that should be excluded. +- `only` (List of String) Headers that should be included. + +## Import + +Import is supported using the following syntax: + +```shell +# Source tables can be imported using the source table id: +terraform import materialize_source_table_webhook.example_source_table_webhook : + +# Source id and information be found in the `mz_catalog.mz_tables` table +# The region is the region where the database is located (e.g. aws/us-east-1) +``` diff --git a/docs/resources/source_webhook.md b/docs/resources/source_webhook.md index 5b9e1024..091737d8 100644 --- a/docs/resources/source_webhook.md +++ b/docs/resources/source_webhook.md @@ -3,12 +3,12 @@ page_title: "materialize_source_webhook Resource - terraform-provider-materialize" subcategory: "" description: |- - A webhook source describes a webhook you want Materialize to read data from. + A webhook source describes a webhook you want Materialize to read data from. This resource is deprecated and will be removed in a future release. Please use materialize_source_table_webhook instead. --- # materialize_source_webhook (Resource) -A webhook source describes a webhook you want Materialize to read data from. +A webhook source describes a webhook you want Materialize to read data from. This resource is deprecated and will be removed in a future release. Please use materialize_source_table_webhook instead. ## Example Usage diff --git a/examples/resources/materialize_source_table_webhook/import.sh b/examples/resources/materialize_source_table_webhook/import.sh new file mode 100644 index 00000000..6d13c91c --- /dev/null +++ b/examples/resources/materialize_source_table_webhook/import.sh @@ -0,0 +1,5 @@ +# Source tables can be imported using the source table id: +terraform import materialize_source_table_webhook.example_source_table_webhook : + +# Source id and information be found in the `mz_catalog.mz_tables` table +# The region is the region where the database is located (e.g. aws/us-east-1) diff --git a/examples/resources/materialize_source_table_webhook/resource.tf b/examples/resources/materialize_source_table_webhook/resource.tf new file mode 100644 index 00000000..81d2053f --- /dev/null +++ b/examples/resources/materialize_source_table_webhook/resource.tf @@ -0,0 +1,33 @@ +resource "materialize_source_table_webhook" "example_webhook" { + name = "example_webhook" + body_format = "json" + check_expression = "headers->'x-mz-api-key' = secret" + include_headers { + not = ["x-mz-api-key"] + } + + check_options { + field { + headers = true + } + } + + check_options { + field { + secret { + name = materialize_secret.password.name + database_name = materialize_secret.password.database_name + schema_name = materialize_secret.password.schema_name + } + } + alias = "secret" + } +} + +# CREATE TABLE example_webhook FROM WEBHOOK +# BODY FORMAT json +# INCLUDE HEADERS ( NOT 'x-mz-api-key' ) +# CHECK ( +# WITH ( HEADERS, SECRET materialize.public.password AS secret) +# headers->'x-mz-api-key' = secret +# ); diff --git a/pkg/materialize/source_table_webhook.go b/pkg/materialize/source_table_webhook.go new file mode 100644 index 00000000..6bf93408 --- /dev/null +++ b/pkg/materialize/source_table_webhook.go @@ -0,0 +1,231 @@ +package materialize + +import ( + "fmt" + "strings" + + "github.com/jmoiron/sqlx" +) + +// SourceTableWebhookParams contains the parameters for a webhook source table +type SourceTableWebhookParams struct { + SourceTableParams +} + +// Query to get webhook source table information +var sourceTableWebhookQuery = ` + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.type AS source_type, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN ( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + ) comments + ON mz_tables.id = comments.id +` + +// SourceTableWebhookId retrieves the ID of a webhook source table +func SourceTableWebhookId(conn *sqlx.DB, obj MaterializeObject) (string, error) { + p := map[string]string{ + "mz_tables.name": obj.Name, + "mz_schemas.name": obj.SchemaName, + "mz_databases.name": obj.DatabaseName, + } + q := NewBaseQuery(sourceTableWebhookQuery).QueryPredicate(p) + + var t SourceTableParams + if err := conn.Get(&t, q); err != nil { + return "", err + } + + return t.TableId.String, nil +} + +// ScanSourceTableWebhook scans a webhook source table by ID +func ScanSourceTableWebhook(conn *sqlx.DB, id string) (SourceTableWebhookParams, error) { + q := NewBaseQuery(sourceTableWebhookQuery).QueryPredicate(map[string]string{"mz_tables.id": id}) + + var params SourceTableWebhookParams + if err := conn.Get(¶ms, q); err != nil { + return params, err + } + + return params, nil +} + +// SourceTableWebhookBuilder builds webhook source tables +type SourceTableWebhookBuilder struct { + ddl Builder + tableName string + schemaName string + databaseName string + bodyFormat string + includeHeader []HeaderStruct + includeHeaders IncludeHeadersStruct + checkOptions []CheckOptionsStruct + checkExpression string +} + +// NewSourceTableWebhookBuilder creates a new webhook source table builder +func NewSourceTableWebhookBuilder(conn *sqlx.DB, obj MaterializeObject) *SourceTableWebhookBuilder { + return &SourceTableWebhookBuilder{ + ddl: Builder{conn, Table}, + tableName: obj.Name, + schemaName: obj.SchemaName, + databaseName: obj.DatabaseName, + } +} + +// QualifiedName returns the fully qualified name of the table +func (b *SourceTableWebhookBuilder) QualifiedName() string { + return QualifiedName(b.databaseName, b.schemaName, b.tableName) +} + +// BodyFormat sets the body format +func (b *SourceTableWebhookBuilder) BodyFormat(f string) *SourceTableWebhookBuilder { + b.bodyFormat = f + return b +} + +// IncludeHeader adds header inclusions +func (b *SourceTableWebhookBuilder) IncludeHeader(h []HeaderStruct) *SourceTableWebhookBuilder { + b.includeHeader = h + return b +} + +// IncludeHeaders sets headers to include +func (b *SourceTableWebhookBuilder) IncludeHeaders(h IncludeHeadersStruct) *SourceTableWebhookBuilder { + b.includeHeaders = h + return b +} + +// CheckOptions sets the check options +func (b *SourceTableWebhookBuilder) CheckOptions(o []CheckOptionsStruct) *SourceTableWebhookBuilder { + b.checkOptions = o + return b +} + +// CheckExpression sets the check expression +func (b *SourceTableWebhookBuilder) CheckExpression(e string) *SourceTableWebhookBuilder { + b.checkExpression = e + return b +} + +// Drop removes the webhook source table +func (b *SourceTableWebhookBuilder) Drop() error { + qn := b.QualifiedName() + return b.ddl.drop(qn) +} + +func (b *SourceTableWebhookBuilder) Rename(newName string) error { + oldName := b.QualifiedName() + b.tableName = newName + newName = b.QualifiedName() + return b.ddl.rename(oldName, newName) +} + +// Create creates the webhook source table +func (b *SourceTableWebhookBuilder) Create() error { + q := strings.Builder{} + q.WriteString(fmt.Sprintf(`CREATE TABLE %s FROM WEBHOOK`, b.QualifiedName())) + + // Add webhook-specific options + var options []string + + // Body Format + options = append(options, fmt.Sprintf(`BODY FORMAT %s`, b.bodyFormat)) + + // Include Header + if len(b.includeHeader) > 0 { + for _, h := range b.includeHeader { + headerOption := fmt.Sprintf(`INCLUDE HEADER %s`, QuoteString(h.Header)) + if h.Alias != "" { + headerOption += fmt.Sprintf(` AS %s`, h.Alias) + } + if h.Bytes { + headerOption += ` BYTES` + } + options = append(options, headerOption) + } + } + + // Include Headers + if b.includeHeaders.All || len(b.includeHeaders.Only) > 0 || len(b.includeHeaders.Not) > 0 { + headerOption := `INCLUDE HEADERS` + var headers []string + + for _, h := range b.includeHeaders.Only { + headers = append(headers, QuoteString(h)) + } + for _, h := range b.includeHeaders.Not { + headers = append(headers, fmt.Sprintf("NOT %s", QuoteString(h))) + } + + if len(headers) > 0 { + headerOption += fmt.Sprintf(` (%s)`, strings.Join(headers, ", ")) + } + options = append(options, headerOption) + } + + // Check Options and Expression + if len(b.checkOptions) > 0 || b.checkExpression != "" { + checkOption := "CHECK (" + + if len(b.checkOptions) > 0 { + var checkOpts []string + for _, opt := range b.checkOptions { + var o string + if opt.Field.Body { + o = "BODY" + } + if opt.Field.Headers { + o = "HEADERS" + } + if opt.Field.Secret.Name != "" { + o = "SECRET " + opt.Field.Secret.QualifiedName() + } + if opt.Alias != "" { + o += fmt.Sprintf(" AS %s", opt.Alias) + } + if opt.Bytes { + o += " BYTES" + } + checkOpts = append(checkOpts, o) + } + checkOption += fmt.Sprintf(" WITH (%s)", strings.Join(checkOpts, ", ")) + } + + if b.checkExpression != "" { + if len(b.checkOptions) > 0 { + checkOption += " " + } + checkOption += b.checkExpression + } + + checkOption += ")" + options = append(options, checkOption) + } + + if len(options) > 0 { + q.WriteString(" ") + q.WriteString(strings.Join(options, " ")) + } + + q.WriteString(";") + return b.ddl.exec(q.String()) +} diff --git a/pkg/materialize/source_table_webhook_test.go b/pkg/materialize/source_table_webhook_test.go new file mode 100644 index 00000000..2d981b77 --- /dev/null +++ b/pkg/materialize/source_table_webhook_test.go @@ -0,0 +1,209 @@ +package materialize + +import ( + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/jmoiron/sqlx" +) + +var sourceTableWebhook = MaterializeObject{Name: "webhook_table", SchemaName: "schema", DatabaseName: "database"} + +func TestSourceTableWebhookCreateExposeHeaders(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."webhook_table" + FROM WEBHOOK BODY FORMAT JSON INCLUDE HEADER 'timestamp' AS ts + INCLUDE HEADER 'x-event-type' AS event_type;`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + var includeHeader = []HeaderStruct{ + { + Header: "timestamp", + Alias: "ts", + }, + { + Header: "x-event-type", + Alias: "event_type", + }, + } + + b := NewSourceTableWebhookBuilder(db, sourceTableWebhook) + b.BodyFormat("JSON") + b.IncludeHeader(includeHeader) + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableWebhookCreateIncludeHeaders(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."webhook_table" + FROM WEBHOOK BODY FORMAT JSON INCLUDE HEADERS \(NOT 'authorization', NOT 'x-api-key'\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableWebhookBuilder(db, sourceTableWebhook) + b.BodyFormat("JSON") + b.IncludeHeaders(IncludeHeadersStruct{ + Not: []string{"authorization", "x-api-key"}, + }) + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableWebhookCreateValidated(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."webhook_table" + FROM WEBHOOK BODY FORMAT JSON CHECK + \( WITH \(HEADERS, BODY AS request_body, SECRET "database"."schema"."my_webhook_shared_secret"\) + decode\(headers->'x-signature', 'base64'\) = hmac\(request_body, my_webhook_shared_secret, 'sha256'\)\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + var checkOptions = []CheckOptionsStruct{ + { + Field: FieldStruct{Headers: true}, + }, + { + Field: FieldStruct{Body: true}, + Alias: "request_body", + }, + { + Field: FieldStruct{ + Secret: IdentifierSchemaStruct{ + DatabaseName: "database", + SchemaName: "schema", + Name: "my_webhook_shared_secret", + }, + }, + }, + } + + b := NewSourceTableWebhookBuilder(db, sourceTableWebhook) + b.BodyFormat("JSON") + b.CheckOptions(checkOptions) + b.CheckExpression("decode(headers->'x-signature', 'base64') = hmac(request_body, my_webhook_shared_secret, 'sha256')") + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableWebhookCreateSegment(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."webhook_table" + FROM WEBHOOK BODY FORMAT JSON INCLUDE HEADER 'event-type' AS event_type INCLUDE HEADERS CHECK + \( WITH \(BODY BYTES, HEADERS, SECRET "database"."schema"."my_webhook_shared_secret" AS secret BYTES\) + decode\(headers->'x-signature', 'hex'\) = hmac\(body, secret, 'sha1'\)\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + var includeHeader = []HeaderStruct{ + { + Header: "event-type", + Alias: "event_type", + }, + } + var checkOptions = []CheckOptionsStruct{ + { + Field: FieldStruct{Body: true}, + Bytes: true, + }, + { + Field: FieldStruct{Headers: true}, + }, + { + Field: FieldStruct{ + Secret: IdentifierSchemaStruct{ + DatabaseName: "database", + SchemaName: "schema", + Name: "my_webhook_shared_secret", + }, + }, + Alias: "secret", + Bytes: true, + }, + } + + b := NewSourceTableWebhookBuilder(db, sourceTableWebhook) + b.BodyFormat("JSON") + b.IncludeHeader(includeHeader) + b.IncludeHeaders(IncludeHeadersStruct{All: true}) + b.CheckOptions(checkOptions) + b.CheckExpression("decode(headers->'x-signature', 'hex') = hmac(body, secret, 'sha1')") + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableWebhookCreateRudderstack(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `CREATE TABLE "database"."schema"."webhook_table" FROM WEBHOOK BODY FORMAT JSON CHECK \( WITH \(HEADERS, BODY AS request_body, SECRET "database"."schema"."my_webhook_shared_secret"\) headers->'authorization' = rudderstack_shared_secret\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + var checkOptions = []CheckOptionsStruct{ + { + Field: FieldStruct{Headers: true}, + }, + { + Field: FieldStruct{Body: true}, + Alias: "request_body", + }, + { + Field: FieldStruct{ + Secret: IdentifierSchemaStruct{ + DatabaseName: "database", + SchemaName: "schema", + Name: "my_webhook_shared_secret", + }, + }, + }, + } + + b := NewSourceTableWebhookBuilder(db, sourceTableWebhook) + b.BodyFormat("JSON") + b.CheckOptions(checkOptions) + b.CheckExpression("headers->'authorization' = rudderstack_shared_secret") + + if err := b.Create(); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableWebhookRename(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `ALTER TABLE "database"."schema"."webhook_table" RENAME TO "database"."schema"."new_webhook_table";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableWebhookBuilder(db, sourceTableWebhook) + if err := b.Rename("new_webhook_table"); err != nil { + t.Fatal(err) + } + }) +} + +func TestSourceTableWebhookDrop(t *testing.T) { + testhelpers.WithMockDb(t, func(db *sqlx.DB, mock sqlmock.Sqlmock) { + mock.ExpectExec( + `DROP TABLE "database"."schema"."webhook_table";`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + b := NewSourceTableWebhookBuilder(db, sourceTableWebhook) + if err := b.Drop(); err != nil { + t.Fatal(err) + } + }) +} diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index c3e9474c..572c0cbb 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -137,6 +137,7 @@ func Provider(version string) *schema.Provider { "materialize_source_table_load_generator": resources.SourceTableLoadGen(), "materialize_source_table_mysql": resources.SourceTableMySQL(), "materialize_source_table_postgres": resources.SourceTablePostgres(), + "materialize_source_table_webhook": resources.SourceTableWebhook(), "materialize_table_grant": resources.GrantTable(), "materialize_table_grant_default_privilege": resources.GrantTableDefaultPrivilege(), "materialize_type": resources.Type(), diff --git a/pkg/resources/resource_source_table_webhook.go b/pkg/resources/resource_source_table_webhook.go new file mode 100644 index 00000000..21e04a6d --- /dev/null +++ b/pkg/resources/resource_source_table_webhook.go @@ -0,0 +1,367 @@ +package resources + +import ( + "context" + "database/sql" + "log" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/materialize" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" +) + +var sourceTableWebhookSchema = map[string]*schema.Schema{ + "name": ObjectNameSchema("table", true, false), + "schema_name": SchemaNameSchema("table", false), + "database_name": DatabaseNameSchema("table", false), + "qualified_sql_name": QualifiedNameSchema("table"), + "comment": CommentSchema(false), + "body_format": { + Description: "The body format of the webhook.", + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validation.StringInSlice([]string{ + "TEXT", + "JSON", + "BYTES", + }, true), + }, + "include_header": { + Description: "Map a header value from a request into a column.", + Type: schema.TypeList, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "header": { + Description: "The name for the header.", + Type: schema.TypeString, + Required: true, + }, + "alias": { + Description: "The alias for the header.", + Type: schema.TypeString, + Optional: true, + }, + "bytes": { + Description: "Change type to `bytea`.", + Type: schema.TypeBool, + Optional: true, + }, + }, + }, + ForceNew: true, + }, + "include_headers": { + Description: "Include headers in the webhook.", + Type: schema.TypeList, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "all": { + Description: "Include all headers.", + Type: schema.TypeBool, + Optional: true, + ConflictsWith: []string{"include_headers.0.only", "include_headers.0.not"}, + AtLeastOneOf: []string{"include_headers.0.all", "include_headers.0.only", "include_headers.0.not"}, + }, + "only": { + Description: "Headers that should be included.", + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + ConflictsWith: []string{"include_headers.0.all"}, + AtLeastOneOf: []string{"include_headers.0.all", "include_headers.0.only", "include_headers.0.not"}, + }, + "not": { + Description: "Headers that should be excluded.", + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + ConflictsWith: []string{"include_headers.0.all"}, + AtLeastOneOf: []string{"include_headers.0.all", "include_headers.0.only", "include_headers.0.not"}, + }, + }, + }, + Optional: true, + MinItems: 1, + MaxItems: 1, + ForceNew: true, + }, + "check_options": { + Description: "The check options for the webhook.", + Type: schema.TypeList, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "field": { + Description: "The field for the check options.", + Type: schema.TypeList, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "body": { + Description: "The body for the check options.", + Type: schema.TypeBool, + Optional: true, + }, + "headers": { + Description: "The headers for the check options.", + Type: schema.TypeBool, + Optional: true, + }, + "secret": IdentifierSchema(IdentifierSchemaParams{ + Elem: "secret", + Description: "The secret for the check options.", + Required: false, + ForceNew: true, + }), + }, + }, + MinItems: 1, + MaxItems: 1, + Required: true, + }, + "alias": { + Description: "The alias for the check options.", + Type: schema.TypeString, + Optional: true, + }, + "bytes": { + Description: "Change type to `bytea`.", + Type: schema.TypeBool, + Optional: true, + }, + }, + }, + ForceNew: true, + }, + "check_expression": { + Description: "The check expression for the webhook.", + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + "ownership_role": OwnershipRoleSchema(), + "region": RegionSchema(), +} + +func SourceTableWebhook() *schema.Resource { + return &schema.Resource{ + Description: "A webhook source table allows reading data directly from webhooks.", + + CreateContext: sourceTableWebhookCreate, + ReadContext: sourceTableWebhookRead, + UpdateContext: sourceTableWebhookUpdate, + DeleteContext: sourceTableDelete, + + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + + Schema: sourceTableWebhookSchema, + } +} + +func sourceTableWebhookCreate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + tableName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewSourceTableWebhookBuilder(metaDb, o) + + b.BodyFormat(d.Get("body_format").(string)) + + if v, ok := d.GetOk("include_header"); ok { + var headers []materialize.HeaderStruct + for _, header := range v.([]interface{}) { + h := header.(map[string]interface{}) + headers = append(headers, materialize.HeaderStruct{ + Header: h["header"].(string), + Alias: h["alias"].(string), + Bytes: h["bytes"].(bool), + }) + } + b.IncludeHeader(headers) + } + + if v, ok := d.GetOk("include_headers"); ok { + var i materialize.IncludeHeadersStruct + u := v.([]interface{})[0].(map[string]interface{}) + + if v, ok := u["all"]; ok { + i.All = v.(bool) + } + + if v, ok := u["only"]; ok { + o, err := materialize.GetSliceValueString("only", v.([]interface{})) + if err != nil { + return diag.FromErr(err) + } + i.Only = o + } + + if v, ok := u["not"]; ok { + n, err := materialize.GetSliceValueString("not", v.([]interface{})) + if err != nil { + return diag.FromErr(err) + } + i.Not = n + } + b.IncludeHeaders(i) + } + + if v, ok := d.GetOk("check_options"); ok { + var options []materialize.CheckOptionsStruct + for _, option := range v.([]interface{}) { + t := option.(map[string]interface{}) + fieldMap := t["field"].([]interface{})[0].(map[string]interface{}) + + var secret = materialize.IdentifierSchemaStruct{} + if secretMap, ok := fieldMap["secret"].([]interface{}); ok && len(secretMap) > 0 && secretMap[0] != nil { + secret = materialize.GetIdentifierSchemaStruct(secretMap) + } + + field := materialize.FieldStruct{ + Body: fieldMap["body"].(bool), + Headers: fieldMap["headers"].(bool), + Secret: secret, + } + + options = append(options, materialize.CheckOptionsStruct{ + Field: field, + Alias: t["alias"].(string), + Bytes: t["bytes"].(bool), + }) + } + b.CheckOptions(options) + } + + if v, ok := d.GetOk("check_expression"); ok { + b.CheckExpression(v.(string)) + } + + // Create resource + if err := b.Create(); err != nil { + return diag.FromErr(err) + } + + // Handle ownership + if v, ok := d.GetOk("ownership_role"); ok { + ownership := materialize.NewOwnershipBuilder(metaDb, o) + if err := ownership.Alter(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed ownership, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + // Handle comments + if v, ok := d.GetOk("comment"); ok { + comment := materialize.NewCommentBuilder(metaDb, o) + if err := comment.Object(v.(string)); err != nil { + log.Printf("[DEBUG] resource failed comment, dropping object: %s", o.Name) + b.Drop() + return diag.FromErr(err) + } + } + + // Set ID + i, err := materialize.SourceTableWebhookId(metaDb, o) + if err != nil { + return diag.FromErr(err) + } + d.SetId(utils.TransformIdWithRegion(string(region), i)) + + return sourceTableWebhookRead(ctx, d, meta) +} + +func sourceTableWebhookRead(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { + i := d.Id() + + metaDb, region, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + t, err := materialize.ScanSourceTableWebhook(metaDb, utils.ExtractId(i)) + if err == sql.ErrNoRows { + d.SetId("") + return nil + } else if err != nil { + return diag.FromErr(err) + } + + d.SetId(utils.TransformIdWithRegion(string(region), i)) + + if err := d.Set("name", t.TableName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("schema_name", t.SchemaName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("database_name", t.DatabaseName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("ownership_role", t.OwnerName.String); err != nil { + return diag.FromErr(err) + } + + if err := d.Set("comment", t.Comment.String); err != nil { + return diag.FromErr(err) + } + + return nil +} + +func sourceTableWebhookUpdate(ctx context.Context, d *schema.ResourceData, meta any) diag.Diagnostics { + tableName := d.Get("name").(string) + schemaName := d.Get("schema_name").(string) + databaseName := d.Get("database_name").(string) + + metaDb, _, err := utils.GetDBClientFromMeta(meta, d) + if err != nil { + return diag.FromErr(err) + } + + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: tableName, SchemaName: schemaName, DatabaseName: databaseName} + + if d.HasChange("name") { + oldName, newName := d.GetChange("name") + o := materialize.MaterializeObject{ObjectType: "TABLE", Name: oldName.(string), SchemaName: schemaName, DatabaseName: databaseName} + b := materialize.NewSourceTableBuilder(metaDb, o) + if err := b.Rename(newName.(string)); err != nil { + return diag.FromErr(err) + } + } + + if d.HasChange("ownership_role") { + _, newRole := d.GetChange("ownership_role") + b := materialize.NewOwnershipBuilder(metaDb, o) + + if err := b.Alter(newRole.(string)); err != nil { + return diag.FromErr(err) + } + } + + if d.HasChange("comment") { + _, newComment := d.GetChange("comment") + b := materialize.NewCommentBuilder(metaDb, o) + + if err := b.Object(newComment.(string)); err != nil { + return diag.FromErr(err) + } + } + + return sourceTableWebhookRead(ctx, d, meta) +} diff --git a/pkg/resources/resource_source_table_webhook_test.go b/pkg/resources/resource_source_table_webhook_test.go new file mode 100644 index 00000000..9a7f7e8d --- /dev/null +++ b/pkg/resources/resource_source_table_webhook_test.go @@ -0,0 +1,116 @@ +package resources + +import ( + "context" + "testing" + + "github.com/MaterializeInc/terraform-provider-materialize/pkg/testhelpers" + "github.com/MaterializeInc/terraform-provider-materialize/pkg/utils" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/stretchr/testify/require" +) + +var inSourceTableWebhook = map[string]interface{}{ + "name": "webhook_table", + "schema_name": "schema", + "database_name": "database", + "body_format": "JSON", + "include_headers": []interface{}{ + map[string]interface{}{ + "all": true, + }, + }, + "check_options": []interface{}{ + map[string]interface{}{ + "field": []interface{}{map[string]interface{}{ + "body": true, + }}, + "alias": "bytes", + }, + map[string]interface{}{ + "field": []interface{}{map[string]interface{}{ + "headers": true, + }}, + "alias": "headers", + }, + }, + "check_expression": "check_expression", +} + +func TestResourceSourceTableWebhookCreate(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableWebhook().Schema, inSourceTableWebhook) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Create + mock.ExpectExec( + `CREATE TABLE "database"."schema"."webhook_table" FROM WEBHOOK BODY FORMAT JSON INCLUDE HEADERS CHECK \( WITH \(BODY AS bytes\, HEADERS AS headers\) check_expression\);`, + ).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Id + ip := `WHERE mz_databases.name = 'database' AND mz_schemas.name = 'schema' AND mz_tables.name = 'webhook_table'` + testhelpers.MockSourceTableWebhookScan(mock, ip) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableWebhookScan(mock, pp) + + if err := sourceTableWebhookCreate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableWebhookDelete(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableWebhook().Schema, inSourceTableWebhook) + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + mock.ExpectExec(`DROP TABLE "database"."schema"."webhook_table";`).WillReturnResult(sqlmock.NewResult(1, 1)) + + if err := sourceTableDelete(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableWebhookUpdate(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableWebhook().Schema, inSourceTableWebhook) + d.SetId("u1") + d.Set("name", "webhook_table") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + mock.ExpectExec(`ALTER TABLE "database"."schema"."" RENAME TO "database"."schema"."webhook_table"`).WillReturnResult(sqlmock.NewResult(1, 1)) + + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableWebhookScan(mock, pp) + + if err := sourceTableWebhookUpdate(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} + +func TestResourceSourceTableWebhookRead(t *testing.T) { + r := require.New(t) + d := schema.TestResourceDataRaw(t, SourceTableWebhook().Schema, inSourceTableWebhook) + d.SetId("u1") + r.NotNil(d) + + testhelpers.WithMockProviderMeta(t, func(db *utils.ProviderMeta, mock sqlmock.Sqlmock) { + // Query Params + pp := `WHERE mz_tables.id = 'u1'` + testhelpers.MockSourceTableWebhookScan(mock, pp) + + if err := sourceTableWebhookRead(context.TODO(), d, db); err != nil { + t.Fatal(err) + } + }) +} diff --git a/pkg/resources/resource_source_webhook.go b/pkg/resources/resource_source_webhook.go index 617753e2..ae803727 100644 --- a/pkg/resources/resource_source_webhook.go +++ b/pkg/resources/resource_source_webhook.go @@ -158,7 +158,12 @@ var sourceWebhookSchema = map[string]*schema.Schema{ func SourceWebhook() *schema.Resource { return &schema.Resource{ - Description: "A webhook source describes a webhook you want Materialize to read data from.", + Description: "A webhook source describes a webhook you want Materialize to read data from. " + + "This resource is deprecated and will be removed in a future release. " + + "Please use materialize_source_table_webhook instead.", + + DeprecationMessage: "This resource is deprecated and will be removed in a future release. " + + "Please use materialize_source_table_webhook instead.", CreateContext: sourceWebhookCreate, ReadContext: sourceRead, diff --git a/pkg/testhelpers/mock_scans.go b/pkg/testhelpers/mock_scans.go index 4ca40e4b..87e7c43d 100644 --- a/pkg/testhelpers/mock_scans.go +++ b/pkg/testhelpers/mock_scans.go @@ -961,6 +961,38 @@ func MockSourceTableScan(mock sqlmock.Sqlmock, predicate string) { mock.ExpectQuery(q).WillReturnRows(ir) } +func MockSourceTableWebhookScan(mock sqlmock.Sqlmock, predicate string) { + b := ` + SELECT + mz_tables.id, + mz_tables.name, + mz_schemas.name AS schema_name, + mz_databases.name AS database_name, + mz_sources.type AS source_type, + comments.comment AS comment, + mz_roles.name AS owner_name, + mz_tables.privileges + FROM mz_tables + JOIN mz_schemas + ON mz_tables.schema_id = mz_schemas.id + JOIN mz_databases + ON mz_schemas.database_id = mz_databases.id + JOIN mz_roles + ON mz_tables.owner_id = mz_roles.id + LEFT JOIN \( + SELECT id, comment + FROM mz_internal.mz_comments + WHERE object_type = 'table' + AND object_sub_id IS NULL + \) comments + ON mz_tables.id = comments.id` + + q := mockQueryBuilder(b, predicate, "") + ir := mock.NewRows([]string{"id", "name", "schema_name", "database_name", "source_type", "comment", "owner_name", "privileges"}). + AddRow("u1", "table", "schema", "database", "webhook", "comment", "materialize", defaultPrivilege) + mock.ExpectQuery(q).WillReturnRows(ir) +} + func MockSourceReferenceScan(mock sqlmock.Sqlmock, predicate string) { b := ` SELECT