Skip to content

Commit

Permalink
materialize-databricks: fix commit logging for split queries
Browse files Browse the repository at this point in the history
Databricks Store queries may be split into multiple separate items for large
file sizes. This fixing the logging so that the started / ending commit
messaging is not emitted for each of the split queries, but rather for the
entire binding.
  • Loading branch information
williamhbaker committed Dec 12, 2024
1 parent f554efa commit 418f8e5
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions materialize-databricks/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -465,8 +465,8 @@ func (d *transactor) Acknowledge(ctx context.Context) (*pf.ConnectorState, error
}
queries = []string{item.Query}
}
d.be.StartedResourceCommit(path)
for _, query := range queries {
d.be.StartedResourceCommit(path)
if _, err := db.ExecContext(ctx, query); err != nil {
// When doing a recovery apply, it may be the case that some tables & files have already been deleted after being applied
// it is okay to skip them in this case
Expand All @@ -477,8 +477,8 @@ func (d *transactor) Acknowledge(ctx context.Context) (*pf.ConnectorState, error
}
return nil, fmt.Errorf("query %q failed: %w", query, err)
}
d.be.FinishedResourceCommit(path)
}
d.be.FinishedResourceCommit(path)

// Cleanup files.
d.deleteFiles(ctx, item.ToDelete)
Expand Down

0 comments on commit 418f8e5

Please sign in to comment.