From 187ea537cac9b562bfe03dbb6a72e953cc8e901f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Sala=C3=BCn?= Date: Tue, 27 Feb 2024 08:45:44 +0100 Subject: [PATCH] feat: faster timestamp queries (#479) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: postgres timestamp casting should include timezone Signed-off-by: Clément Salaün * fix: fix: test Signed-off-by: Clément Salaün * feat: faster endtime based queries Signed-off-by: Clément Salaün * feat: add temporal index on transactions Signed-off-by: Clément Salaün * fix: enable optimisation only when pageSize is one Signed-off-by: Clément Salaün * fix: apply optimized filter only for postgres Signed-off-by: Clément Salaün --------- Signed-off-by: Clément Salaün --- .../21-optimized-temporal/postgres.sql | 2 ++ pkg/storage/sqlstorage/transactions.go | 21 ++++++++++++++++++- pkg/storage/sqlstorage/transactions_test.go | 3 +++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 pkg/storage/sqlstorage/migrates/21-optimized-temporal/postgres.sql diff --git a/pkg/storage/sqlstorage/migrates/21-optimized-temporal/postgres.sql b/pkg/storage/sqlstorage/migrates/21-optimized-temporal/postgres.sql new file mode 100644 index 000000000..1cb956ddd --- /dev/null +++ b/pkg/storage/sqlstorage/migrates/21-optimized-temporal/postgres.sql @@ -0,0 +1,2 @@ +--statement +CREATE INDEX IF NOT EXISTS transactions_ts_desc ON "VAR_LEDGER_NAME".transactions ("timestamp" DESC); \ No newline at end of file diff --git a/pkg/storage/sqlstorage/transactions.go b/pkg/storage/sqlstorage/transactions.go index 4f12f85a3..c5513406c 100644 --- a/pkg/storage/sqlstorage/transactions.go +++ b/pkg/storage/sqlstorage/transactions.go @@ -122,6 +122,25 @@ func (s *Store) buildTransactionsQuery(flavor Flavor, p ledger.TransactionsQuery } if !endTime.IsZero() { sb.Where(sb.L("timestamp", endTime.UTC())) + + if flavor == PostgreSQL { + // We nudge the query planner in the right direction, + // by reducing the search space according to the end time. + // We have to use a raw query as the sqlbuilder + // does not support LTE+subqueries in the where clause. + sb.SQL(fmt.Sprintf(` + AND "id" <= ( + SELECT "id" + FROM "%s".transactions + WHERE "timestamp" < '%s'::timestamptz + ORDER BY "timestamp" DESC, "id" DESC + LIMIT 1 + )`, + s.schema.Name(), + endTime.UTC().Format(time.RFC3339), + )) + } + t.EndTime = endTime } @@ -461,7 +480,7 @@ func (s *Store) insertTransactions(ctx context.Context, txs ...core.ExpandedTran pre_commit_volumes, post_commit_volumes) (SELECT * FROM unnest( $1::int[], - $2::timestamp[], + $2::timestamptz[], $3::varchar[], $4::jsonb[], $5::jsonb[], diff --git a/pkg/storage/sqlstorage/transactions_test.go b/pkg/storage/sqlstorage/transactions_test.go index 3fa52f019..349de128c 100644 --- a/pkg/storage/sqlstorage/transactions_test.go +++ b/pkg/storage/sqlstorage/transactions_test.go @@ -353,6 +353,9 @@ func testTransactions(t *testing.T, store *sqlstorage.Store) { // Should get only the first transaction. require.Equal(t, 1, cursor.PageSize) + // Transaction timestamp fetched should be equal to the timestamp of the committed transaction. + require.True(t, cursor.Data[0].Timestamp.Equal(tx3.Timestamp)) + cursor, err = store.GetTransactions(context.Background(), ledger.TransactionsQuery{ AfterTxID: cursor.Data[0].ID, PageSize: 1,