diff --git a/durable_state_actor_test.go b/durable_state_actor_test.go index c6030e2..ebd606d 100644 --- a/durable_state_actor_test.go +++ b/durable_state_actor_test.go @@ -32,6 +32,8 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + memory "github.com/tochemey/ego-contrib/durablestore/memory" + pgstore "github.com/tochemey/ego-contrib/durablestore/postgres" "github.com/tochemey/goakt/v2/actors" "github.com/tochemey/goakt/v2/log" "google.golang.org/protobuf/proto" @@ -39,9 +41,6 @@ import ( "github.com/tochemey/ego/v3/egopb" "github.com/tochemey/ego/v3/eventstream" "github.com/tochemey/ego/v3/internal/lib" - "github.com/tochemey/ego/v3/internal/postgres" - "github.com/tochemey/ego/v3/plugins/statestore/memory" - pgstore "github.com/tochemey/ego/v3/plugins/statestore/postgres" testpb "github.com/tochemey/ego/v3/test/data/pb/v3" ) @@ -253,7 +252,7 @@ func TestDurableStateBehavior(t *testing.T) { testDatabasePassword = "testPass" ) - testContainer := postgres.NewTestContainer(testDatabase, testUser, testDatabasePassword) + testContainer := pgstore.NewTestContainer(testDatabase, testUser, testDatabasePassword) db := testContainer.GetTestDB() require.NoError(t, db.Connect(ctx)) schemaUtils := pgstore.NewSchemaUtils(db) @@ -267,7 +266,7 @@ func TestDurableStateBehavior(t *testing.T) { DBPassword: testDatabasePassword, DBSchema: testContainer.Schema(), } - durableStore := pgstore.NewStateStore(config) + durableStore := pgstore.NewDurableStore(config) require.NoError(t, durableStore.Connect(ctx)) lib.Pause(time.Second) diff --git a/engine_test.go b/engine_test.go index 3c0a279..ef70673 100644 --- a/engine_test.go +++ b/engine_test.go @@ -39,15 +39,15 @@ import ( "github.com/travisjeffery/go-dynaport" "google.golang.org/protobuf/proto" + memstore "github.com/tochemey/ego-contrib/durablestore/memory" + memory "github.com/tochemey/ego-contrib/eventstore/memory" + offsetstore "github.com/tochemey/ego-contrib/offsetstore/memory" "github.com/tochemey/goakt/v2/log" mockdisco "github.com/tochemey/goakt/v2/mocks/discovery" "github.com/tochemey/ego/v3/egopb" samplepb "github.com/tochemey/ego/v3/example/pbs/sample/pb/v1" "github.com/tochemey/ego/v3/internal/lib" - offsetstore "github.com/tochemey/ego/v3/offsetstore/memory" - "github.com/tochemey/ego/v3/plugins/eventstore/memory" - memstore "github.com/tochemey/ego/v3/plugins/statestore/memory" "github.com/tochemey/ego/v3/projection" testpb "github.com/tochemey/ego/v3/test/data/pb/v3" ) diff --git a/event_sourced_actor_test.go b/event_sourced_actor_test.go index 4b258b2..f2aecb7 100644 --- a/event_sourced_actor_test.go +++ b/event_sourced_actor_test.go @@ -35,15 +35,14 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" + memory "github.com/tochemey/ego-contrib/eventstore/memory" + pgstore "github.com/tochemey/ego-contrib/eventstore/postgres" "github.com/tochemey/goakt/v2/actors" "github.com/tochemey/goakt/v2/log" "github.com/tochemey/ego/v3/egopb" "github.com/tochemey/ego/v3/eventstream" "github.com/tochemey/ego/v3/internal/lib" - "github.com/tochemey/ego/v3/internal/postgres" - "github.com/tochemey/ego/v3/plugins/eventstore/memory" - pgstore "github.com/tochemey/ego/v3/plugins/eventstore/postgres" testpb "github.com/tochemey/ego/v3/test/data/pb/v3" ) @@ -333,7 +332,7 @@ func TestEventSourcedActor(t *testing.T) { testDatabasePassword = "testPass" ) - testContainer := postgres.NewTestContainer(testDatabase, testUser, testDatabasePassword) + testContainer := pgstore.NewTestContainer(testDatabase, testUser, testDatabasePassword) db := testContainer.GetTestDB() // create the event store table require.NoError(t, db.Connect(ctx)) diff --git a/example/durablestate/main.go b/example/durablestate/main.go index f48a30c..8898a87 100644 --- a/example/durablestate/main.go +++ b/example/durablestate/main.go @@ -34,11 +34,11 @@ import ( "time" "github.com/google/uuid" + memory "github.com/tochemey/ego-contrib/durablestore/memory" "google.golang.org/protobuf/proto" "github.com/tochemey/ego/v3" samplepb "github.com/tochemey/ego/v3/example/pbs/sample/pb/v1" - "github.com/tochemey/ego/v3/plugins/statestore/memory" ) func main() { diff --git a/example/eventssourced/main.go b/example/eventssourced/main.go index 1a510d4..b5ec56d 100644 --- a/example/eventssourced/main.go +++ b/example/eventssourced/main.go @@ -34,11 +34,11 @@ import ( "time" "github.com/google/uuid" + memory "github.com/tochemey/ego-contrib/eventstore/memory" "google.golang.org/protobuf/proto" "github.com/tochemey/ego/v3" samplepb "github.com/tochemey/ego/v3/example/pbs/sample/pb/v1" - "github.com/tochemey/ego/v3/plugins/eventstore/memory" ) func main() { diff --git a/go.mod b/go.mod index 8c99bd3..032dba2 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,14 @@ module github.com/tochemey/ego/v3 go 1.22.0 require ( - github.com/Masterminds/squirrel v1.5.4 - github.com/deckarep/golang-set/v2 v2.7.0 github.com/flowchartsman/retry v1.2.0 - github.com/georgysavva/scany/v2 v2.1.3 github.com/google/uuid v1.6.0 - github.com/hashicorp/go-memdb v1.3.4 - github.com/jackc/pgx/v5 v5.7.2 - github.com/lib/pq v1.10.9 - github.com/ory/dockertest/v3 v3.11.0 github.com/stretchr/testify v1.10.0 + github.com/tochemey/ego-contrib/durablestore/memory v0.0.0-20241226194523-04ecdcbf3fb3 + github.com/tochemey/ego-contrib/durablestore/postgres v0.0.0-20241226194523-04ecdcbf3fb3 + github.com/tochemey/ego-contrib/eventstore/memory v0.0.0-20241226194523-04ecdcbf3fb3 + github.com/tochemey/ego-contrib/eventstore/postgres v0.0.0-20241226194523-04ecdcbf3fb3 + github.com/tochemey/ego-contrib/offsetstore/memory v0.0.0-20241226194523-04ecdcbf3fb3 github.com/tochemey/goakt/v2 v2.11.0 github.com/travisjeffery/go-dynaport v1.0.0 go.uber.org/atomic v1.11.0 @@ -26,6 +24,7 @@ require ( connectrpc.com/connect v1.17.0 // indirect dario.cat/mergo v1.0.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/Masterminds/squirrel v1.5.4 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/RoaringBitmap/roaring v1.9.4 // indirect @@ -38,13 +37,15 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/continuity v0.4.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/deckarep/golang-set/v2 v2.7.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/docker/cli v27.4.0+incompatible // indirect - github.com/docker/docker v27.4.0+incompatible // indirect + github.com/docker/cli v27.4.1+incompatible // indirect + github.com/docker/docker v27.4.1+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/georgysavva/scany/v2 v2.1.3 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.21.0 // indirect @@ -59,6 +60,7 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-memdb v1.3.4 // indirect github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-sockaddr v1.0.7 // indirect @@ -67,12 +69,14 @@ require ( github.com/hashicorp/memberlist v0.5.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.2 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/mailru/easyjson v0.9.0 // indirect github.com/miekg/dns v1.1.62 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect @@ -85,6 +89,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runc v1.2.3 // indirect + github.com/ory/dockertest/v3 v3.11.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/redis/go-redis/v9 v9.7.0 // indirect diff --git a/go.sum b/go.sum index 9556462..cde81a3 100644 --- a/go.sum +++ b/go.sum @@ -72,10 +72,10 @@ github.com/deckarep/golang-set/v2 v2.7.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpO github.com/dgryski/go-ddmin v0.0.0-20210904190556-96a6d69f1034/go.mod h1:zz4KxBkcXUWKjIcrc+uphJ1gPh/t18ymGm3PmQ+VGTk= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/docker/cli v27.4.0+incompatible h1:/nJzWkcI1MDMN+U+px/YXnQWJqnu4J+QKGTfD6ptiTc= -github.com/docker/cli v27.4.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v27.4.0+incompatible h1:I9z7sQ5qyzO0BfAb9IMOawRkAGxhYsidKiTMcm0DU+A= -github.com/docker/docker v27.4.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= +github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/docker v27.4.1+incompatible h1:ZJvcY7gfwHn1JF48PfbyXg7Jyt9ZCWDW+GGXOIxEwp4= +github.com/docker/docker v27.4.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -340,6 +340,26 @@ github.com/tidwall/redcon v1.6.2 h1:5qfvrrybgtO85jnhSravmkZyC0D+7WstbfCs3MmPhow= github.com/tidwall/redcon v1.6.2/go.mod h1:p5Wbsgeyi2VSTBWOcA5vRXrOb9arFTcU2+ZzFjqV75Y= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= +github.com/tochemey/ego-contrib/durablestore/memory v0.0.0-20241226191059-fe9589656e1d h1:WWJeCPdN3fETVHfEIEbDhDBNbBweV3apUH/QOpRTdGg= +github.com/tochemey/ego-contrib/durablestore/memory v0.0.0-20241226191059-fe9589656e1d/go.mod h1:ZiiSRYjYP66Len2QZtQaq2suPCQAg9zLMymJl0zwPBo= +github.com/tochemey/ego-contrib/durablestore/memory v0.0.0-20241226194523-04ecdcbf3fb3 h1:mXAKyQ37qiQdqHyaID+rEvLNePHR9wNKOAqyhFRmzLM= +github.com/tochemey/ego-contrib/durablestore/memory v0.0.0-20241226194523-04ecdcbf3fb3/go.mod h1:ZiiSRYjYP66Len2QZtQaq2suPCQAg9zLMymJl0zwPBo= +github.com/tochemey/ego-contrib/durablestore/postgres v0.0.0-20241226191059-fe9589656e1d h1:5hN0j9oIi41CTZ7nqVOhcOgAp0AqZU9CJAnwxrHiYrY= +github.com/tochemey/ego-contrib/durablestore/postgres v0.0.0-20241226191059-fe9589656e1d/go.mod h1:jWcNje9EmFMgsgyxbue3FEQdCJFv/cdCHVUAoZUnyb8= +github.com/tochemey/ego-contrib/durablestore/postgres v0.0.0-20241226194523-04ecdcbf3fb3 h1:nbVlFznzFcyciZrIM1eThEHchBWe86DS18NafNkozq8= +github.com/tochemey/ego-contrib/durablestore/postgres v0.0.0-20241226194523-04ecdcbf3fb3/go.mod h1:jWcNje9EmFMgsgyxbue3FEQdCJFv/cdCHVUAoZUnyb8= +github.com/tochemey/ego-contrib/eventstore/memory v0.0.0-20241226191059-fe9589656e1d h1:YdOrvoAzsbUZVznJjqP3VDVwjEsYHlfcmXaxq9nFII0= +github.com/tochemey/ego-contrib/eventstore/memory v0.0.0-20241226191059-fe9589656e1d/go.mod h1:fIPRJ0Vh/hltHDOyqCZXqQBv+u1OAeTM3pDQn2Tg5cw= +github.com/tochemey/ego-contrib/eventstore/memory v0.0.0-20241226194523-04ecdcbf3fb3 h1:Zh5ndookRvlRFCS5pjEMw5sjcl2bboRlkQA1Wg+dW+k= +github.com/tochemey/ego-contrib/eventstore/memory v0.0.0-20241226194523-04ecdcbf3fb3/go.mod h1:fIPRJ0Vh/hltHDOyqCZXqQBv+u1OAeTM3pDQn2Tg5cw= +github.com/tochemey/ego-contrib/eventstore/postgres v0.0.0-20241226191059-fe9589656e1d h1:rCcusqecSQt2z8jaGcqu+WnTKL5JCZv8Ti7y/tUAIdc= +github.com/tochemey/ego-contrib/eventstore/postgres v0.0.0-20241226191059-fe9589656e1d/go.mod h1:amP9+eDvwyodN5ZAkeQnek+9PM28n4JxXiaxv2ghnZI= +github.com/tochemey/ego-contrib/eventstore/postgres v0.0.0-20241226194523-04ecdcbf3fb3 h1:blYCCLs8TH812sNyyaHEI1CQpSTk2ubPwOzLlZFzPqs= +github.com/tochemey/ego-contrib/eventstore/postgres v0.0.0-20241226194523-04ecdcbf3fb3/go.mod h1:amP9+eDvwyodN5ZAkeQnek+9PM28n4JxXiaxv2ghnZI= +github.com/tochemey/ego-contrib/offsetstore/memory v0.0.0-20241226191059-fe9589656e1d h1:hd3VCJstrA2N1Mp2j32oUrJhQVoZhjr8pgh7koqv+8U= +github.com/tochemey/ego-contrib/offsetstore/memory v0.0.0-20241226191059-fe9589656e1d/go.mod h1:POUIFQ21A1RJW8HKkntl1cnhPHwZIwRXZn+ArjZM/Kw= +github.com/tochemey/ego-contrib/offsetstore/memory v0.0.0-20241226194523-04ecdcbf3fb3 h1:oljKEGjJsqM1EPVpZ0jM0SajeItf3+J28YgVUXK22ks= +github.com/tochemey/ego-contrib/offsetstore/memory v0.0.0-20241226194523-04ecdcbf3fb3/go.mod h1:POUIFQ21A1RJW8HKkntl1cnhPHwZIwRXZn+ArjZM/Kw= github.com/tochemey/goakt/v2 v2.11.0 h1:4reuqeN8mvsUWu9ecn3Vd+qtpTASj/kEI9W3FRYHofM= github.com/tochemey/goakt/v2 v2.11.0/go.mod h1:gMba2xHDzLSDojw5m1SPHDzVFgNWn1I6G9zoagym5e0= github.com/travisjeffery/go-dynaport v1.0.0 h1:m/qqf5AHgB96CMMSworIPyo1i7NZueRsnwdzdCJ8Ajw= diff --git a/internal/postgres/config.go b/internal/postgres/config.go deleted file mode 100644 index c4282b9..0000000 --- a/internal/postgres/config.go +++ /dev/null @@ -1,60 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import "time" - -// Config defines the postgres configuration -// This configuration does not take into consideration the SSL mode -// TODO: enhance with SSL mode -type Config struct { - DBHost string // DBHost represents the database host - DBPort int // DBPort is the database port - DBName string // DBName is the database name - DBUser string // DBUser is the database user used to connect - DBPassword string // DBPassword is the database password - DBSchema string // DBSchema represents the database schema - MaxConnections int // MaxConnections represents the number of max connections in the pool - MinConnections int // MinConnections represents the number of minimum connections in the pool - MaxConnectionLifetime time.Duration // MaxConnectionLifetime represents the duration since creation after which a connection will be automatically closed. - MaxConnIdleTime time.Duration // MaxConnIdleTime is the duration after which an idle connection will be automatically closed by the health check. - HealthCheckPeriod time.Duration // HeathCheckPeriod is the duration between checks of the health of idle connections. -} - -// NewConfig creates an instance of Config -func NewConfig(host string, port int, user, password, dbName string) *Config { - return &Config{ - DBHost: host, - DBPort: port, - DBName: dbName, - DBUser: user, - DBPassword: password, - MaxConnections: 4, - MinConnections: 0, - MaxConnectionLifetime: time.Hour, - MaxConnIdleTime: 30 * time.Minute, - HealthCheckPeriod: time.Minute, - } -} diff --git a/internal/postgres/postgres.go b/internal/postgres/postgres.go deleted file mode 100644 index ba20876..0000000 --- a/internal/postgres/postgres.go +++ /dev/null @@ -1,166 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "fmt" - - "github.com/georgysavva/scany/v2/pgxscan" - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgconn" - "github.com/jackc/pgx/v5/pgxpool" -) - -// Postgres will be implemented by concrete RDBMS store -type Postgres interface { - // Connect connects to the underlying database - Connect(ctx context.Context) error - // Disconnect closes the underlying opened underlying connection database - Disconnect(ctx context.Context) error - // Select fetches a single row from the database and automatically scanned it into the dst. - // It returns an error in case of failure. When there is no record no errors is return. - Select(ctx context.Context, dst any, query string, args ...any) error - // SelectAll fetches a set of rows as defined by the query and scanned those record in the dst. - // It returns nil when there is no records to fetch. - SelectAll(ctx context.Context, dst any, query string, args ...any) error - // Exec executes an SQL statement against the database and returns the appropriate result or an error. - Exec(ctx context.Context, query string, args ...any) (pgconn.CommandTag, error) - // BeginTx helps start an SQL transaction. The return transaction object is expected to be used in - // the subsequent queries following the BeginTx. - BeginTx(ctx context.Context, txOptions pgx.TxOptions) (pgx.Tx, error) -} - -// Postgres helps interact with the Postgres database -type postgres struct { - connStr string - pool *pgxpool.Pool - config *Config -} - -var _ Postgres = (*postgres)(nil) - -// New returns a store connecting to the given Postgres database. -func New(config *Config) Postgres { - postgres := new(postgres) - postgres.config = config - postgres.connStr = createConnectionString(config.DBHost, config.DBPort, config.DBName, config.DBUser, config.DBPassword, config.DBSchema) - return postgres -} - -// Connect will connect to our Postgres database -func (pg *postgres) Connect(ctx context.Context) error { - // create the connection config - config, err := pgxpool.ParseConfig(pg.connStr) - if err != nil { - return fmt.Errorf("failed to parse connection string: %w", err) - } - - // amend some of the configuration - config.MaxConns = int32(pg.config.MaxConnections) - config.MaxConnLifetime = pg.config.MaxConnectionLifetime - config.MaxConnIdleTime = pg.config.MaxConnIdleTime - config.MinConns = int32(pg.config.MinConnections) - config.HealthCheckPeriod = pg.config.HealthCheckPeriod - - // connect to the pool - pool, err := pgxpool.NewWithConfig(ctx, config) - if err != nil { - return fmt.Errorf("failed to create the connection pool: %w", err) - } - - // let us test the connection - if err := pool.Ping(ctx); err != nil { - return fmt.Errorf("failed to ping the database connection: %w", err) - } - - // set the db handle - pg.pool = pool - return nil -} - -// createConnectionString will create the Postgres connection string from the -// supplied connection details -// TODO: enhance this with the SSL settings -func createConnectionString(host string, port int, name, user string, password string, schema string) string { - info := fmt.Sprintf("host=%s port=%d user=%s dbname=%s sslmode=disable", host, port, user, name) - // The Postgres driver gets confused in cases where the user has no password - // set but a password is passed, so only set password if its non-empty - if password != "" { - info += fmt.Sprintf(" password=%s", password) - } - - if schema != "" { - info += fmt.Sprintf(" search_path=%s", schema) - } - - return info -} - -// Exec executes a sql query without returning rows against the database -func (pg *postgres) Exec(ctx context.Context, query string, args ...interface{}) (pgconn.CommandTag, error) { - return pg.pool.Exec(ctx, query, args...) -} - -// BeginTx starts a new database transaction -func (pg *postgres) BeginTx(ctx context.Context, txOptions pgx.TxOptions) (pgx.Tx, error) { - return pg.pool.BeginTx(ctx, txOptions) -} - -// SelectAll fetches rows -func (pg *postgres) SelectAll(ctx context.Context, dst interface{}, query string, args ...interface{}) error { - err := pgxscan.Select(ctx, pg.pool, dst, query, args...) - if err != nil { - if pgxscan.NotFound(err) { - return nil - } - - return err - } - return nil -} - -// Select fetches only one row -func (pg *postgres) Select(ctx context.Context, dst interface{}, query string, args ...interface{}) error { - err := pgxscan.Get(ctx, pg.pool, dst, query, args...) - if err != nil { - if pgxscan.NotFound(err) { - return nil - } - return err - } - - return nil -} - -// Disconnect the database connection. -// nolint -func (pg *postgres) Disconnect(ctx context.Context) error { - if pg.pool == nil { - return nil - } - pg.pool.Close() - return nil -} diff --git a/internal/postgres/postgres_test.go b/internal/postgres/postgres_test.go deleted file mode 100644 index 717132e..0000000 --- a/internal/postgres/postgres_test.go +++ /dev/null @@ -1,325 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "testing" - "time" - - "github.com/google/uuid" - "github.com/stretchr/testify/suite" -) - -// account is a test struct -type account struct { - AccountID string - AccountName string -} - -// PostgresTestSuite will run the Postgres tests -type PostgresTestSuite struct { - suite.Suite - container *TestContainer -} - -// SetupSuite starts the Postgres database engine and set the container -// host and port to use in the tests -func (s *PostgresTestSuite) SetupSuite() { - s.container = NewTestContainer("testdb", "test", "test") -} - -func (s *PostgresTestSuite) TearDownSuite() { - s.container.Cleanup() -} - -// In order for 'go test' to run this suite, we need to create -// a normal test function and pass our suite to suite.Run -func TestPostgresTestSuite(t *testing.T) { - suite.Run(t, new(PostgresTestSuite)) -} - -func (s *PostgresTestSuite) TestConnect() { - s.Run("with valid connection settings", func() { - ctx := context.TODO() - db := s.container.GetTestDB() - - err := db.Connect(ctx) - s.Assert().NoError(err) - }) - - s.Run("with invalid database port", func() { - ctx := context.TODO() - db := New(&Config{ - DBUser: "test", - DBName: "testdb", - DBPassword: "test", - DBSchema: s.container.Schema(), - DBHost: s.container.Host(), - DBPort: -2, - MaxConnections: 4, - MinConnections: 0, - MaxConnectionLifetime: time.Hour, - MaxConnIdleTime: 30 * time.Minute, - HealthCheckPeriod: time.Minute, - }) - err := db.Connect(ctx) - s.Assert().Error(err) - }) - - s.Run("with invalid database name", func() { - ctx := context.TODO() - db := New(&Config{ - DBUser: "test", - DBName: "wrong-name", - DBPassword: "test", - DBSchema: s.container.Schema(), - DBHost: s.container.Host(), - DBPort: s.container.Port(), - MaxConnections: 4, - MinConnections: 0, - MaxConnectionLifetime: time.Hour, - MaxConnIdleTime: 30 * time.Minute, - HealthCheckPeriod: time.Minute, - }) - err := db.Connect(ctx) - s.Assert().Error(err) - }) - - s.Run("with invalid database user", func() { - ctx := context.TODO() - db := New(&Config{ - DBUser: "test-user", - DBName: "testdb", - DBPassword: "test", - DBSchema: s.container.Schema(), - DBHost: s.container.Host(), - DBPort: s.container.Port(), - MaxConnections: 4, - MinConnections: 0, - MaxConnectionLifetime: time.Hour, - MaxConnIdleTime: 30 * time.Minute, - HealthCheckPeriod: time.Minute, - }) - err := db.Connect(ctx) - s.Assert().Error(err) - }) - - s.Run("with invalid database password", func() { - ctx := context.TODO() - db := New(&Config{ - DBUser: "test", - DBName: "testdb", - DBPassword: "invalid-db-pass", - DBSchema: s.container.Schema(), - DBHost: s.container.Host(), - DBPort: s.container.Port(), - MaxConnections: 4, - MinConnections: 0, - MaxConnectionLifetime: time.Hour, - MaxConnIdleTime: 30 * time.Minute, - HealthCheckPeriod: time.Minute, - }) - - err := db.Connect(ctx) - s.Assert().Error(err) - }) -} - -func (s *PostgresTestSuite) TestExec() { - ctx := context.TODO() - db := s.container.GetTestDB() - err := db.Connect(ctx) - s.Assert().NoError(err) - - s.Run("with valid SQL statement", func() { - // let us create a test table - const schemaDDL = ` - CREATE TABLE accounts - ( - account_id UUID, - account_name VARCHAR(255) NOT NULL, - PRIMARY KEY (account_id) - ); - ` - _, err = db.Exec(ctx, schemaDDL) - s.Assert().NoError(err) - }) - - s.Run("with invalid SQL statement", func() { - const schemaDDL = `SOME-INVALID-SQL` - _, err = db.Exec(ctx, schemaDDL) - s.Assert().Error(err) - }) -} - -func (s *PostgresTestSuite) TestSelect() { - ctx := context.TODO() - db := s.container.GetTestDB() - err := db.Connect(ctx) - s.Assert().NoError(err) - - const selectSQL = `SELECT account_id, account_name FROM accounts WHERE account_id = $1` - - s.Run("with valid record", func() { - // first drop the table - err = db.DropTable(ctx, "accounts") - s.Assert().NoError(err) - - // create the database table - err = createTable(ctx, db) - s.Assert().NoError(err) - - // let us insert into that table - inserted := &account{ - AccountID: uuid.New().String(), - AccountName: "some-account", - } - err = insertInto(ctx, db, inserted) - s.Assert().NoError(err) - - // let us select the record inserted - selected := &account{} - err = db.Select(ctx, selected, selectSQL, inserted.AccountID) - s.Assert().NoError(err) - - // let us compare the selected data and the record added - s.Assert().Equal(inserted.AccountID, selected.AccountID) - s.Assert().Equal(inserted.AccountName, selected.AccountName) - }) - - s.Run("with no records", func() { - // first drop the table - err = db.DropTable(ctx, "accounts") - s.Assert().NoError(err) - - // create the database table - err = createTable(ctx, db) - s.Assert().NoError(err) - - var selected *account - err = db.Select(ctx, selected, selectSQL, uuid.New().String()) - s.Assert().NoError(err) - s.Assert().Nil(selected) - }) - - s.Run("with invalid SQL statement", func() { - var selected *account - err = db.Select(ctx, selected, "weird-sql", uuid.New().String()) - s.Assert().Error(err) - s.Assert().Nil(selected) - }) -} - -func (s *PostgresTestSuite) TestSelectAll() { - ctx := context.TODO() - db := s.container.GetTestDB() - err := db.Connect(ctx) - s.Assert().NoError(err) - - const selectSQL = `SELECT account_id, account_name FROM accounts;` - - s.Run("with valid records", func() { - // first drop the table - err = db.DropTable(ctx, "accounts") - s.Assert().NoError(err) - - // create the database table - err = createTable(ctx, db) - s.Assert().NoError(err) - - // let us insert into that table - inserted := &account{ - AccountID: uuid.New().String(), - AccountName: "some-account", - } - err = insertInto(ctx, db, inserted) - s.Assert().NoError(err) - - // let us select the record inserted - var selected []*account - err = db.SelectAll(ctx, &selected, selectSQL) - s.Assert().NoError(err) - s.Assert().Equal(1, len(selected)) - }) - - s.Run("with no records", func() { - // first drop the table - err = db.DropTable(ctx, "accounts") - s.Assert().NoError(err) - - // create the database table - err = createTable(ctx, db) - s.Assert().NoError(err) - - var selected []*account - err = db.SelectAll(ctx, &selected, selectSQL) - s.Assert().NoError(err) - s.Assert().Nil(selected) - }) - - s.Run("with invalid SQL statement", func() { - var selected []*account - err = db.SelectAll(ctx, selected, "weird-sql", uuid.New().String()) - s.Assert().Error(err) - s.Assert().Nil(selected) - }) -} - -func (s *PostgresTestSuite) TestClose() { - ctx := context.TODO() - db := s.container.GetTestDB() - err := db.Connect(ctx) - s.Assert().NoError(err) - - // close the db connection - err = db.Disconnect(ctx) - s.Assert().NoError(err) - - // let us execute a query against a closed connection - err = db.TableExists(ctx, "accounts") - s.Assert().Error(err) - s.Assert().EqualError(err, "closed pool") -} - -func createTable(ctx context.Context, db Postgres) error { - // let us create a test table - const schemaDDL = ` - CREATE TABLE IF NOT EXISTS accounts - ( - account_id UUID, - account_name VARCHAR(255) NOT NULL, - PRIMARY KEY (account_id) - ); - ` - _, err := db.Exec(ctx, schemaDDL) - return err -} - -func insertInto(ctx context.Context, db Postgres, account *account) error { - const insertSQL = `INSERT INTO accounts(account_id, account_name) VALUES($1, $2);` - _, err := db.Exec(ctx, insertSQL, account.AccountID, account.AccountName) - return err -} diff --git a/internal/postgres/testkit.go b/internal/postgres/testkit.go deleted file mode 100644 index 77a8bf8..0000000 --- a/internal/postgres/testkit.go +++ /dev/null @@ -1,252 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "database/sql" - "errors" - "fmt" - "log" - "net" - "strconv" - "time" - - _ "github.com/lib/pq" //nolint - dockertest "github.com/ory/dockertest/v3" - "github.com/ory/dockertest/v3/docker" -) - -// TestContainer helps creates a Postgres docker container to -// run unit tests -type TestContainer struct { - host string - port int - schema string - - resource *dockertest.Resource - pool *dockertest.Pool - - // connection credentials - dbUser string - dbName string - dbPass string -} - -// NewTestContainer create a Postgres test container useful for unit and integration tests -// This function will exit when there is an error.Call this function inside your SetupTest to create the container before each test. -func NewTestContainer(dbName, dbUser, dbPassword string) *TestContainer { - // create the docker pool - pool, err := dockertest.NewPool("") - if err != nil { - log.Fatalf("Could not connect to docker: %s", err) - } - // pulls an image, creates a container based on it and runs it - resource, err := pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "postgres", - Tag: "11", - Env: []string{ - fmt.Sprintf("POSTGRES_PASSWORD=%s", dbPassword), - fmt.Sprintf("POSTGRES_USER=%s", dbUser), - fmt.Sprintf("POSTGRES_DB=%s", dbName), - "listen_addresses = '*'", - }, - Cmd: []string{ - "postgres", "-c", "log_statement=all", "-c", "log_connections=on", "-c", "log_disconnections=on", - }, - }, func(config *docker.HostConfig) { - // set AutoRemove to true so that stopped container goes away by itself - config.AutoRemove = true - config.RestartPolicy = docker.RestartPolicy{Name: "no"} - }) - // handle the error - if err != nil { - log.Fatalf("Could not start resource: %s", err) - } - // get the host and port of the database connection - hostAndPort := resource.GetHostPort("5432/tcp") - databaseURL := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", dbUser, dbPassword, hostAndPort, dbName) - log.Println("Connecting to database on url: ", databaseURL) - // Tell docker to hard kill the container in 120 seconds - _ = resource.Expire(120) - // exponential backoff-retry, because the application in the container might not be ready to accept connections yet - pool.MaxWait = 120 * time.Second - - if err = pool.Retry(func() error { - db, err := sql.Open("postgres", databaseURL) - if err != nil { - return err - } - return db.Ping() - }); err != nil { - log.Fatalf("Could not connect to docker: %s", err) - } - - // create an instance of TestContainer - container := new(TestContainer) - container.pool = pool - container.resource = resource - host, port, err := splitHostAndPort(hostAndPort) - if err != nil { - log.Fatalf("Unable to get database host and port: %s", err) - } - // set the container host, port and schema - container.dbName = dbName - container.dbUser = dbUser - container.dbPass = dbPassword - container.host = host - container.port = port - container.schema = "public" - return container -} - -// GetTestDB returns a Postgres TestDB that can be used in the tests -// to perform some database queries -func (c TestContainer) GetTestDB() *TestDB { - return &TestDB{ - New(&Config{ - DBHost: c.host, - DBPort: c.port, - DBName: c.dbName, - DBUser: c.dbUser, - DBPassword: c.dbPass, - DBSchema: c.schema, - MaxConnections: 4, - MinConnections: 0, - MaxConnectionLifetime: time.Hour, - MaxConnIdleTime: 30 * time.Minute, - HealthCheckPeriod: time.Minute, - }), - } -} - -// Host return the host of the test container -func (c TestContainer) Host() string { - return c.host -} - -// Port return the port of the test container -func (c TestContainer) Port() int { - return c.port -} - -// Schema return the test schema of the test container -func (c TestContainer) Schema() string { - return c.schema -} - -// Cleanup frees the resource by removing a container and linked volumes from docker. -// Call this function inside your TearDownSuite to clean-up resources after each test -func (c TestContainer) Cleanup() { - if err := c.pool.Purge(c.resource); err != nil { - log.Fatalf("Could not purge resource: %s", err) - } -} - -// TestDB is used in test to perform -// some database queries -type TestDB struct { - Postgres -} - -// DropTable utility function to drop a database table -func (c TestDB) DropTable(ctx context.Context, tableName string) error { - var dropSQL = fmt.Sprintf("DROP TABLE IF EXISTS %s CASCADE;", tableName) - _, err := c.Exec(ctx, dropSQL) - return err -} - -// TableExists utility function to help check the existence of table in Postgres -// tableName is in the format: . e.g: public.users -func (c TestDB) TableExists(ctx context.Context, tableName string) error { - var stmt = fmt.Sprintf("SELECT to_regclass('%s');", tableName) - _, err := c.Exec(ctx, stmt) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil - } - return err - } - - return nil -} - -// Count utility function to help count the number of rows in a Postgres table. -// tableName is in the format: . e.g: public.users -// It returns -1 when there is an error -func (c TestDB) Count(ctx context.Context, tableName string) (int, error) { - var count int - if err := c.Select(ctx, &count, fmt.Sprintf("SELECT COUNT(*) FROM %s", tableName)); err != nil { - return -1, err - } - return count, nil -} - -// CreateSchema helps create a test schema in a Postgres database -func (c TestDB) CreateSchema(ctx context.Context, schemaName string) error { - stmt := fmt.Sprintf("CREATE SCHEMA %s", schemaName) - if _, err := c.Exec(ctx, stmt); err != nil { - return err - } - return nil -} - -// SchemaExists helps check the existence of a Postgres schema. Very useful when implementing tests -func (c TestDB) SchemaExists(ctx context.Context, schemaName string) (bool, error) { - stmt := fmt.Sprintf("SELECT schema_name FROM information_schema.schemata WHERE schema_name = '%s';", schemaName) - var check string - if err := c.Select(ctx, &check, stmt); err != nil { - return false, err - } - - // this redundant check is necessary - if check == schemaName { - return true, nil - } - - return false, nil -} - -// DropSchema utility function to drop a database schema -func (c TestDB) DropSchema(ctx context.Context, schemaName string) error { - var dropSQL = fmt.Sprintf("DROP SCHEMA IF EXISTS %s CASCADE;", schemaName) - _, err := c.Exec(ctx, dropSQL) - return err -} - -// splitHostAndPort helps get the host address and port of and address -func splitHostAndPort(hostAndPort string) (string, int, error) { - host, port, err := net.SplitHostPort(hostAndPort) - if err != nil { - return "", -1, err - } - - portValue, err := strconv.Atoi(port) - if err != nil { - return "", -1, err - } - - return host, portValue, nil -} diff --git a/internal/postgres/testkit_test.go b/internal/postgres/testkit_test.go deleted file mode 100644 index 9d9360c..0000000 --- a/internal/postgres/testkit_test.go +++ /dev/null @@ -1,215 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "testing" - - "github.com/stretchr/testify/suite" -) - -type testkitSuite struct { - suite.Suite - container *TestContainer -} - -// SetupSuite starts the Postgres database engine and set the container -// host and port to use in the tests -func (s *testkitSuite) SetupSuite() { - s.container = NewTestContainer("testdb", "test", "test") -} - -func (s *testkitSuite) TearDownSuite() { - s.container.Cleanup() -} - -// In order for 'go test' to run this suite, we need to create -// a normal test function and pass our suite to suite.Run -func TestTestKitSuite(t *testing.T) { - suite.Run(t, new(testkitSuite)) -} - -func (s *testkitSuite) TestDropTable() { - s.Run("with no table defined", func() { - ctx := context.TODO() - db := s.container.GetTestDB() - - err := db.Connect(ctx) - s.Assert().NoError(err) - - // drop fake table - err = db.DropTable(ctx, "fake") - s.Assert().NoError(err) - s.Assert().Nil(err) - - err = db.Disconnect(ctx) - s.Assert().NoError(err) - }) -} - -func (s *testkitSuite) TestTableExist() { - s.Run("with no table defined", func() { - ctx := context.TODO() - db := s.container.GetTestDB() - - err := db.Connect(ctx) - s.Assert().NoError(err) - - // check fake table existence - err = db.TableExists(ctx, "fake") - s.Assert().NoError(err) - s.Assert().Nil(err) - err = db.Disconnect(ctx) - s.Assert().NoError(err) - }) -} - -func (s *testkitSuite) TestCreateAndCheckExistence() { - s.Run("happy path", func() { - ctx := context.TODO() - const schemaName = "example" - - db := s.container.GetTestDB() - - err := db.Connect(ctx) - s.Assert().NoError(err) - - err = db.CreateSchema(ctx, schemaName) - s.Assert().NoError(err) - - ok, err := db.SchemaExists(ctx, schemaName) - s.Assert().NoError(err) - s.Assert().True(ok) - - err = db.DropSchema(ctx, schemaName) - s.Assert().NoError(err) - - err = db.Disconnect(ctx) - s.Assert().NoError(err) - }) - s.Run("schema does not exist", func() { - ctx := context.TODO() - const schemaName = "example" - - db := s.container.GetTestDB() - - err := db.Connect(ctx) - s.Assert().NoError(err) - ok, err := db.SchemaExists(ctx, schemaName) - s.Assert().NoError(err) - s.Assert().False(ok) - - err = db.Disconnect(ctx) - s.Assert().NoError(err) - }) -} - -func (s *testkitSuite) TestCreateTable() { - s.Run("happy path", func() { - ctx := context.TODO() - const stmt = `create table mangoes(id serial, taste varchar(10));` - - db := s.container.GetTestDB() - - err := db.Connect(ctx) - s.Assert().NoError(err) - - _, err = db.Exec(ctx, stmt) - s.Assert().NoError(err) - - err = db.TableExists(ctx, "public.mangoes") - s.Assert().NoError(err) - s.Assert().Nil(err) - - err = db.DropTable(ctx, "public.mangoes") - s.Assert().NoError(err) - - err = db.Disconnect(ctx) - s.Assert().NoError(err) - }) - s.Run("happy path in a different schema", func() { - ctx := context.TODO() - const schemaName = "example" - const stmt = `create table example.mangoes(id serial, taste varchar(10));` - - db := s.container.GetTestDB() - - err := db.Connect(ctx) - s.Assert().NoError(err) - - err = db.CreateSchema(ctx, schemaName) - s.Assert().NoError(err) - - ok, err := db.SchemaExists(ctx, schemaName) - s.Assert().NoError(err) - s.Assert().True(ok) - - _, err = db.Exec(ctx, stmt) - s.Assert().NoError(err) - - err = db.TableExists(ctx, "example.mangoes") - s.Assert().NoError(err) - s.Assert().Nil(err) - - err = db.DropSchema(ctx, schemaName) - s.Assert().NoError(err) - }) -} - -func (s *testkitSuite) TestCount() { - ctx := context.TODO() - const schemaName = "example" - const stmt = `create table example.mangoes(id serial, taste varchar(10));` - - db := s.container.GetTestDB() - - err := db.Connect(ctx) - s.Assert().NoError(err) - - err = db.CreateSchema(ctx, schemaName) - s.Assert().NoError(err) - - ok, err := db.SchemaExists(ctx, schemaName) - s.Assert().NoError(err) - s.Assert().True(ok) - - _, err = db.Exec(ctx, stmt) - s.Assert().NoError(err) - - err = db.TableExists(ctx, "example.mangoes") - s.Assert().NoError(err) - s.Assert().Nil(err) - - count, err := db.Count(ctx, "example.mangoes") - s.Assert().NoError(err) - s.Assert().Equal(0, count) - - err = db.DropSchema(ctx, schemaName) - s.Assert().NoError(err) - - err = db.Disconnect(ctx) - s.Assert().NoError(err) -} diff --git a/offsetstore/memory/memory.go b/offsetstore/memory/memory.go deleted file mode 100644 index e29c0ab..0000000 --- a/offsetstore/memory/memory.go +++ /dev/null @@ -1,248 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package memory - -import ( - "context" - "errors" - "fmt" - "time" - - "github.com/google/uuid" - "github.com/hashicorp/go-memdb" - "go.uber.org/atomic" - - "github.com/tochemey/ego/v3/egopb" - "github.com/tochemey/ego/v3/offsetstore" -) - -// OffsetStore implements the offset store interface -// NOTE: NOT RECOMMENDED FOR PRODUCTION CODE because all records are in memory and there is no durability. -// This is recommended for tests or PoC -type OffsetStore struct { - // specifies the underlying database - db *memdb.MemDB - // this is only useful for tests - KeepRecordsAfterDisconnect bool - // hold the connection state to avoid multiple connection of the same instance - connected *atomic.Bool -} - -var _ offsetstore.OffsetStore = &OffsetStore{} - -// NewOffsetStore creates an instance of OffsetStore -func NewOffsetStore() *OffsetStore { - return &OffsetStore{ - KeepRecordsAfterDisconnect: false, - connected: atomic.NewBool(false), - } -} - -// Connect connects to the offset store -func (x *OffsetStore) Connect(context.Context) error { - // check whether this instance of the journal is connected or not - if x.connected.Load() { - return nil - } - - // create an instance of the database - db, err := memdb.NewMemDB(offsetSchema) - // handle the eventual error - if err != nil { - return err - } - // set the journal store underlying database - x.db = db - - // set the connection status - x.connected.Store(true) - - return nil -} - -// Disconnect disconnects the offset store -func (x *OffsetStore) Disconnect(context.Context) error { - // check whether this instance of the journal is connected or not - if !x.connected.Load() { - return nil - } - - // clear all records - if !x.KeepRecordsAfterDisconnect { - // spawn a db transaction for read-only - txn := x.db.Txn(true) - - // free memory resource - if _, err := txn.DeleteAll(offsetTableName, offsetPK); err != nil { - txn.Abort() - return fmt.Errorf("failed to free memory resource: %w", err) - } - txn.Commit() - } - // set the connection status - x.connected.Store(false) - - return nil -} - -// Ping verifies a connection to the database is still alive, establishing a connection if necessary. -func (x *OffsetStore) Ping(ctx context.Context) error { - // check whether we are connected or not - if !x.connected.Load() { - return x.Connect(ctx) - } - - return nil -} - -// WriteOffset writes an offset to the offset store -func (x *OffsetStore) WriteOffset(_ context.Context, offset *egopb.Offset) error { - // check whether this instance of the journal is connected or not - if !x.connected.Load() { - return errors.New("offset store is not connected") - } - - // spawn a db transaction - txn := x.db.Txn(true) - - // create an offset row - record := &offsetRow{ - Ordering: uuid.NewString(), - ProjectionName: offset.GetProjectionName(), - ShardNumber: offset.GetShardNumber(), - Value: offset.GetValue(), - Timestamp: offset.GetTimestamp(), - } - - // persist the record - if err := txn.Insert(offsetTableName, record); err != nil { - // abort the transaction - txn.Abort() - // return the error - return fmt.Errorf("failed to persist offset record on to the offset store: %w", err) - } - // commit the transaction - txn.Commit() - - return nil -} - -// GetCurrentOffset return the offset of a projection -func (x *OffsetStore) GetCurrentOffset(_ context.Context, projectionID *egopb.ProjectionId) (current *egopb.Offset, err error) { - // check whether this instance of the journal is connected or not - if !x.connected.Load() { - return nil, errors.New("offset store is not connected") - } - - // spawn a db transaction for read-only - txn := x.db.Txn(false) - defer txn.Abort() - // let us fetch the last record - raw, err := txn.Last(offsetTableName, rowIndex, projectionID.GetProjectionName(), projectionID.GetShardNumber()) - if err != nil { - // if the error is not found then return nil - if errors.Is(err, memdb.ErrNotFound) { - return nil, nil - } - return nil, fmt.Errorf( - "failed to get the current offset for shard=%d given projection=%s: %w", - projectionID.GetShardNumber(), - projectionID.GetProjectionName(), - err) - } - - // no record found - if raw == nil { - return nil, nil - } - - // cast the record - if offsetRow, ok := raw.(*offsetRow); ok { - current = &egopb.Offset{ - ShardNumber: offsetRow.ShardNumber, - ProjectionName: offsetRow.ProjectionName, - Value: offsetRow.Value, - Timestamp: offsetRow.Timestamp, - } - return - } - - return nil, fmt.Errorf("failed to get the current offset for shard=%d given projection=%s", - projectionID.GetShardNumber(), projectionID.GetProjectionName()) -} - -// ResetOffset resets the offset of given projection to a given value across all shards -func (x *OffsetStore) ResetOffset(_ context.Context, projectionName string, value int64) error { - // check whether this instance of the offset store is connected or not - if !x.connected.Load() { - return errors.New("offset store is not connected") - } - - // spawn a db transaction for read-only - txn := x.db.Txn(false) - // fetch all the records for the given projection - it, err := txn.Get(offsetTableName, projectionNameIndex, projectionName) - // handle the error - if err != nil { - // abort the transaction - txn.Abort() - return fmt.Errorf("failed to fetch the list of shard number: %w", err) - } - - // loop over the records - var offsetRows []*offsetRow - for row := it.Next(); row != nil; row = it.Next() { - if journal, ok := row.(*offsetRow); ok { - offsetRows = append(offsetRows, journal) - } - } - // let us abort the transaction after fetching the matching records - txn.Abort() - - // update the records - ts := time.Now().UnixMilli() - for _, row := range offsetRows { - row.Value = value - row.Timestamp = ts - } - - // spawn a db write transaction - txn = x.db.Txn(true) - // iterate the list of offset rows and update the values - for _, row := range offsetRows { - // persist the record - if err := txn.Insert(offsetTableName, row); err != nil { - // abort the transaction - txn.Abort() - // return the error - return fmt.Errorf("failed to persist offset record on to the offset store: %w", err) - } - } - - // commit the transaction - txn.Commit() - - return nil -} diff --git a/offsetstore/memory/memory_test.go b/offsetstore/memory/memory_test.go deleted file mode 100644 index 3550f44..0000000 --- a/offsetstore/memory/memory_test.go +++ /dev/null @@ -1,192 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package memory - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" - - "github.com/tochemey/ego/v3/egopb" - "github.com/tochemey/ego/v3/offsetstore" -) - -func TestOffsetStore(t *testing.T) { - t.Run("testNew", func(t *testing.T) { - store := NewOffsetStore() - assert.NotNil(t, store) - var p interface{} = store - _, ok := p.(offsetstore.OffsetStore) - assert.True(t, ok) - }) - t.Run("testConnect", func(t *testing.T) { - ctx := context.TODO() - store := NewOffsetStore() - assert.NotNil(t, store) - err := store.Connect(ctx) - assert.NoError(t, err) - }) - t.Run("testWriteOffset", func(t *testing.T) { - ctx := context.TODO() - - store := NewOffsetStore() - assert.NotNil(t, store) - require.NoError(t, store.Connect(ctx)) - - shardNumber := uint64(9) - projectionName := "DB_WRITER" - timestamp := time.Now().UnixMilli() - - offset := &egopb.Offset{ - ShardNumber: shardNumber, - ProjectionName: projectionName, - Value: 15, - Timestamp: timestamp, - } - - require.NoError(t, store.WriteOffset(ctx, offset)) - - err := store.Disconnect(ctx) - assert.NoError(t, err) - }) - t.Run("testGetCurrentOffset: happy path", func(t *testing.T) { - ctx := context.TODO() - - store := NewOffsetStore() - assert.NotNil(t, store) - require.NoError(t, store.Connect(ctx)) - - shardNumber := uint64(9) - projectionName := "DB_WRITER" - timestamp := time.Now().UnixMilli() - - offset := &egopb.Offset{ - ShardNumber: shardNumber, - ProjectionName: projectionName, - Value: 15, - Timestamp: timestamp, - } - - require.NoError(t, store.WriteOffset(ctx, offset)) - - offset = &egopb.Offset{ - ShardNumber: shardNumber, - ProjectionName: projectionName, - Value: 24, - Timestamp: timestamp, - } - - require.NoError(t, store.WriteOffset(ctx, offset)) - - projectionID := &egopb.ProjectionId{ - ProjectionName: projectionName, - ShardNumber: shardNumber, - } - - actual, err := store.GetCurrentOffset(ctx, projectionID) - assert.NoError(t, err) - assert.NotNil(t, actual) - assert.True(t, proto.Equal(offset, actual)) - - assert.NoError(t, store.Disconnect(ctx)) - }) - t.Run("testGetCurrentOffset: not found", func(t *testing.T) { - ctx := context.TODO() - - store := NewOffsetStore() - assert.NotNil(t, store) - require.NoError(t, store.Connect(ctx)) - - shardNumber := uint64(9) - projectionName := "DB_WRITER" - projectionID := &egopb.ProjectionId{ - ProjectionName: projectionName, - ShardNumber: shardNumber, - } - actual, err := store.GetCurrentOffset(ctx, projectionID) - assert.NoError(t, err) - assert.Nil(t, actual) - - assert.NoError(t, store.Disconnect(ctx)) - }) - t.Run("testResetOffset: happy path", func(t *testing.T) { - ctx := context.TODO() - - store := NewOffsetStore() - assert.NotNil(t, store) - require.NoError(t, store.Connect(ctx)) - - projectionName := "DB_WRITER" - timestamp := time.Now().UnixMilli() - - offset := &egopb.Offset{ - ShardNumber: 9, - ProjectionName: projectionName, - Value: 15, - Timestamp: timestamp, - } - - require.NoError(t, store.WriteOffset(ctx, offset)) - - offset = &egopb.Offset{ - ShardNumber: 8, - ProjectionName: projectionName, - Value: 24, - Timestamp: timestamp, - } - - require.NoError(t, store.WriteOffset(ctx, offset)) - - shardNumber := uint64(9) - projectionID := &egopb.ProjectionId{ - ProjectionName: projectionName, - ShardNumber: shardNumber, - } - - actual, err := store.GetCurrentOffset(ctx, projectionID) - assert.NoError(t, err) - assert.NotNil(t, actual) - expected := &egopb.Offset{ - ShardNumber: 9, - ProjectionName: projectionName, - Value: 15, - Timestamp: timestamp, - } - assert.True(t, proto.Equal(expected, actual)) - - // reset the offset - require.NoError(t, store.ResetOffset(ctx, projectionName, 100)) - actual, err = store.GetCurrentOffset(ctx, projectionID) - assert.NoError(t, err) - assert.NotNil(t, actual) - assert.EqualValues(t, 100, actual.GetValue()) - - assert.NoError(t, store.Disconnect(ctx)) - }) -} diff --git a/offsetstore/memory/schemas.go b/offsetstore/memory/schemas.go deleted file mode 100644 index f804f8a..0000000 --- a/offsetstore/memory/schemas.go +++ /dev/null @@ -1,113 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package memory - -import "github.com/hashicorp/go-memdb" - -// offsetRow represent the offset entry in the offset store -type offsetRow struct { - // Ordering basically used as PK - Ordering string - // ProjectionName is the projection name - ProjectionName string - // Shard Number - ShardNumber uint64 - // Value is the current offset - Value int64 - // Specifies the last update time - Timestamp int64 -} - -const ( - offsetTableName = "offsets" - offsetPK = "id" - currentOffsetIndex = "currentOffset" - projectionNameIndex = "projectionName" - rowIndex = "rowIndex" - shardNumberIndex = "shardNumber" -) - -var ( - // offsetSchema defines the offset schema - offsetSchema = &memdb.DBSchema{ - Tables: map[string]*memdb.TableSchema{ - offsetTableName: { - Name: offsetTableName, - Indexes: map[string]*memdb.IndexSchema{ - offsetPK: { - Name: offsetPK, - AllowMissing: false, - Unique: true, - Indexer: &memdb.StringFieldIndex{ - Field: "Ordering", - Lowercase: false, - }, - }, - shardNumberIndex: { - Name: shardNumberIndex, - AllowMissing: false, - Unique: false, - Indexer: &memdb.UintFieldIndex{ - Field: "ShardNumber", - }, - }, - currentOffsetIndex: { - Name: currentOffsetIndex, - AllowMissing: false, - Unique: false, - Indexer: &memdb.IntFieldIndex{ - Field: "Value", - }, - }, - projectionNameIndex: { - Name: projectionNameIndex, - AllowMissing: false, - Unique: false, - Indexer: &memdb.StringFieldIndex{ - Field: "ProjectionName", - }, - }, - rowIndex: { - Name: rowIndex, - AllowMissing: false, - Unique: true, - Indexer: &memdb.CompoundIndex{ - Indexes: []memdb.Indexer{ - &memdb.StringFieldIndex{ - Field: "ProjectionName", - Lowercase: false, - }, - &memdb.UintFieldIndex{ - Field: "ShardNumber", - }, - }, - AllowMissing: false, - }, - }, - }, - }, - }, - } -) diff --git a/offsetstore/postgres/config.go b/offsetstore/postgres/config.go deleted file mode 100644 index b1717c4..0000000 --- a/offsetstore/postgres/config.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -// Config defines the postgres offset store configuration -type Config struct { - DBHost string // DBHost represents the database host - DBPort int // DBPort is the database port - DBName string // DBName is the database name - DBUser string // DBUser is the database user used to connect - DBPassword string // DBPassword is the database password - DBSchema string // DBSchema represents the database schema -} diff --git a/offsetstore/postgres/helper_test.go b/offsetstore/postgres/helper_test.go deleted file mode 100644 index 8fe971a..0000000 --- a/offsetstore/postgres/helper_test.go +++ /dev/null @@ -1,63 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "os" - "testing" - - "github.com/tochemey/ego/v3/internal/postgres" -) - -var testContainer *postgres.TestContainer - -const ( - testUser = "test" - testDatabase = "testdb" - testDatabasePassword = "test" -) - -// TestMain will spawn a postgres database container that will be used for all tests -// making use of the postgres database container -func TestMain(m *testing.M) { - // set the test container - testContainer = postgres.NewTestContainer(testDatabase, testUser, testDatabasePassword) - // execute the tests - code := m.Run() - // free resources - testContainer.Cleanup() - // exit the tests - os.Exit(code) -} - -// dbHandle returns a test db -func dbHandle(ctx context.Context) (*postgres.TestDB, error) { - db := testContainer.GetTestDB() - if err := db.Connect(ctx); err != nil { - return nil, err - } - return db, nil -} diff --git a/offsetstore/postgres/postgres.go b/offsetstore/postgres/postgres.go deleted file mode 100644 index 4000227..0000000 --- a/offsetstore/postgres/postgres.go +++ /dev/null @@ -1,310 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "errors" - "fmt" - "time" - - sq "github.com/Masterminds/squirrel" - "github.com/jackc/pgx/v5" - "go.uber.org/atomic" - "google.golang.org/protobuf/proto" - - "github.com/tochemey/ego/v3/egopb" - "github.com/tochemey/ego/v3/internal/postgres" - "github.com/tochemey/ego/v3/offsetstore" -) - -var ( - columns = []string{ - "projection_name", - "shard_number", - "current_offset", - "timestamp", - } - - tableName = "offsets_store" -) - -// offsetRow represent the offset entry in the offset store -type offsetRow struct { - // ProjectionName is the projection name - ProjectionName string - // Shard Number - ShardNumber uint64 - // Value is the current offset - CurrentOffset int64 - // Specifies the last update time - Timestamp int64 -} - -// OffsetStore implements the OffsetStore interface -// and helps persist events in a Postgres database -type OffsetStore struct { - db postgres.Postgres - sb sq.StatementBuilderType - // insertBatchSize represents the chunk of data to bulk insert. - // This helps avoid the postgres 65535 parameter limit. - // This is necessary because Postgres uses a 32-bit int for binding input parameters and - // is not able to track anything larger. - // Note: Change this value when you know the size of data to bulk insert at once. Otherwise, you - // might encounter the postgres 65535 parameter limit error. - insertBatchSize int - // hold the connection state to avoid multiple connection of the same instance - connected *atomic.Bool -} - -// ensure the complete implementation of the OffsetStore interface -var _ offsetstore.OffsetStore = (*OffsetStore)(nil) - -// NewOffsetStore creates an instance of OffsetStore -func NewOffsetStore(config *Config) *OffsetStore { - // create the underlying db connection - db := postgres.New(postgres.NewConfig(config.DBHost, config.DBPort, config.DBUser, config.DBPassword, config.DBName)) - return &OffsetStore{ - db: db, - sb: sq.StatementBuilder.PlaceholderFormat(sq.Dollar), - insertBatchSize: 500, - connected: atomic.NewBool(false), - } -} - -// Connect connects to the underlying postgres database -func (x *OffsetStore) Connect(ctx context.Context) error { - // check whether this instance of the journal is connected or not - if x.connected.Load() { - return nil - } - - // connect to the underlying db - if err := x.db.Connect(ctx); err != nil { - return err - } - - // set the connection status - x.connected.Store(true) - - return nil -} - -// Disconnect disconnects from the underlying postgres database -func (x *OffsetStore) Disconnect(ctx context.Context) error { - // check whether this instance of the journal is connected or not - if !x.connected.Load() { - return nil - } - - // disconnect the underlying database - if err := x.db.Disconnect(ctx); err != nil { - return err - } - // set the connection status - x.connected.Store(false) - - return nil -} - -// WriteOffset writes an offset into the offset store -func (x *OffsetStore) WriteOffset(ctx context.Context, offset *egopb.Offset) error { - // check whether this instance of the offset store is connected or not - if !x.connected.Load() { - return errors.New("offset store is not connected") - } - - // make sure the record is defined - if offset == nil || proto.Equal(offset, new(egopb.Offset)) { - return errors.New("offset record is not defined") - } - - // let us begin a database transaction to make sure we atomically write those events into the database - tx, err := x.db.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) - // return the error in case we are unable to get a database transaction - if err != nil { - return fmt.Errorf("failed to obtain a database transaction: %w", err) - } - - var ( - query string - args []any - ) - - // remove existing offset - deleteBuilder := x.sb. - Delete(tableName). - Where(sq.Eq{"projection_name": offset.GetProjectionName()}). - Where(sq.Eq{"shard_number": offset.GetShardNumber()}) - - // get the SQL statement to run - query, args, err = deleteBuilder.ToSql() - // handle the error while generating the SQL - if err != nil { - return fmt.Errorf("unable to build sql delete statement: %w", err) - } - - // execute the query - _, execErr := tx.Exec(ctx, query, args...) - if execErr != nil { - // attempt to roll back the transaction and log the error in case there is an error - if err = tx.Rollback(ctx); err != nil { - return fmt.Errorf("unable to rollback db transaction: %w", err) - } - // return the main error - return fmt.Errorf("failed to record events: %w", execErr) - } - - // create the insert statement - insertBuilder := x.sb. - Insert(tableName). - Columns(columns...). - Values( - offset.GetProjectionName(), - offset.GetShardNumber(), - offset.GetValue(), - offset.GetTimestamp()) - - // get the SQL statement to run - query, args, err = insertBuilder.ToSql() - // handle the error while generating the SQL - if err != nil { - return fmt.Errorf("unable to build sql insert statement: %w", err) - } - - // insert into the table - _, execErr = tx.Exec(ctx, query, args...) - if execErr != nil { - // attempt to roll back the transaction and log the error in case there is an error - if err = tx.Rollback(ctx); err != nil { - return fmt.Errorf("unable to rollback db transaction: %w", err) - } - // return the main error - return fmt.Errorf("failed to record events: %w", execErr) - } - - // commit the transaction - if commitErr := tx.Commit(ctx); commitErr != nil { - // return the commit error in case there is one - return fmt.Errorf("failed to record events: %w", commitErr) - } - // every looks good - return nil -} - -// GetCurrentOffset returns the current offset of a given projection id -func (x *OffsetStore) GetCurrentOffset(ctx context.Context, projectionID *egopb.ProjectionId) (currentOffset *egopb.Offset, err error) { - // check whether this instance of the offset store is connected or not - if !x.connected.Load() { - return nil, errors.New("offset store is not connected") - } - - // create the SQL statement - statement := x.sb. - Select(columns...). - From(tableName). - Where(sq.Eq{"projection_name": projectionID.GetProjectionName()}). - Where(sq.Eq{"shard_number": projectionID.GetShardNumber()}) - - // get the sql statement and the arguments - query, args, err := statement.ToSql() - if err != nil { - return nil, fmt.Errorf("failed to build the select sql statement: %w", err) - } - - row := new(offsetRow) - err = x.db.Select(ctx, row, query, args...) - if err != nil { - return nil, fmt.Errorf("failed to fetch the current offset from the database: %w", err) - } - - return &egopb.Offset{ - ShardNumber: row.ShardNumber, - ProjectionName: row.ProjectionName, - Value: row.CurrentOffset, - Timestamp: row.Timestamp, - }, nil -} - -// ResetOffset resets the offset of given projection to a given value across all shards -func (x *OffsetStore) ResetOffset(ctx context.Context, projectionName string, value int64) error { - // check whether this instance of the offset store is connected or not - if !x.connected.Load() { - return errors.New("offset store is not connected") - } - - // let us begin a database transaction to make sure we atomically write those events into the database - tx, err := x.db.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) - // return the error in case we are unable to get a database transaction - if err != nil { - return fmt.Errorf("failed to obtain a database transaction: %w", err) - } - - // define the current timestamp - timestamp := time.Now().UnixMilli() - - // create the sql statement - statement := x.sb. - Update(tableName). - Set("current_offset", value). - Set("timestamp", timestamp). - Where(sq.Eq{"projection_name": projectionName}) - - // get the SQL statement to run - query, args, err := statement.ToSql() - // handle the error while generating the SQL - if err != nil { - return fmt.Errorf("unable to build sql insert statement: %w", err) - } - - // insert into the table - _, execErr := tx.Exec(ctx, query, args...) - if execErr != nil { - // attempt to roll back the transaction and log the error in case there is an error - if err = tx.Rollback(ctx); err != nil { - return fmt.Errorf("unable to rollback db transaction: %w", err) - } - // return the main error - return fmt.Errorf("failed to record events: %w", execErr) - } - - // commit the transaction - if commitErr := tx.Commit(ctx); commitErr != nil { - // return the commit error in case there is one - return fmt.Errorf("failed to record events: %w", commitErr) - } - // every looks good - return nil -} - -// Ping verifies a connection to the database is still alive, establishing a connection if necessary. -func (x *OffsetStore) Ping(ctx context.Context) error { - // check whether we are connected or not - if !x.connected.Load() { - return x.Connect(ctx) - } - - return nil -} diff --git a/offsetstore/postgres/postgres_test.go b/offsetstore/postgres/postgres_test.go deleted file mode 100644 index 64c965d..0000000 --- a/offsetstore/postgres/postgres_test.go +++ /dev/null @@ -1,218 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" - - "github.com/tochemey/ego/v3/egopb" - "github.com/tochemey/ego/v3/offsetstore" -) - -func TestPostgresOffsetStore(t *testing.T) { - t.Run("testNewOffsetStore", func(t *testing.T) { - config := &Config{ - testContainer.Host(), - testContainer.Port(), - testDatabase, - testUser, - testDatabasePassword, - testContainer.Schema(), - } - - estore := NewOffsetStore(config) - assert.NotNil(t, estore) - var p interface{} = estore - _, ok := p.(offsetstore.OffsetStore) - assert.True(t, ok) - }) - t.Run("testConnect:happy path", func(t *testing.T) { - ctx := context.TODO() - config := &Config{ - testContainer.Host(), - testContainer.Port(), - testDatabase, - testUser, - testDatabasePassword, - testContainer.Schema(), - } - - store := NewOffsetStore(config) - assert.NotNil(t, store) - err := store.Connect(ctx) - assert.NoError(t, err) - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) - t.Run("testConnect:database does not exist", func(t *testing.T) { - ctx := context.TODO() - config := &Config{ - testContainer.Host(), - testContainer.Port(), - "testDatabase", - testUser, - testDatabasePassword, - testContainer.Schema(), - } - - store := NewOffsetStore(config) - assert.NotNil(t, store) - err := store.Connect(ctx) - assert.Error(t, err) - }) - t.Run("testWriteOffset", func(t *testing.T) { - ctx := context.TODO() - config := &Config{ - testContainer.Host(), - testContainer.Port(), - testDatabase, - testUser, - testDatabasePassword, - testContainer.Schema(), - } - - db, err := dbHandle(ctx) - require.NoError(t, err) - schemaUtil := NewSchemaUtils(db) - err = schemaUtil.CreateTable(ctx) - require.NoError(t, err) - - store := NewOffsetStore(config) - assert.NotNil(t, store) - err = store.Connect(ctx) - require.NoError(t, err) - - offset := &egopb.Offset{ - ShardNumber: uint64(9), - ProjectionName: "some-projection", - Value: int64(10), - Timestamp: time.Now().UnixMilli(), - } - - // write the offset - assert.NoError(t, store.WriteOffset(ctx, offset)) - - // get the current shard offset - current, err := store.GetCurrentOffset(ctx, &egopb.ProjectionId{ - ProjectionName: "some-projection", - ShardNumber: uint64(9), - }) - require.NoError(t, err) - assert.True(t, proto.Equal(offset, current)) - - err = schemaUtil.DropTable(ctx) - assert.NoError(t, err) - - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) - t.Run("testResetOffset", func(t *testing.T) { - ctx := context.TODO() - config := &Config{ - testContainer.Host(), - testContainer.Port(), - testDatabase, - testUser, - testDatabasePassword, - testContainer.Schema(), - } - - db, err := dbHandle(ctx) - require.NoError(t, err) - schemaUtil := NewSchemaUtils(db) - err = schemaUtil.CreateTable(ctx) - require.NoError(t, err) - - store := NewOffsetStore(config) - assert.NotNil(t, store) - err = store.Connect(ctx) - require.NoError(t, err) - - projection1 := "projection-1" - projection2 := "projection-2" - ts := time.Now().UnixMilli() - // write offset into 10 shards for projection1 - for i := 0; i < 10; i++ { - shard := uint64(i + 1) - offset := &egopb.Offset{ - ShardNumber: shard, - ProjectionName: projection1, - Value: int64(i + 1), - Timestamp: ts, - } - // write the offset - assert.NoError(t, store.WriteOffset(ctx, offset)) - } - - // write offset into 10 for projection2 - // this only to have multiple records in the storage - for i := 0; i < 10; i++ { - shard := uint64(i + 1) - offset := &egopb.Offset{ - ShardNumber: shard, - ProjectionName: projection2, - Value: 2 * int64(i+1), - Timestamp: ts, - } - // write the offset - assert.NoError(t, store.WriteOffset(ctx, offset)) - } - - // get the current shard offset - current, err := store.GetCurrentOffset(ctx, &egopb.ProjectionId{ - ProjectionName: projection1, - ShardNumber: uint64(9), - }) - require.NoError(t, err) - expected := &egopb.Offset{ - ShardNumber: uint64(9), - ProjectionName: projection1, - Value: int64(9), - Timestamp: ts, - } - assert.True(t, proto.Equal(expected, current)) - - // reset the projection 1 offset - require.NoError(t, store.ResetOffset(ctx, projection1, int64(1000))) - current, err = store.GetCurrentOffset(ctx, &egopb.ProjectionId{ - ProjectionName: projection1, - ShardNumber: uint64(9), - }) - require.NoError(t, err) - assert.EqualValues(t, 1000, current.GetValue()) - - err = schemaUtil.DropTable(ctx) - assert.NoError(t, err) - - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) -} diff --git a/offsetstore/postgres/schema_utils.go b/offsetstore/postgres/schema_utils.go deleted file mode 100644 index 4397594..0000000 --- a/offsetstore/postgres/schema_utils.go +++ /dev/null @@ -1,68 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - - "github.com/tochemey/ego/v3/internal/postgres" -) - -// SchemaUtils help create the various test tables in unit/integration tests -type SchemaUtils struct { - db *postgres.TestDB -} - -// NewSchemaUtils creates an instance of SchemaUtils -func NewSchemaUtils(db *postgres.TestDB) *SchemaUtils { - return &SchemaUtils{db: db} -} - -// CreateTable creates the event store table used for unit tests -func (d SchemaUtils) CreateTable(ctx context.Context) error { - schemaDDL := ` - DROP TABLE IF EXISTS offsets_store; - CREATE TABLE IF NOT EXISTS offsets_store - ( - projection_name VARCHAR(255) NOT NULL, - shard_number BIGINT NOT NULL, - current_offset BIGINT NOT NULL, - timestamp BIGINT NOT NULL, - PRIMARY KEY (projection_name, shard_number) - ); - ---- create an index on the projection_name column -CREATE INDEX IF NOT EXISTS idx_offsets_store_name ON offsets_store (projection_name); -CREATE INDEX IF NOT EXISTS idx_offsets_store_shard ON offsets_store (shard_number); - ` - _, err := d.db.Exec(ctx, schemaDDL) - return err -} - -// DropTable drop the table used in unit test -// This is useful for resource cleanup after a unit test -func (d SchemaUtils) DropTable(ctx context.Context) error { - return d.db.DropTable(ctx, "offsets_store") -} diff --git a/offsetstore/postgres/schema_utils_test.go b/offsetstore/postgres/schema_utils_test.go deleted file mode 100644 index d5f427c..0000000 --- a/offsetstore/postgres/schema_utils_test.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSchemaUtils(t *testing.T) { - ctx := context.TODO() - db, err := dbHandle(ctx) - assert.NoError(t, err) - - // create the tables - schemaUtils := NewSchemaUtils(db) - err = schemaUtils.CreateTable(ctx) - assert.NoError(t, err) - - // assert existence of the table - err = db.TableExists(ctx, "offsets_store") - assert.NoError(t, err) - - // clean up - assert.NoError(t, schemaUtils.DropTable(ctx)) - assert.NoError(t, db.Disconnect(ctx)) -} diff --git a/plugins/eventstore/memory/memory.go b/plugins/eventstore/memory/memory.go deleted file mode 100644 index 148a7b0..0000000 --- a/plugins/eventstore/memory/memory.go +++ /dev/null @@ -1,559 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package memory - -import ( - "context" - "errors" - "fmt" - "sort" - - goset "github.com/deckarep/golang-set/v2" - "github.com/google/uuid" - "github.com/hashicorp/go-memdb" - "go.uber.org/atomic" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protoreflect" - "google.golang.org/protobuf/reflect/protoregistry" - "google.golang.org/protobuf/types/known/anypb" - - "github.com/tochemey/ego/v3/egopb" - "github.com/tochemey/ego/v3/persistence" -) - -// EventsStore keep in memory every journal -// NOTE: NOT RECOMMENDED FOR PRODUCTION CODE because all records are in memory and there is no durability. -// This is recommended for tests or PoC -type EventsStore struct { - // specifies the underlying database - db *memdb.MemDB - // this is only useful for tests - KeepRecordsAfterDisconnect bool - // hold the connection state to avoid multiple connection of the same instance - connected *atomic.Bool -} - -// enforce interface implementation -var _ persistence.EventsStore = (*EventsStore)(nil) - -// NewEventsStore creates a new instance of EventsStore -func NewEventsStore() *EventsStore { - return &EventsStore{ - KeepRecordsAfterDisconnect: false, - connected: atomic.NewBool(false), - } -} - -// Connect connects to the journal store -func (s *EventsStore) Connect(context.Context) error { - // check whether this instance of the journal is connected or not - if s.connected.Load() { - return nil - } - - // create an instance of the database - db, err := memdb.NewMemDB(journalSchema) - // handle the eventual error - if err != nil { - return err - } - // set the journal store underlying database - s.db = db - - // set the connection status - s.connected.Store(true) - - return nil -} - -// Disconnect disconnect the journal store -func (s *EventsStore) Disconnect(context.Context) error { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil - } - - // clear all records - if !s.KeepRecordsAfterDisconnect { - // spawn a db transaction for read-only - txn := s.db.Txn(true) - - // free memory resource - if _, err := txn.DeleteAll(journalTableName, journalPK); err != nil { - txn.Abort() - return fmt.Errorf("failed to free memory resource: %w", err) - } - txn.Commit() - } - - // set the connection status - s.connected.Store(false) - - return nil -} - -// Ping verifies a connection to the database is still alive, establishing a connection if necessary. -func (s *EventsStore) Ping(ctx context.Context) error { - // check whether we are connected or not - if !s.connected.Load() { - return s.Connect(ctx) - } - - return nil -} - -// PersistenceIDs returns the distinct list of all the persistence ids in the journal store -// FIXME: enhance the implementation. As it stands it will be a bit slow when there are a lot of records -func (s *EventsStore) PersistenceIDs(_ context.Context, pageSize uint64, pageToken string) (persistenceIDs []string, nextPageToken string, err error) { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil, "", errors.New("journal store is not connected") - } - - // spawn a db transaction for read-only - txn := s.db.Txn(false) - defer txn.Abort() - - // define the records iterator and error variables - var ( - it memdb.ResultIterator - ) - - // check whether the page token is set - if pageToken != "" { - // execute the query to fetch the records - it, err = txn.LowerBound(journalTableName, persistenceIDIndex, pageToken) - } else { - // fetch all the records. default behavior - it, err = txn.Get(journalTableName, persistenceIDIndex) - } - - // handle the error - if err != nil { - return nil, "", fmt.Errorf("failed to get the persistence Ids: %w", err) - } - - var journals []*journal - // fetch the records - for row := it.Next(); row != nil; row = it.Next() { - // check whether we have reached the page size - if len(journals) == int(pageSize) { - break - } - // parse the next record - if journal, ok := row.(*journal); ok { - journals = append(journals, journal) - } - } - - // build the persistence ids fetched - // iterate the records that have been fetched earlier - for _, journal := range journals { - // TODO: refactor this cowboy code - // skip the page token record - if journal.PersistenceID == pageToken { - continue - } - // grab the id - persistenceIDs = append(persistenceIDs, journal.PersistenceID) - } - - // short-circuit when there are no records - if len(persistenceIDs) == 0 { - return nil, "", nil - } - - // let us sort the fetch ids - sort.SliceStable(persistenceIDs, func(i, j int) bool { - return persistenceIDs[i] <= persistenceIDs[j] - }) - - // set the next page token - nextPageToken = persistenceIDs[len(persistenceIDs)-1] - - return -} - -// WriteEvents persist events in batches for a given persistenceID -func (s *EventsStore) WriteEvents(_ context.Context, events []*egopb.Event) error { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return errors.New("journal store is not connected") - } - - // spawn a db transaction - txn := s.db.Txn(true) - // iterate the event and persist the record - for _, event := range events { - // serialize the event and resulting state - eventBytes, _ := proto.Marshal(event.GetEvent()) - stateBytes, _ := proto.Marshal(event.GetResultingState()) - - // grab the manifest - eventManifest := string(event.GetEvent().ProtoReflect().Descriptor().FullName()) - stateManifest := string(event.GetResultingState().ProtoReflect().Descriptor().FullName()) - - // create an instance of Journal - journal := &journal{ - Ordering: uuid.NewString(), - PersistenceID: event.GetPersistenceId(), - SequenceNumber: event.GetSequenceNumber(), - IsDeleted: event.GetIsDeleted(), - EventPayload: eventBytes, - EventManifest: eventManifest, - StatePayload: stateBytes, - StateManifest: stateManifest, - Timestamp: event.GetTimestamp(), - ShardNumber: event.GetShard(), - } - - // persist the record - if err := txn.Insert(journalTableName, journal); err != nil { - // abort the transaction - txn.Abort() - // return the error - return fmt.Errorf("failed to persist event on to the journal store: %w", err) - } - } - // commit the transaction - txn.Commit() - - return nil -} - -// DeleteEvents deletes events from the store upt to a given sequence number (inclusive) -// FIXME: enhance the implementation. As it stands it may be a bit slow when there are a lot of records -func (s *EventsStore) DeleteEvents(_ context.Context, persistenceID string, toSequenceNumber uint64) error { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return errors.New("journal store is not connected") - } - - // spawn a db transaction for read-only - txn := s.db.Txn(false) - // fetch all the records that are not deleted and filter them out - it, err := txn.Get(journalTableName, persistenceIDIndex, persistenceID) - // handle the error - if err != nil { - // abort the transaction - txn.Abort() - return fmt.Errorf("failed to delete %d persistenceId=%s events: %w", toSequenceNumber, persistenceID, err) - } - - // loop over the records and delete them - var journals []*journal - for row := it.Next(); row != nil; row = it.Next() { - if journal, ok := row.(*journal); ok { - journals = append(journals, journal) - } - } - // let us abort the transaction after fetching the matching records - txn.Abort() - - // now let us delete the records whose sequence number are less or equal to the given sequence number - // spawn a db transaction for write-only - txn = s.db.Txn(true) - - // iterate over the records and delete them - // TODO enhance this operation using the DeleteAll feature - for _, journal := range journals { - if journal.SequenceNumber <= toSequenceNumber { - // delete that record - if err := txn.Delete(journalTableName, journal); err != nil { - // abort the transaction - txn.Abort() - return fmt.Errorf("failed to delete %d persistenceId=%s events: %w", toSequenceNumber, persistenceID, err) - } - } - } - // commit the transaction - txn.Commit() - return nil -} - -// ReplayEvents fetches events for a given persistence ID from a given sequence number(inclusive) to a given sequence number(inclusive) -func (s *EventsStore) ReplayEvents(_ context.Context, persistenceID string, fromSequenceNumber, toSequenceNumber uint64, max uint64) ([]*egopb.Event, error) { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil, errors.New("journal store is not connected") - } - - // spawn a db transaction for read-only - txn := s.db.Txn(false) - // fetch all the records for the given persistence ID - it, err := txn.Get(journalTableName, persistenceIDIndex, persistenceID) - // handle the error - if err != nil { - // abort the transaction - txn.Abort() - return nil, fmt.Errorf("failed to replay events %d for persistenceId=%s events: %w", (toSequenceNumber-fromSequenceNumber)+1, persistenceID, err) - } - - // loop over the records and delete them - var journals []*journal - for row := it.Next(); row != nil; row = it.Next() { - if journal, ok := row.(*journal); ok { - journals = append(journals, journal) - } - } - // let us abort the transaction after fetching the matching records - txn.Abort() - - // short circuit the operation when there are no records - if len(journals) == 0 { - return nil, nil - } - - var events []*egopb.Event - for _, journal := range journals { - if journal.SequenceNumber >= fromSequenceNumber && journal.SequenceNumber <= toSequenceNumber { - // unmarshal the event and the state - evt, err := toProto(journal.EventManifest, journal.EventPayload) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal the journal event: %w", err) - } - state, err := toProto(journal.StateManifest, journal.StatePayload) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal the journal state: %w", err) - } - - if uint64(len(events)) <= max { - // create the event and add it to the list of events - events = append(events, &egopb.Event{ - PersistenceId: journal.PersistenceID, - SequenceNumber: journal.SequenceNumber, - IsDeleted: journal.IsDeleted, - Event: evt, - ResultingState: state, - Timestamp: journal.Timestamp, - Shard: journal.ShardNumber, - }) - } - } - } - - // sort the subset by sequence number - sort.SliceStable(events, func(i, j int) bool { - return events[i].GetSequenceNumber() < events[j].GetSequenceNumber() - }) - - return events, nil -} - -// GetLatestEvent fetches the latest event -func (s *EventsStore) GetLatestEvent(_ context.Context, persistenceID string) (*egopb.Event, error) { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil, errors.New("journal store is not connected") - } - - // spawn a db transaction for read-only - txn := s.db.Txn(false) - defer txn.Abort() - // let us fetch the last record - raw, err := txn.Last(journalTableName, persistenceIDIndex, persistenceID) - if err != nil { - // if the error is not found then return nil - if errors.Is(err, memdb.ErrNotFound) { - return nil, nil - } - return nil, fmt.Errorf("failed to fetch the latest event from the database for persistenceId=%s: %w", persistenceID, err) - } - - // no record found - if raw == nil { - return nil, nil - } - - // let us cast the raw data - if journal, ok := raw.(*journal); ok { - // unmarshal the event and the state - evt, err := toProto(journal.EventManifest, journal.EventPayload) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal the journal event: %w", err) - } - state, err := toProto(journal.StateManifest, journal.StatePayload) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal the journal state: %w", err) - } - - return &egopb.Event{ - PersistenceId: journal.PersistenceID, - SequenceNumber: journal.SequenceNumber, - IsDeleted: journal.IsDeleted, - Event: evt, - ResultingState: state, - Timestamp: journal.Timestamp, - Shard: journal.ShardNumber, - }, nil - } - - return nil, fmt.Errorf("failed to fetch the latest event from the database for persistenceId=%s", persistenceID) -} - -// GetShardEvents returns the next (max) events after the offset in the journal for a given shard -func (s *EventsStore) GetShardEvents(_ context.Context, shardNumber uint64, offset int64, max uint64) ([]*egopb.Event, int64, error) { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil, 0, errors.New("journal store is not connected") - } - - // spawn a db transaction for read-only - txn := s.db.Txn(false) - // fetch all the records for the given shard - it, err := txn.Get(journalTableName, persistenceIDIndex) - // handle the error - if err != nil { - // abort the transaction - txn.Abort() - return nil, 0, fmt.Errorf("failed to get events of shard=(%d): %w", shardNumber, err) - } - - // loop over the records and delete them - var journals []*journal - for row := it.Next(); row != nil; row = it.Next() { - // cast the elt into the journal - if journal, ok := row.(*journal); ok { - // filter out the journal of the given shard number - if journal.ShardNumber == shardNumber { - journals = append(journals, journal) - } - } - } - // let us abort the transaction after fetching the matching records - txn.Abort() - - // short circuit the operation when there are no records - if len(journals) == 0 { - return nil, 0, nil - } - - var events []*egopb.Event - for _, journal := range journals { - // only fetch record which timestamp is greater than the offset - if journal.Timestamp > offset { - // unmarshal the event and the state - evt, err := toProto(journal.EventManifest, journal.EventPayload) - if err != nil { - return nil, 0, fmt.Errorf("failed to unmarshal the journal event: %w", err) - } - state, err := toProto(journal.StateManifest, journal.StatePayload) - if err != nil { - return nil, 0, fmt.Errorf("failed to unmarshal the journal state: %w", err) - } - - if uint64(len(events)) <= max { - // create the event and add it to the list of events - events = append(events, &egopb.Event{ - PersistenceId: journal.PersistenceID, - SequenceNumber: journal.SequenceNumber, - IsDeleted: journal.IsDeleted, - Event: evt, - ResultingState: state, - Timestamp: journal.Timestamp, - Shard: journal.ShardNumber, - }) - } - } - } - - // short circuit the operation when there are no records - if len(events) == 0 { - return nil, 0, nil - } - - // sort the subset by timestamp - sort.SliceStable(events, func(i, j int) bool { - return events[i].GetTimestamp() <= events[j].GetTimestamp() - }) - - // grab the next offset - nextOffset := events[len(events)-1].GetTimestamp() - - return events, nextOffset, nil -} - -// ShardNumbers returns the distinct list of all the shards in the journal store -func (s *EventsStore) ShardNumbers(context.Context) ([]uint64, error) { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil, errors.New("journal store is not connected") - } - - // spawn a db transaction for read-only - txn := s.db.Txn(false) - // fetch all the records - it, err := txn.Get(journalTableName, persistenceIDIndex) - // handle the error - if err != nil { - // abort the transaction - txn.Abort() - return nil, fmt.Errorf("failed to fetch the list of shard number: %w", err) - } - - // loop over the records - var journals []*journal - for row := it.Next(); row != nil; row = it.Next() { - if journal, ok := row.(*journal); ok { - journals = append(journals, journal) - } - } - // let us abort the transaction after fetching the matching records - txn.Abort() - - // short circuit the operation when there are no records - if len(journals) == 0 { - return nil, nil - } - - // create a set to hold the unique list of shard numbers - shards := goset.NewSet[uint64]() - // iterate the list of journals and extract the shard numbers - for _, journal := range journals { - shards.Add(journal.ShardNumber) - } - - // return the list - return shards.ToSlice(), nil -} - -// toProto converts a byte array given its manifest into a valid proto message -func toProto(manifest string, bytea []byte) (*anypb.Any, error) { - mt, err := protoregistry.GlobalTypes.FindMessageByName(protoreflect.FullName(manifest)) - if err != nil { - return nil, err - } - - pm := mt.New().Interface() - err = proto.Unmarshal(bytea, pm) - if err != nil { - return nil, err - } - - if cast, ok := pm.(*anypb.Any); ok { - return cast, nil - } - return nil, fmt.Errorf("failed to unpack message=%s", manifest) -} diff --git a/plugins/eventstore/memory/memory_test.go b/plugins/eventstore/memory/memory_test.go deleted file mode 100644 index b1b553d..0000000 --- a/plugins/eventstore/memory/memory_test.go +++ /dev/null @@ -1,260 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package memory - -import ( - "context" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/anypb" - "google.golang.org/protobuf/types/known/timestamppb" - - "github.com/tochemey/ego/v3/egopb" - "github.com/tochemey/ego/v3/persistence" - testpb "github.com/tochemey/ego/v3/test/data/pb/v3" -) - -func TestEventsStore(t *testing.T) { - t.Run("testNew", func(t *testing.T) { - eventsStore := NewEventsStore() - assert.NotNil(t, eventsStore) - var p interface{} = eventsStore - _, ok := p.(persistence.EventsStore) - assert.True(t, ok) - }) - t.Run("testConnect", func(t *testing.T) { - ctx := context.TODO() - store := NewEventsStore() - assert.NotNil(t, store) - err := store.Connect(ctx) - assert.NoError(t, err) - }) - t.Run("testWriteEvents", func(t *testing.T) { - ctx := context.TODO() - state, err := anypb.New(new(testpb.Account)) - assert.NoError(t, err) - event, err := anypb.New(&testpb.AccountCredited{}) - assert.NoError(t, err) - - timestamp := timestamppb.Now() - shardNumber := uint64(9) - - journal := &egopb.Event{ - PersistenceId: "persistence-1", - SequenceNumber: 1, - IsDeleted: false, - Event: event, - ResultingState: state, - Timestamp: timestamp.AsTime().Unix(), - Shard: shardNumber, - } - - store := NewEventsStore() - assert.NotNil(t, store) - require.NoError(t, store.Connect(ctx)) - - err = store.WriteEvents(ctx, []*egopb.Event{journal}) - assert.NoError(t, err) - - // fetch the data we insert back - actual, err := store.GetLatestEvent(ctx, "persistence-1") - assert.NoError(t, err) - assert.NotNil(t, actual) - assert.True(t, proto.Equal(journal, actual)) - - // assert the number of shards - shards, err := store.ShardNumbers(ctx) - assert.NoError(t, err) - assert.Len(t, shards, 1) - - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) - t.Run("testDeleteEvents", func(t *testing.T) { - ctx := context.TODO() - state, err := anypb.New(new(testpb.Account)) - assert.NoError(t, err) - event, err := anypb.New(&testpb.AccountCredited{}) - assert.NoError(t, err) - - timestamp := timestamppb.Now() - - journal := &egopb.Event{ - PersistenceId: "persistence-1", - SequenceNumber: 1, - IsDeleted: false, - Event: event, - ResultingState: state, - Timestamp: timestamp.AsTime().Unix(), - } - - store := NewEventsStore() - assert.NotNil(t, store) - require.NoError(t, store.Connect(ctx)) - - err = store.WriteEvents(ctx, []*egopb.Event{journal}) - assert.NoError(t, err) - - // fetch the data we insert back - actual, err := store.GetLatestEvent(ctx, "persistence-1") - assert.NoError(t, err) - assert.NotNil(t, actual) - assert.True(t, proto.Equal(journal, actual)) - - // delete the journal - err = store.DeleteEvents(ctx, "persistence-1", 2) - assert.NoError(t, err) - - actual, err = store.GetLatestEvent(ctx, "persistence-1") - assert.NoError(t, err) - assert.Nil(t, actual) - - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) - t.Run("testReplayEvents", func(t *testing.T) { - ctx := context.TODO() - state, err := anypb.New(new(testpb.Account)) - assert.NoError(t, err) - event, err := anypb.New(&testpb.AccountCredited{}) - assert.NoError(t, err) - - timestamp := timestamppb.Now() - - count := 10 - journals := make([]*egopb.Event, count) - for i := 0; i < count; i++ { - seqNr := i + 1 - journals[i] = &egopb.Event{ - PersistenceId: "persistence-1", - SequenceNumber: uint64(seqNr), - IsDeleted: false, - Event: event, - ResultingState: state, - Timestamp: timestamp.AsTime().Unix(), - } - } - - store := NewEventsStore() - assert.NotNil(t, store) - require.NoError(t, store.Connect(ctx)) - - err = store.WriteEvents(ctx, journals) - assert.NoError(t, err) - - from := uint64(3) - to := uint64(6) - - actual, err := store.ReplayEvents(ctx, "persistence-1", from, to, 6) - assert.NoError(t, err) - assert.NotEmpty(t, actual) - assert.Len(t, actual, 4) - - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) - t.Run("testPersistenceIDs", func(t *testing.T) { - ctx := context.TODO() - state, err := anypb.New(new(testpb.Account)) - assert.NoError(t, err) - event, err := anypb.New(&testpb.AccountCredited{}) - assert.NoError(t, err) - - timestamp := timestamppb.Now() - - count := 5 - journals := make([]*egopb.Event, count) - for i := 0; i < count; i++ { - seqNr := i + 1 - journals[i] = &egopb.Event{ - PersistenceId: fmt.Sprintf("persistence-%d", i), - SequenceNumber: uint64(seqNr), - IsDeleted: false, - Event: event, - ResultingState: state, - Timestamp: timestamp.AsTime().Unix(), - } - } - - store := NewEventsStore() - assert.NotNil(t, store) - require.NoError(t, store.Connect(ctx)) - - err = store.WriteEvents(ctx, journals) - assert.NoError(t, err) - - expected := []string{ - "persistence-0", - "persistence-1", - "persistence-2", - } - - pageSize := uint64(3) - pageToken := "" - actual, nextPageToken, err := store.PersistenceIDs(ctx, pageSize, pageToken) - assert.NoError(t, err) - assert.NotEmpty(t, nextPageToken) - assert.Equal(t, nextPageToken, "persistence-2") - assert.ElementsMatch(t, expected, actual) - - actual2, nextPageToken2, err := store.PersistenceIDs(ctx, pageSize, nextPageToken) - expected = []string{ - "persistence-3", - "persistence-4", - } - - assert.NoError(t, err) - assert.NotEmpty(t, nextPageToken2) - assert.Equal(t, nextPageToken2, "persistence-4") - assert.ElementsMatch(t, expected, actual2) - - actual3, nextPageToken3, err := store.PersistenceIDs(ctx, pageSize, nextPageToken2) - assert.NoError(t, err) - assert.Empty(t, nextPageToken3) - assert.Empty(t, actual3) - - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) - t.Run("testGetLatestEvent: not found", func(t *testing.T) { - ctx := context.TODO() - - store := NewEventsStore() - assert.NotNil(t, store) - require.NoError(t, store.Connect(ctx)) - - // fetch the data we insert back - actual, err := store.GetLatestEvent(ctx, "persistence-1") - assert.NoError(t, err) - assert.Nil(t, actual) - - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) -} diff --git a/plugins/eventstore/memory/schemas.go b/plugins/eventstore/memory/schemas.go deleted file mode 100644 index 5b60d0b..0000000 --- a/plugins/eventstore/memory/schemas.go +++ /dev/null @@ -1,128 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package memory - -import ( - "github.com/hashicorp/go-memdb" -) - -// journal represents the journal entry -// This matches the RDBMS counter-part. -type journal struct { - // Ordering basically used as PK - Ordering string - // PersistenceID is the persistence ID - PersistenceID string - // SequenceNumber - SequenceNumber uint64 - // Specifies whether the journal is deleted - IsDeleted bool - // Specifies the event byte array - EventPayload []byte - // Specifies the event manifest - EventManifest string - // Specifies the state payload - StatePayload []byte - // Specifies the state manifest - StateManifest string - // Specifies time the record has been persisted - Timestamp int64 - // Specifies the shard number - ShardNumber uint64 -} - -const ( - journalTableName = "event_store" - journalPK = "id" - isDeletedIndex = "deletion" - persistenceIDIndex = "persistenceId" - sequenceNumberIndex = "sequenceNumber" - shardNumberIndex = "shardNumber" - timestampIndex = "timestamp" -) - -var ( - // journalSchema defines the journal schema - journalSchema = &memdb.DBSchema{ - Tables: map[string]*memdb.TableSchema{ - journalTableName: { - Name: journalTableName, - Indexes: map[string]*memdb.IndexSchema{ - journalPK: { - Name: journalPK, - AllowMissing: false, - Unique: true, - Indexer: &memdb.StringFieldIndex{ - Field: "Ordering", - Lowercase: false, - }, - }, - isDeletedIndex: { - Name: isDeletedIndex, - AllowMissing: false, - Unique: false, - Indexer: &memdb.StringFieldIndex{ - Field: "IsDeleted", - Lowercase: false, - }, - }, - persistenceIDIndex: { - Name: persistenceIDIndex, - AllowMissing: false, - Unique: false, - Indexer: &memdb.StringFieldIndex{ - Field: "PersistenceID", - Lowercase: false, - }, - }, - sequenceNumberIndex: { - Name: sequenceNumberIndex, - AllowMissing: false, - Unique: false, - Indexer: &memdb.UintFieldIndex{ - Field: "SequenceNumber", - }, - }, - shardNumberIndex: { - Name: shardNumberIndex, - AllowMissing: false, - Unique: false, - Indexer: &memdb.UintFieldIndex{ - Field: "ShardNumber", - }, - }, - timestampIndex: { - Name: timestampIndex, - AllowMissing: false, - Unique: false, - Indexer: &memdb.IntFieldIndex{ - Field: "Timestamp", - }, - }, - }, - }, - }, - } -) diff --git a/plugins/eventstore/postgres/config.go b/plugins/eventstore/postgres/config.go deleted file mode 100644 index d407d6b..0000000 --- a/plugins/eventstore/postgres/config.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -// Config defines the postgres events store configuration -type Config struct { - DBHost string // DBHost represents the database host - DBPort int // DBPort is the database port - DBName string // DBName is the database name - DBUser string // DBUser is the database user used to connect - DBPassword string // DBPassword is the database password - DBSchema string // DBSchema represents the database schema -} diff --git a/plugins/eventstore/postgres/helper_test.go b/plugins/eventstore/postgres/helper_test.go deleted file mode 100644 index 8fe971a..0000000 --- a/plugins/eventstore/postgres/helper_test.go +++ /dev/null @@ -1,63 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "os" - "testing" - - "github.com/tochemey/ego/v3/internal/postgres" -) - -var testContainer *postgres.TestContainer - -const ( - testUser = "test" - testDatabase = "testdb" - testDatabasePassword = "test" -) - -// TestMain will spawn a postgres database container that will be used for all tests -// making use of the postgres database container -func TestMain(m *testing.M) { - // set the test container - testContainer = postgres.NewTestContainer(testDatabase, testUser, testDatabasePassword) - // execute the tests - code := m.Run() - // free resources - testContainer.Cleanup() - // exit the tests - os.Exit(code) -} - -// dbHandle returns a test db -func dbHandle(ctx context.Context) (*postgres.TestDB, error) { - db := testContainer.GetTestDB() - if err := db.Connect(ctx); err != nil { - return nil, err - } - return db, nil -} diff --git a/plugins/eventstore/postgres/postgres.go b/plugins/eventstore/postgres/postgres.go deleted file mode 100644 index 537a5bb..0000000 --- a/plugins/eventstore/postgres/postgres.go +++ /dev/null @@ -1,437 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "errors" - "fmt" - - sq "github.com/Masterminds/squirrel" - "github.com/jackc/pgx/v5" - "go.uber.org/atomic" - "google.golang.org/protobuf/proto" - - "github.com/tochemey/ego/v3/egopb" - "github.com/tochemey/ego/v3/internal/postgres" - "github.com/tochemey/ego/v3/persistence" -) - -var ( - columns = []string{ - "persistence_id", - "sequence_number", - "is_deleted", - "event_payload", - "event_manifest", - "state_payload", - "state_manifest", - "timestamp", - "shard_number", - } - - tableName = "events_store" -) - -// EventsStore implements the EventsStore interface -// and helps persist events in a Postgres database -type EventsStore struct { - db postgres.Postgres - sb sq.StatementBuilderType - // insertBatchSize represents the chunk of data to bulk insert. - // This helps avoid the postgres 65535 parameter limit. - // This is necessary because Postgres uses a 32-bit int for binding input parameters and - // is not able to track anything larger. - // Note: Change this value when you know the size of data to bulk insert at once. Otherwise, you - // might encounter the postgres 65535 parameter limit error. - insertBatchSize int - // hold the connection state to avoid multiple connection of the same instance - connected *atomic.Bool -} - -// enforce interface implementation -var _ persistence.EventsStore = (*EventsStore)(nil) - -// NewEventsStore creates a new instance of PostgresEventStore -func NewEventsStore(config *Config) *EventsStore { - // create the underlying db connection - db := postgres.New(postgres.NewConfig(config.DBHost, config.DBPort, config.DBUser, config.DBPassword, config.DBName)) - return &EventsStore{ - db: db, - sb: sq.StatementBuilder.PlaceholderFormat(sq.Dollar), - insertBatchSize: 500, - connected: atomic.NewBool(false), - } -} - -// Connect connects to the underlying postgres database -func (s *EventsStore) Connect(ctx context.Context) error { - // check whether this instance of the journal is connected or not - if s.connected.Load() { - return nil - } - - // connect to the underlying db - if err := s.db.Connect(ctx); err != nil { - return err - } - - // set the connection status - s.connected.Store(true) - - return nil -} - -// Disconnect disconnects from the underlying postgres database -func (s *EventsStore) Disconnect(ctx context.Context) error { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil - } - - // disconnect the underlying database - if err := s.db.Disconnect(ctx); err != nil { - return err - } - // set the connection status - s.connected.Store(false) - - return nil -} - -// Ping verifies a connection to the database is still alive, establishing a connection if necessary. -func (s *EventsStore) Ping(ctx context.Context) error { - // check whether we are connected or not - if !s.connected.Load() { - return s.Connect(ctx) - } - - return nil -} - -// PersistenceIDs returns the distinct list of all the persistence ids in the journal store -func (s *EventsStore) PersistenceIDs(ctx context.Context, pageSize uint64, pageToken string) (persistenceIDs []string, nextPageToken string, err error) { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil, "", errors.New("journal store is not connected") - } - - // create the database delete statement - statement := s.sb. - Select("persistence_id"). - Distinct(). - From(tableName). - Limit(pageSize). - OrderBy("persistence_id ASC") - - // set the page token - if pageToken != "" { - statement = statement.Where(sq.Gt{"persistence_id": pageToken}) - } - - // get the sql statement and the arguments - query, args, err := statement.ToSql() - // handle the error - if err != nil { - return nil, "", fmt.Errorf("failed to build the sql statement: %w", err) - } - - // create the ds to hold the database record - type row struct { - PersistenceID string - } - - // execute the query against the database - var rows []*row - err = s.db.SelectAll(ctx, &rows, query, args...) - if err != nil { - return nil, "", fmt.Errorf("failed to fetch the events from the database: %w", err) - } - - // grab the fetched records - persistenceIDs = make([]string, len(rows)) - for index, row := range rows { - persistenceIDs[index] = row.PersistenceID - } - - // set the next page token - nextPageToken = persistenceIDs[len(persistenceIDs)-1] - - return -} - -// WriteEvents writes a bunch of events into the underlying postgres database -func (s *EventsStore) WriteEvents(ctx context.Context, events []*egopb.Event) error { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return errors.New("journal store is not connected") - } - - // check whether the journals list is empty - if len(events) == 0 { - // do nothing - return nil - } - - // let us begin a database transaction to make sure we atomically write those events into the database - tx, err := s.db.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) - // return the error in case we are unable to get a database transaction - if err != nil { - return fmt.Errorf("failed to obtain a database transaction: %w", err) - } - - // start creating the sql statement for insertion - statement := s.sb.Insert(tableName).Columns(columns...) - for index, event := range events { - var ( - eventManifest string - eventBytes []byte - stateManifest string - stateBytes []byte - ) - - // serialize the event and resulting state - eventBytes, _ = proto.Marshal(event.GetEvent()) - stateBytes, _ = proto.Marshal(event.GetResultingState()) - - // grab the manifest - eventManifest = string(event.GetEvent().ProtoReflect().Descriptor().FullName()) - stateManifest = string(event.GetResultingState().ProtoReflect().Descriptor().FullName()) - - // build the insertion values - statement = statement.Values( - event.GetPersistenceId(), - event.GetSequenceNumber(), - event.GetIsDeleted(), - eventBytes, - eventManifest, - stateBytes, - stateManifest, - event.GetTimestamp(), - event.GetShard(), - ) - - if (index+1)%s.insertBatchSize == 0 || index == len(events)-1 { - // get the SQL statement to run - query, args, err := statement.ToSql() - // handle the error while generating the SQL - if err != nil { - return fmt.Errorf("unable to build sql insert statement: %w", err) - } - // insert into the table - _, execErr := tx.Exec(ctx, query, args...) - if execErr != nil { - // attempt to roll back the transaction and log the error in case there is an error - if err = tx.Rollback(ctx); err != nil { - return fmt.Errorf("unable to rollback db transaction: %w", err) - } - // return the main error - return fmt.Errorf("failed to record events: %w", execErr) - } - - // reset the statement for the next bulk - statement = s.sb.Insert(tableName).Columns(columns...) - } - } - - // commit the transaction - if commitErr := tx.Commit(ctx); commitErr != nil { - // return the commit error in case there is one - return fmt.Errorf("failed to record events: %w", commitErr) - } - // every looks good - return nil -} - -// DeleteEvents deletes events from the postgres up to a given sequence number (inclusive) -func (s *EventsStore) DeleteEvents(ctx context.Context, persistenceID string, toSequenceNumber uint64) error { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return errors.New("journal store is not connected") - } - - // create the database delete statement - statement := s.sb. - Delete(tableName). - Where(sq.Eq{"persistence_id": persistenceID}). - Where(sq.LtOrEq{"sequence_number": toSequenceNumber}) - - // get the sql statement and the arguments - query, args, err := statement.ToSql() - if err != nil { - return fmt.Errorf("failed to build the delete events sql statement: %w", err) - } - - // execute the sql statement - if _, err := s.db.Exec(ctx, query, args...); err != nil { - return fmt.Errorf("failed to delete events from the database: %w", err) - } - - return nil -} - -// ReplayEvents fetches events for a given persistence ID from a given sequence number(inclusive) to a given sequence number(inclusive) -func (s *EventsStore) ReplayEvents(ctx context.Context, persistenceID string, fromSequenceNumber, toSequenceNumber uint64, max uint64) ([]*egopb.Event, error) { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil, errors.New("journal store is not connected") - } - - // create the database select statement - statement := s.sb. - Select(columns...). - From(tableName). - Where(sq.Eq{"persistence_id": persistenceID}). - Where(sq.GtOrEq{"sequence_number": fromSequenceNumber}). - Where(sq.LtOrEq{"sequence_number": toSequenceNumber}). - OrderBy("sequence_number ASC"). - Limit(max) - - // get the sql statement and the arguments - query, args, err := statement.ToSql() - if err != nil { - return nil, fmt.Errorf("failed to build the select sql statement: %w", err) - } - - // execute the query against the database - var rows rows - err = s.db.SelectAll(ctx, &rows, query, args...) - if err != nil { - return nil, fmt.Errorf("failed to fetch the events from the database: %w", err) - } - - // return the derivative events - return rows.ToEvents() -} - -// GetLatestEvent fetches the latest event -func (s *EventsStore) GetLatestEvent(ctx context.Context, persistenceID string) (*egopb.Event, error) { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil, errors.New("journal store is not connected") - } - - // create the database select statement - statement := s.sb. - Select(columns...). - From(tableName). - Where(sq.Eq{"persistence_id": persistenceID}). - OrderBy("sequence_number DESC"). - Limit(1) - - // get the sql statement and the arguments - query, args, err := statement.ToSql() - if err != nil { - return nil, fmt.Errorf("failed to build the select sql statement: %w", err) - } - - // execute the query against the database - row := new(row) - err = s.db.Select(ctx, row, query, args...) - if err != nil { - return nil, fmt.Errorf("failed to fetch the latest event from the database: %w", err) - } - - // check whether we do have data - if row.PersistenceID == "" { - return nil, nil - } - - // return the derivative event - return row.ToEvent() -} - -// GetShardEvents returns the next (max) events after the offset in the journal for a given shard -func (s *EventsStore) GetShardEvents(ctx context.Context, shardNumber uint64, offset int64, max uint64) ([]*egopb.Event, int64, error) { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil, 0, errors.New("journal store is not connected") - } - - // create the database select statement - statement := s.sb. - Select(columns...). - From(tableName). - Where(sq.Eq{"shard_number": shardNumber}). - Where(sq.Gt{"timestamp": offset}). - OrderBy("timestamp ASC"). - Limit(max) - - // get the sql statement and the arguments - query, args, err := statement.ToSql() - if err != nil { - return nil, 0, fmt.Errorf("failed to build the select sql statement: %w", err) - } - - // execute the query against the database - var rows rows - err = s.db.SelectAll(ctx, &rows, query, args...) - if err != nil { - return nil, 0, fmt.Errorf("failed to fetch the events from the database: %w", err) - } - - // short-circuit the request - if len(rows) == 0 { - return nil, 0, nil - } - - // grab the events - events, err := rows.ToEvents() - // handle the error when parsing - if err != nil { - return nil, 0, err - } - // get the next offset - nextOffset := events[len(events)-1].GetTimestamp() - // return the data - return events, nextOffset, nil -} - -// ShardNumbers returns the distinct list of all the shards in the journal store -func (s *EventsStore) ShardNumbers(ctx context.Context) ([]uint64, error) { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil, errors.New("journal store is not connected") - } - - // create the statement - statement := s.sb. - Select("DISTINCT shard_number"). - From(tableName) - - // get the sql statement and the arguments - query, args, err := statement.ToSql() - if err != nil { - return nil, fmt.Errorf("failed to build the select sql statement: %w", err) - } - - var shardNumbers []uint64 - err = s.db.SelectAll(ctx, &shardNumbers, query, args...) - if err != nil { - return nil, fmt.Errorf("failed to fetch the events from the database: %w", err) - } - - return shardNumbers, nil -} diff --git a/plugins/eventstore/postgres/postgres_test.go b/plugins/eventstore/postgres/postgres_test.go deleted file mode 100644 index 0741b57..0000000 --- a/plugins/eventstore/postgres/postgres_test.go +++ /dev/null @@ -1,373 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/encoding/prototext" - "google.golang.org/protobuf/types/known/anypb" - "google.golang.org/protobuf/types/known/timestamppb" - - "github.com/tochemey/ego/v3/egopb" - "github.com/tochemey/ego/v3/persistence" - testpb "github.com/tochemey/ego/v3/test/data/pb/v3" -) - -func TestPostgresEventsStore(t *testing.T) { - t.Run("testNewEventsStore", func(t *testing.T) { - config := &Config{ - testContainer.Host(), - testContainer.Port(), - testDatabase, - testUser, - testDatabasePassword, - testContainer.Schema(), - } - - estore := NewEventsStore(config) - assert.NotNil(t, estore) - var p interface{} = estore - _, ok := p.(persistence.EventsStore) - assert.True(t, ok) - }) - t.Run("testConnect:happy path", func(t *testing.T) { - ctx := context.TODO() - config := &Config{ - DBHost: testContainer.Host(), - DBPort: testContainer.Port(), - DBName: testDatabase, - DBUser: testUser, - DBPassword: testDatabasePassword, - DBSchema: testContainer.Schema(), - } - - store := NewEventsStore(config) - assert.NotNil(t, store) - err := store.Connect(ctx) - assert.NoError(t, err) - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) - t.Run("testConnect:database does not exist", func(t *testing.T) { - ctx := context.TODO() - config := &Config{ - DBHost: testContainer.Host(), - DBPort: testContainer.Port(), - DBName: "testDatabase", - DBUser: testUser, - DBPassword: testDatabasePassword, - DBSchema: testContainer.Schema(), - } - - store := NewEventsStore(config) - assert.NotNil(t, store) - err := store.Connect(ctx) - assert.Error(t, err) - }) - t.Run("testWriteAndReplayEvents", func(t *testing.T) { - ctx := context.TODO() - config := &Config{ - DBHost: testContainer.Host(), - DBPort: testContainer.Port(), - DBName: testDatabase, - DBUser: testUser, - DBPassword: testDatabasePassword, - DBSchema: testContainer.Schema(), - } - - store := NewEventsStore(config) - assert.NotNil(t, store) - err := store.Connect(ctx) - require.NoError(t, err) - - db, err := dbHandle(ctx) - require.NoError(t, err) - - schemaUtil := NewSchemaUtils(db) - - err = schemaUtil.CreateTable(ctx) - require.NoError(t, err) - - state, err := anypb.New(&testpb.Account{}) - assert.NoError(t, err) - event, err := anypb.New(&testpb.AccountCreated{}) - assert.NoError(t, err) - - ts1 := timestamppb.Now() - ts2 := timestamppb.Now() - shard1 := uint64(5) - shard2 := uint64(4) - - e1 := &egopb.Event{ - PersistenceId: "persistence-1", - SequenceNumber: 1, - IsDeleted: false, - Event: event, - ResultingState: state, - Timestamp: ts1.AsTime().Unix(), - Shard: shard1, - } - - event, err = anypb.New(&testpb.AccountCredited{}) - assert.NoError(t, err) - - e2 := &egopb.Event{ - PersistenceId: "persistence-1", - SequenceNumber: 2, - IsDeleted: false, - Event: event, - ResultingState: state, - Timestamp: ts2.AsTime().Unix(), - Shard: shard2, - } - - events := []*egopb.Event{e1, e2} - err = store.WriteEvents(ctx, events) - assert.NoError(t, err) - - persistenceID := "persistence-1" - max := uint64(4) - from := uint64(1) - to := uint64(2) - replayed, err := store.ReplayEvents(ctx, persistenceID, from, to, max) - assert.NoError(t, err) - assert.NotEmpty(t, replayed) - assert.Len(t, replayed, 2) - assert.Equal(t, prototext.Format(events[0]), prototext.Format(replayed[0])) - assert.Equal(t, prototext.Format(events[1]), prototext.Format(replayed[1])) - - shardNumbers, err := store.ShardNumbers(ctx) - require.NoError(t, err) - require.Len(t, shardNumbers, 2) - - offset := int64(0) - events, nextOffset, err := store.GetShardEvents(ctx, shard1, offset, max) - assert.NoError(t, err) - assert.EqualValues(t, e1.GetTimestamp(), nextOffset) - assert.Len(t, events, 1) - - err = schemaUtil.DropTable(ctx) - assert.NoError(t, err) - - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) - t.Run("testGetLatestEvent", func(t *testing.T) { - ctx := context.TODO() - config := &Config{ - testContainer.Host(), - testContainer.Port(), - testDatabase, - testUser, - testDatabasePassword, - testContainer.Schema(), - } - - store := NewEventsStore(config) - assert.NotNil(t, store) - err := store.Connect(ctx) - require.NoError(t, err) - - db, err := dbHandle(ctx) - require.NoError(t, err) - - schemaUtil := NewSchemaUtils(db) - - err = schemaUtil.CreateTable(ctx) - require.NoError(t, err) - - state, err := anypb.New(&testpb.Account{}) - assert.NoError(t, err) - event, err := anypb.New(&testpb.AccountCreated{}) - assert.NoError(t, err) - - ts1 := timestamppb.New(time.Now().UTC()) - ts2 := timestamppb.New(time.Now().UTC()) - shard1 := uint64(7) - shard2 := uint64(4) - - e1 := &egopb.Event{ - PersistenceId: "persistence-1", - SequenceNumber: 1, - IsDeleted: false, - Event: event, - ResultingState: state, - Timestamp: ts1.AsTime().Unix(), - Shard: shard1, - } - - event, err = anypb.New(&testpb.AccountCredited{}) - assert.NoError(t, err) - - e2 := &egopb.Event{ - PersistenceId: "persistence-1", - SequenceNumber: 2, - IsDeleted: false, - Event: event, - ResultingState: state, - Timestamp: ts2.AsTime().Unix(), - Shard: shard2, - } - - events := []*egopb.Event{e1, e2} - err = store.WriteEvents(ctx, events) - assert.NoError(t, err) - - persistenceID := "persistence-1" - - actual, err := store.GetLatestEvent(ctx, persistenceID) - assert.NoError(t, err) - assert.NotNil(t, actual) - - assert.Equal(t, prototext.Format(e2), prototext.Format(actual)) - - err = schemaUtil.DropTable(ctx) - assert.NoError(t, err) - - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) - t.Run("testDeleteEvents", func(t *testing.T) { - ctx := context.TODO() - config := &Config{ - testContainer.Host(), - testContainer.Port(), - testDatabase, - testUser, - testDatabasePassword, - testContainer.Schema(), - } - - store := NewEventsStore(config) - assert.NotNil(t, store) - err := store.Connect(ctx) - require.NoError(t, err) - - db, err := dbHandle(ctx) - require.NoError(t, err) - - schemaUtil := NewSchemaUtils(db) - - err = schemaUtil.CreateTable(ctx) - require.NoError(t, err) - - state, err := anypb.New(&testpb.Account{}) - assert.NoError(t, err) - event, err := anypb.New(&testpb.AccountCreated{}) - assert.NoError(t, err) - - ts1 := timestamppb.New(time.Now().UTC()) - ts2 := timestamppb.New(time.Now().UTC()) - shard1 := uint64(9) - shard2 := uint64(8) - - e1 := &egopb.Event{ - PersistenceId: "persistence-1", - SequenceNumber: 1, - IsDeleted: false, - Event: event, - ResultingState: state, - Timestamp: ts1.AsTime().Unix(), - Shard: shard1, - } - - event, err = anypb.New(&testpb.AccountCredited{}) - assert.NoError(t, err) - - e2 := &egopb.Event{ - PersistenceId: "persistence-1", - SequenceNumber: 2, - IsDeleted: false, - Event: event, - ResultingState: state, - Timestamp: ts2.AsTime().Unix(), - Shard: shard2, - } - - events := []*egopb.Event{e1, e2} - err = store.WriteEvents(ctx, events) - assert.NoError(t, err) - - persistenceID := "persistence-1" - - actual, err := store.GetLatestEvent(ctx, persistenceID) - assert.NoError(t, err) - assert.NotNil(t, actual) - - assert.Equal(t, prototext.Format(e2), prototext.Format(actual)) - - // let us delete the events - err = store.DeleteEvents(ctx, persistenceID, uint64(3)) - assert.NoError(t, err) - actual, err = store.GetLatestEvent(ctx, persistenceID) - assert.NoError(t, err) - assert.Nil(t, actual) - - err = schemaUtil.DropTable(ctx) - assert.NoError(t, err) - - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) - t.Run("testShardNumbers", func(t *testing.T) { - ctx := context.TODO() - config := &Config{ - testContainer.Host(), - testContainer.Port(), - testDatabase, - testUser, - testDatabasePassword, - testContainer.Schema(), - } - - store := NewEventsStore(config) - assert.NotNil(t, store) - err := store.Connect(ctx) - require.NoError(t, err) - - db, err := dbHandle(ctx) - require.NoError(t, err) - - schemaUtil := NewSchemaUtils(db) - - err = schemaUtil.CreateTable(ctx) - require.NoError(t, err) - - shardNumbers, err := store.ShardNumbers(ctx) - require.NoError(t, err) - require.Empty(t, shardNumbers) - - err = schemaUtil.DropTable(ctx) - assert.NoError(t, err) - - err = store.Disconnect(ctx) - assert.NoError(t, err) - }) -} diff --git a/plugins/eventstore/postgres/row.go b/plugins/eventstore/postgres/row.go deleted file mode 100644 index 28c5062..0000000 --- a/plugins/eventstore/postgres/row.go +++ /dev/null @@ -1,124 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "fmt" - - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protoreflect" - "google.golang.org/protobuf/reflect/protoregistry" - "google.golang.org/protobuf/types/known/anypb" - - "github.com/tochemey/ego/v3/egopb" -) - -// row represents the events store row -type row struct { - PersistenceID string - SequenceNumber uint64 - IsDeleted bool - EventPayload []byte - EventManifest string - StatePayload []byte - StateManifest string - Timestamp int64 - ShardNumber uint64 -} - -// ToEvent convert row to event -func (x row) ToEvent() (*egopb.Event, error) { - // unmarshal the event and the state - evt, err := toProto(x.EventManifest, x.EventPayload) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal the journal event: %w", err) - } - state, err := toProto(x.StateManifest, x.StatePayload) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal the journal state: %w", err) - } - - return &egopb.Event{ - PersistenceId: x.PersistenceID, - SequenceNumber: x.SequenceNumber, - IsDeleted: x.IsDeleted, - Event: evt, - ResultingState: state, - Timestamp: x.Timestamp, - Shard: x.ShardNumber, - }, nil -} - -// rows defines the list of row -type rows []*row - -// ToEvents converts rows to events -func (x rows) ToEvents() ([]*egopb.Event, error) { - // create the list of events - events := make([]*egopb.Event, 0, len(x)) - // iterate the rows - for _, row := range x { - // unmarshal the event and the state - evt, err := toProto(row.EventManifest, row.EventPayload) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal the journal event: %w", err) - } - state, err := toProto(row.StateManifest, row.StatePayload) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal the journal state: %w", err) - } - // create the event and add it to the list of events - events = append(events, &egopb.Event{ - PersistenceId: row.PersistenceID, - SequenceNumber: row.SequenceNumber, - IsDeleted: row.IsDeleted, - Event: evt, - ResultingState: state, - Timestamp: row.Timestamp, - Shard: row.ShardNumber, - }) - } - - return events, nil -} - -// toProto converts a byte array given its manifest into a valid proto message -func toProto(manifest string, bytea []byte) (*anypb.Any, error) { - mt, err := protoregistry.GlobalTypes.FindMessageByName(protoreflect.FullName(manifest)) - if err != nil { - return nil, err - } - - pm := mt.New().Interface() - err = proto.Unmarshal(bytea, pm) - if err != nil { - return nil, err - } - - if cast, ok := pm.(*anypb.Any); ok { - return cast, nil - } - return nil, fmt.Errorf("failed to unpack message=%s", manifest) -} diff --git a/plugins/eventstore/postgres/schema_utils.go b/plugins/eventstore/postgres/schema_utils.go deleted file mode 100644 index 9f8fc4b..0000000 --- a/plugins/eventstore/postgres/schema_utils.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - - "github.com/tochemey/ego/v3/internal/postgres" -) - -// SchemaUtils help create the various test tables in unit/integration tests -type SchemaUtils struct { - db *postgres.TestDB -} - -// NewSchemaUtils creates an instance of SchemaUtils -func NewSchemaUtils(db *postgres.TestDB) *SchemaUtils { - return &SchemaUtils{db: db} -} - -// CreateTable creates the event store table used for unit tests -func (d SchemaUtils) CreateTable(ctx context.Context) error { - schemaDDL := ` - DROP TABLE IF EXISTS events_store; - CREATE TABLE IF NOT EXISTS events_store - ( - persistence_id VARCHAR(255) NOT NULL, - sequence_number BIGINT NOT NULL, - is_deleted BOOLEAN DEFAULT FALSE NOT NULL, - event_payload BYTEA NOT NULL, - event_manifest VARCHAR(255) NOT NULL, - state_payload BYTEA NOT NULL, - state_manifest VARCHAR(255) NOT NULL, - timestamp BIGINT NOT NULL, - shard_number BIGINT NOT NULL , - - PRIMARY KEY (persistence_id, sequence_number) - ); - - --- create an index on the is_deleted column - CREATE INDEX IF NOT EXISTS idx_event_journal_deleted ON events_store (is_deleted); - CREATE INDEX IF NOT EXISTS idx_event_journal_shard ON events_store (shard_number); - ` - _, err := d.db.Exec(ctx, schemaDDL) - return err -} - -// DropTable drop the table used in unit test -// This is useful for resource cleanup after a unit test -func (d SchemaUtils) DropTable(ctx context.Context) error { - return d.db.DropTable(ctx, tableName) -} diff --git a/plugins/eventstore/postgres/schema_utils_test.go b/plugins/eventstore/postgres/schema_utils_test.go deleted file mode 100644 index aa79c0a..0000000 --- a/plugins/eventstore/postgres/schema_utils_test.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSchemaUtils(t *testing.T) { - ctx := context.TODO() - db, err := dbHandle(ctx) - assert.NoError(t, err) - - // create the tables - schemaUtils := NewSchemaUtils(db) - err = schemaUtils.CreateTable(ctx) - assert.NoError(t, err) - - // assert existence of the table - err = db.TableExists(ctx, tableName) - assert.NoError(t, err) - - // clean up - assert.NoError(t, schemaUtils.DropTable(ctx)) - assert.NoError(t, db.Disconnect(ctx)) -} diff --git a/plugins/statestore/memory/memory.go b/plugins/statestore/memory/memory.go deleted file mode 100644 index 22c7801..0000000 --- a/plugins/statestore/memory/memory.go +++ /dev/null @@ -1,110 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package memory - -import ( - "context" - "errors" - "sync" - - "go.uber.org/atomic" - - "github.com/tochemey/ego/v3/egopb" - "github.com/tochemey/ego/v3/persistence" -) - -// StateStore keep in memory every durable state actor -// NOTE: NOT RECOMMENDED FOR PRODUCTION CODE because all records are in memory and there is no durability. -// This is recommended for tests or PoC -type StateStore struct { - db *sync.Map - connected *atomic.Bool -} - -// enforce compilation error -var _ persistence.StateStore = (*StateStore)(nil) - -// NewStateStore creates an instance StateStore -func NewStateStore() *StateStore { - return &StateStore{ - db: &sync.Map{}, - connected: atomic.NewBool(false), - } -} - -// Connect connects the durable store -// nolint -func (d *StateStore) Connect(ctx context.Context) error { - if d.connected.Load() { - return nil - } - d.connected.Store(true) - return nil -} - -// Disconnect disconnect the durable store -// nolint -func (d *StateStore) Disconnect(ctx context.Context) error { - if !d.connected.Load() { - return nil - } - d.db.Range(func(key interface{}, value interface{}) bool { - d.db.Delete(key) - return true - }) - d.connected.Store(false) - return nil -} - -// Ping verifies a connection to the database is still alive, establishing a connection if necessary. -func (d *StateStore) Ping(ctx context.Context) error { - if !d.connected.Load() { - return d.Connect(ctx) - } - return nil -} - -// WriteState persist durable state for a given persistenceID. -// nolint -func (d *StateStore) WriteState(ctx context.Context, state *egopb.DurableState) error { - if !d.connected.Load() { - return errors.New("durable store is not connected") - } - d.db.Store(state.GetPersistenceId(), state) - return nil -} - -// GetLatestState fetches the latest durable state -// nolint -func (d *StateStore) GetLatestState(ctx context.Context, persistenceID string) (*egopb.DurableState, error) { - if !d.connected.Load() { - return nil, errors.New("durable store is not connected") - } - value, ok := d.db.Load(persistenceID) - if !ok { - return nil, nil - } - return value.(*egopb.DurableState), nil -} diff --git a/plugins/statestore/postgres/config.go b/plugins/statestore/postgres/config.go deleted file mode 100644 index d407d6b..0000000 --- a/plugins/statestore/postgres/config.go +++ /dev/null @@ -1,35 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -// Config defines the postgres events store configuration -type Config struct { - DBHost string // DBHost represents the database host - DBPort int // DBPort is the database port - DBName string // DBName is the database name - DBUser string // DBUser is the database user used to connect - DBPassword string // DBPassword is the database password - DBSchema string // DBSchema represents the database schema -} diff --git a/plugins/statestore/postgres/postgres.go b/plugins/statestore/postgres/postgres.go deleted file mode 100644 index f547fa7..0000000 --- a/plugins/statestore/postgres/postgres.go +++ /dev/null @@ -1,207 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - "errors" - "fmt" - - sq "github.com/Masterminds/squirrel" - "github.com/jackc/pgx/v5" - "go.uber.org/atomic" - "google.golang.org/protobuf/proto" - - "github.com/tochemey/ego/v3/egopb" - "github.com/tochemey/ego/v3/internal/postgres" - "github.com/tochemey/ego/v3/persistence" -) - -var ( - columns = []string{ - "persistence_id", - "version_number", - "state_payload", - "state_manifest", - "timestamp", - "shard_number", - } - - tableName = "states_store" -) - -// DurableStore implements the DurableStore interface -// and helps persist events in a Postgres database -type DurableStore struct { - db postgres.Postgres - sb sq.StatementBuilderType - // hold the connection state to avoid multiple connection of the same instance - connected *atomic.Bool -} - -// enforce interface implementation -var _ persistence.StateStore = (*DurableStore)(nil) - -// NewStateStore creates a new instance of StateStore -func NewStateStore(config *Config) *DurableStore { - // create the underlying db connection - db := postgres.New(postgres.NewConfig(config.DBHost, config.DBPort, config.DBUser, config.DBPassword, config.DBName)) - return &DurableStore{ - db: db, - sb: sq.StatementBuilder.PlaceholderFormat(sq.Dollar), - connected: atomic.NewBool(false), - } -} - -// Connect connects to the underlying postgres database -func (s *DurableStore) Connect(ctx context.Context) error { - // check whether this instance of the journal is connected or not - if s.connected.Load() { - return nil - } - - // connect to the underlying db - if err := s.db.Connect(ctx); err != nil { - return err - } - - // set the connection status - s.connected.Store(true) - - return nil -} - -// Disconnect disconnects from the underlying postgres database -func (s *DurableStore) Disconnect(ctx context.Context) error { - // check whether this instance of the journal is connected or not - if !s.connected.Load() { - return nil - } - - // disconnect the underlying database - if err := s.db.Disconnect(ctx); err != nil { - return err - } - // set the connection status - s.connected.Store(false) - - return nil -} - -// Ping verifies a connection to the database is still alive, establishing a connection if necessary. -func (s *DurableStore) Ping(ctx context.Context) error { - // check whether we are connected or not - if !s.connected.Load() { - return s.Connect(ctx) - } - - return nil -} - -// WriteState writes a durable state into the underlying postgres database -func (s *DurableStore) WriteState(ctx context.Context, state *egopb.DurableState) error { - if !s.connected.Load() { - return errors.New("durable store is not connected") - } - - if state == nil || proto.Equal(state, &egopb.DurableState{}) { - return nil - } - - tx, err := s.db.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) - if err != nil { - return fmt.Errorf("failed to obtain a database transaction: %w", err) - } - - bytea, _ := proto.Marshal(state.GetResultingState()) - manifest := string(state.GetResultingState().ProtoReflect().Descriptor().FullName()) - - statement := s.sb. - Insert(tableName). - Columns(columns...). - Values( - state.GetPersistenceId(), - state.GetVersionNumber(), - bytea, - manifest, - state.GetTimestamp(), - state.GetShard(), - ).Suffix("ON CONFLICT (persistence_id) " + - "DO UPDATE SET " + - "version_number = excluded.version_number," + - "state_payload = excluded.state_payload, " + - "state_manifest = excluded.state_manifest," + - "timestamp = excluded.timestamp", - ) - - query, args, err := statement.ToSql() - if err != nil { - return fmt.Errorf("unable to build sql insert statement: %w", err) - } - - _, execErr := tx.Exec(ctx, query, args...) - if execErr != nil { - if err = tx.Rollback(ctx); err != nil { - return fmt.Errorf("unable to rollback db transaction: %w", err) - } - return fmt.Errorf("failed to record durable state: %w", execErr) - } - - // commit the transaction - if commitErr := tx.Commit(ctx); commitErr != nil { - return fmt.Errorf("failed to record durable state: %w", commitErr) - } - return nil -} - -// GetLatestState fetches the latest durable state of a persistenceID -func (s *DurableStore) GetLatestState(ctx context.Context, persistenceID string) (*egopb.DurableState, error) { - if !s.connected.Load() { - return nil, errors.New("durable store is not connected") - } - - statement := s.sb. - Select(columns...). - From(tableName). - Where(sq.Eq{"persistence_id": persistenceID}). - Limit(1) - - query, args, err := statement.ToSql() - if err != nil { - return nil, fmt.Errorf("failed to build the select sql statement: %w", err) - } - - row := new(row) - err = s.db.Select(ctx, row, query, args...) - if err != nil { - return nil, fmt.Errorf("failed to fetch the latest event from the database: %w", err) - } - - if row.PersistenceID == "" { - return nil, nil - } - - return row.ToDurableState() -} diff --git a/plugins/statestore/postgres/row.go b/plugins/statestore/postgres/row.go deleted file mode 100644 index 660d400..0000000 --- a/plugins/statestore/postgres/row.go +++ /dev/null @@ -1,82 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "fmt" - - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/reflect/protoreflect" - "google.golang.org/protobuf/reflect/protoregistry" - "google.golang.org/protobuf/types/known/anypb" - - "github.com/tochemey/ego/v3/egopb" -) - -// row represents the durable state store row -type row struct { - PersistenceID string - VersionNumber uint64 - StatePayload []byte - StateManifest string - Timestamp int64 - ShardNumber uint64 -} - -// ToDurableState convert row to durable state -func (x row) ToDurableState() (*egopb.DurableState, error) { - // unmarshal the event and the state - state, err := toProto(x.StateManifest, x.StatePayload) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal the durable state: %w", err) - } - - return &egopb.DurableState{ - PersistenceId: x.PersistenceID, - VersionNumber: x.VersionNumber, - ResultingState: state, - Timestamp: x.Timestamp, - Shard: x.ShardNumber, - }, nil -} - -// toProto converts a byte array given its manifest into a valid proto message -func toProto(manifest string, bytea []byte) (*anypb.Any, error) { - mt, err := protoregistry.GlobalTypes.FindMessageByName(protoreflect.FullName(manifest)) - if err != nil { - return nil, err - } - - pm := mt.New().Interface() - err = proto.Unmarshal(bytea, pm) - if err != nil { - return nil, err - } - - if cast, ok := pm.(*anypb.Any); ok { - return cast, nil - } - return nil, fmt.Errorf("failed to unpack message=%s", manifest) -} diff --git a/plugins/statestore/postgres/schema_utils.go b/plugins/statestore/postgres/schema_utils.go deleted file mode 100644 index 65cd438..0000000 --- a/plugins/statestore/postgres/schema_utils.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - * MIT License - * - * Copyright (c) 2022-2024 Tochemey - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package postgres - -import ( - "context" - - "github.com/tochemey/ego/v3/internal/postgres" -) - -// SchemaUtils help create the various test tables in unit/integration tests -type SchemaUtils struct { - db *postgres.TestDB -} - -// NewSchemaUtils creates an instance of SchemaUtils -func NewSchemaUtils(db *postgres.TestDB) *SchemaUtils { - return &SchemaUtils{db: db} -} - -// CreateTable creates the event store table used for unit tests -func (d SchemaUtils) CreateTable(ctx context.Context) error { - schemaDDL := ` - DROP TABLE IF EXISTS states_store; - CREATE TABLE IF NOT EXISTS states_store - ( - persistence_id VARCHAR(255) PRIMARY KEY, - version_number BIGINT NOT NULL, - state_payload BYTEA NOT NULL, - state_manifest VARCHAR(255) NOT NULL, - timestamp BIGINT NOT NULL, - shard_number BIGINT NOT NULL - ); - ` - _, err := d.db.Exec(ctx, schemaDDL) - return err -} - -// DropTable drop the table used in unit test -// This is useful for resource cleanup after a unit test -func (d SchemaUtils) DropTable(ctx context.Context) error { - return d.db.DropTable(ctx, tableName) -} diff --git a/projection/projection_test.go b/projection/projection_test.go index fa111bd..4dd4227 100644 --- a/projection/projection_test.go +++ b/projection/projection_test.go @@ -36,13 +36,13 @@ import ( "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/timestamppb" + memory "github.com/tochemey/ego-contrib/eventstore/memory" + memoffsetstore "github.com/tochemey/ego-contrib/offsetstore/memory" "github.com/tochemey/goakt/v2/actors" "github.com/tochemey/goakt/v2/log" "github.com/tochemey/ego/v3/egopb" "github.com/tochemey/ego/v3/internal/lib" - memoffsetstore "github.com/tochemey/ego/v3/offsetstore/memory" - "github.com/tochemey/ego/v3/plugins/eventstore/memory" testpb "github.com/tochemey/ego/v3/test/data/pb/v3" ) diff --git a/projection/runner_test.go b/projection/runner_test.go index f630806..3e326cb 100644 --- a/projection/runner_test.go +++ b/projection/runner_test.go @@ -39,14 +39,14 @@ import ( "google.golang.org/protobuf/types/known/anypb" "google.golang.org/protobuf/types/known/timestamppb" + memory "github.com/tochemey/ego-contrib/eventstore/memory" + memoffsetstore "github.com/tochemey/ego-contrib/offsetstore/memory" "github.com/tochemey/goakt/v2/log" "github.com/tochemey/ego/v3/egopb" "github.com/tochemey/ego/v3/internal/lib" mocksoffsetstore "github.com/tochemey/ego/v3/mocks/offsetstore" mockseventstore "github.com/tochemey/ego/v3/mocks/persistence" - memoffsetstore "github.com/tochemey/ego/v3/offsetstore/memory" - "github.com/tochemey/ego/v3/plugins/eventstore/memory" testpb "github.com/tochemey/ego/v3/test/data/pb/v3" ) diff --git a/readme.md b/readme.md index ec7877d..a903f9f 100644 --- a/readme.md +++ b/readme.md @@ -86,19 +86,13 @@ persisted by the write model. The offset used in eGo is a _timestamp-based offse #### Events Store -One can implement a custom events store. See [EventsStore](persistence/events_store.go). eGo comes packaged with two -events store: - -- [Postgres](plugins/eventstore/postgres/postgres.go): Schema can be found [here](./resources/eventstore_postgres.sql) -- [Memory](plugins/eventstore/memory/memory.go) (for testing purpose only) +One can implement a custom events store. See [EventsStore](persistence/events_store.go). +There are some pre-built events stores one can use out of the box. See [Contrib](https://github.com/Tochemey/ego-contrib/tree/main/eventstore) #### Offsets Store -One can implement a custom offsets store. See [OffsetStore](./offsetstore/iface.go). eGo comes packaged with two offset -store: - -- [Postgres](./offsetstore/postgres/postgres.go): Schema can be found [here](./resources/offsetstore_postgres.sql) -- [Memory](./offsetstore/memory/memory.go) (for testing purpose only) +One can implement a custom offsets store. See [OffsetStore](./offsetstore/iface.go). +There are some pre-built offsets stores one can use out of the box. See [Contrib](https://github.com/Tochemey/ego-contrib/tree/main/offsetstore) ### Durable State Behavior @@ -119,13 +113,10 @@ is imperative that the newer version of the state is greater than the current ve During a normal shutdown process, it will persist its current state to the durable store prior to shutting down. This behavior help maintain some consistency across the actor state evolution. -#### State Store - -One can implement a custom state store. See [StateStore](persistence/state_store.go). eGo comes packaged with two state -stores: +#### Durable Store -- [Postgres](plugins/statestore/postgres/postgres.go): Schema can be found [here](./resources/durablestore_postgres.sql) -- [Memory](plugins/statestore/memory/memory.go) (for testing purpose only) +One can implement a custom state store. See [Durable Store](persistence/state_store.go). +There are some pre-built durable stores one can use out of the box. See [Contrib](https://github.com/Tochemey/ego-contrib/tree/main/durablestore) #### Howto