diff --git a/Earthfile b/Earthfile index cf82bcd594..e1e7c60fca 100644 --- a/Earthfile +++ b/Earthfile @@ -149,6 +149,9 @@ tests-integration: pre-commit: # Generate the final spec and run all the pre-commit hooks LOCALLY BUILD --pass-args ./releases+sdk-generate + FOR component IN $(cd ./libs && ls -d */) + BUILD --pass-args ./libs/${component}+pre-commit + END FOR component IN $(cd ./tools && ls -d */) BUILD --pass-args ./tools/${component}+pre-commit END diff --git a/components/ledger/libs/.golangci.yml b/components/ledger/libs/.golangci.yml new file mode 100644 index 0000000000..5cafc8d3d0 --- /dev/null +++ b/components/ledger/libs/.golangci.yml @@ -0,0 +1,43 @@ +allow-parallel-runners: true +run: + timeout: 5m +linters: + disable-all: true + enable: + - gofmt + - goimports + - unused + - gosec +linters-settings: + gosec: + # To select a subset of rules to run. + # Available rules: https://github.com/securego/gosec#available-rules + includes: + - G103 # Audit the use of unsafe block + - G104 # Audit errors not checked + - G106 # Audit the use of ssh.InsecureIgnoreHostKey + - G108 # Profiling endpoint automatically exposed on /debug/pprof + - G109 # Potential Integer overflow made by strconv.Atoi result conversion to int16/32 + - G110 # Potential DoS vulnerability via decompression bomb + - G111 # Potential directory traversal + - G112 # Potential slowloris attack +# - G113 # Usage of Rat.SetString in math/big with an overflow (CVE-2022-23772) + - G201 # SQL query construction using format string + - G202 # SQL query construction using string concatenation + - G203 # Use of unescaped data in HTML templates + - G204 # Audit use of command execution + - G301 # Poor file permissions used when creating a directory + - G302 # Poor file permissions used with chmod + - G303 # Creating tempfile using a predictable path + - G304 # File path provided as taint input + - G305 # File traversal when extracting zip/tar archive + - G306 # Poor file permissions used when writing to a new file + - G307 # Poor file permissions used when creating a file with os.Create + - G401 # Detect the usage of DES, RC4, MD5 or SHA1 + - G403 # Ensure minimum RSA key length of 2048 bits + - G501 # Import blocklist: crypto/md5 + - G502 # Import blocklist: crypto/des + - G503 # Import blocklist: crypto/rc4 + - G504 # Import blocklist: net/http/cgi + - G505 # Import blocklist: crypto/sha1 + - G602 # Slice access out of bounds \ No newline at end of file diff --git a/components/ledger/libs/Earthfile b/components/ledger/libs/Earthfile index e1df0cd8ad..2b6579d6f2 100644 --- a/components/ledger/libs/Earthfile +++ b/components/ledger/libs/Earthfile @@ -2,6 +2,7 @@ VERSION --arg-scope-and-set --pass-args --use-function-keyword 0.7 ARG core=github.com/formancehq/earthly IMPORT $core AS core +IMPORT ../.. AS stack FROM core+base-image @@ -15,4 +16,18 @@ tidy: COPY (+sources/*) /src WORKDIR /src DO --pass-args stack+GO_TIDY - SAVE ARTIFACT go.* AS LOCAL ./ \ No newline at end of file + SAVE ARTIFACT go.* AS LOCAL ./ + +lint: + FROM core+builder-image + COPY (+sources/*) /src + WORKDIR /src + COPY --pass-args +tidy/go.* . + DO --pass-args stack+GO_LINT + SAVE ARTIFACT * AS LOCAL ./ + +pre-commit: + WAIT + BUILD --pass-args +tidy + END + BUILD --pass-args +lint \ No newline at end of file diff --git a/components/ledger/libs/aws/iam/load.go b/components/ledger/libs/aws/iam/load.go index 332207e7d0..86aa4cab89 100644 --- a/components/ledger/libs/aws/iam/load.go +++ b/components/ledger/libs/aws/iam/load.go @@ -2,6 +2,7 @@ package iam import ( "context" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/spf13/pflag" diff --git a/components/ledger/libs/bun/bunconnect/flags.go b/components/ledger/libs/bun/bunconnect/flags.go index 5fc94cf3e4..973bce9d00 100644 --- a/components/ledger/libs/bun/bunconnect/flags.go +++ b/components/ledger/libs/bun/bunconnect/flags.go @@ -3,6 +3,8 @@ package bunconnect import ( "context" "database/sql/driver" + "time" + "github.com/aws/aws-sdk-go-v2/config" "github.com/formancehq/stack/libs/go-libs/aws/iam" "github.com/formancehq/stack/libs/go-libs/logging" @@ -10,7 +12,6 @@ import ( "github.com/lib/pq" "github.com/spf13/pflag" "github.com/spf13/viper" - "time" ) const ( diff --git a/components/ledger/libs/bun/bunconnect/iam.go b/components/ledger/libs/bun/bunconnect/iam.go index f7f90acd11..48f4667fe2 100644 --- a/components/ledger/libs/bun/bunconnect/iam.go +++ b/components/ledger/libs/bun/bunconnect/iam.go @@ -4,6 +4,7 @@ import ( "context" "database/sql/driver" "fmt" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/feature/rds/auth" "github.com/formancehq/stack/libs/go-libs/logging" diff --git a/components/ledger/libs/bun/bunconnect/module.go b/components/ledger/libs/bun/bunconnect/module.go index 664e1df353..8a403e877a 100644 --- a/components/ledger/libs/bun/bunconnect/module.go +++ b/components/ledger/libs/bun/bunconnect/module.go @@ -2,6 +2,7 @@ package bunconnect import ( "context" + "github.com/formancehq/stack/libs/go-libs/logging" "github.com/uptrace/bun" "go.uber.org/fx" diff --git a/components/ledger/libs/bun/bunmigrate/command.go b/components/ledger/libs/bun/bunmigrate/command.go index 82e6ccd5b3..0a083f42e0 100644 --- a/components/ledger/libs/bun/bunmigrate/command.go +++ b/components/ledger/libs/bun/bunmigrate/command.go @@ -4,6 +4,7 @@ import ( "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" "github.com/spf13/cobra" "github.com/uptrace/bun" + // Import the postgres driver. _ "github.com/lib/pq" ) diff --git a/components/ledger/libs/bun/bunmigrate/run.go b/components/ledger/libs/bun/bunmigrate/run.go index b0fa2a61c8..ef937ee178 100644 --- a/components/ledger/libs/bun/bunmigrate/run.go +++ b/components/ledger/libs/bun/bunmigrate/run.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "fmt" + "io" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" sharedlogging "github.com/formancehq/stack/libs/go-libs/logging" "github.com/formancehq/stack/libs/go-libs/pointer" @@ -14,7 +16,6 @@ import ( "github.com/uptrace/bun" "github.com/uptrace/bun/extra/bundebug" "github.com/xo/dburl" - "io" ) func isDatabaseExists(ctx context.Context, db *bun.DB, name string) (bool, error) { diff --git a/components/ledger/libs/bun/bunmigrate/run_test.go b/components/ledger/libs/bun/bunmigrate/run_test.go index afc794ddde..8de8732abf 100644 --- a/components/ledger/libs/bun/bunmigrate/run_test.go +++ b/components/ledger/libs/bun/bunmigrate/run_test.go @@ -1,13 +1,14 @@ package bunmigrate import ( + "os" + "testing" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/stretchr/testify/require" "github.com/uptrace/bun" - "os" - "testing" ) func TestRunMigrate(t *testing.T) { @@ -19,7 +20,6 @@ func TestRunMigrate(t *testing.T) { connectionOptions := &bunconnect.ConnectionOptions{ DatabaseSourceName: pgtesting.Server().GetDatabaseDSN("testing"), Debug: testing.Verbose(), - Writer: os.Stdout, } executor := func(args []string, db *bun.DB) error { return nil diff --git a/components/ledger/libs/bun/bunpaginate/pagination_column_test.go b/components/ledger/libs/bun/bunpaginate/pagination_column_test.go index e0801a781c..f76766b9f8 100644 --- a/components/ledger/libs/bun/bunpaginate/pagination_column_test.go +++ b/components/ledger/libs/bun/bunpaginate/pagination_column_test.go @@ -2,11 +2,12 @@ package bunpaginate_test import ( "context" + "math/big" + "testing" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" bunpaginate2 "github.com/formancehq/stack/libs/go-libs/bun/bunpaginate" "github.com/formancehq/stack/libs/go-libs/logging" - "math/big" - "testing" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/stretchr/testify/require" diff --git a/components/ledger/libs/bun/bunpaginate/pagination_offset_test.go b/components/ledger/libs/bun/bunpaginate/pagination_offset_test.go index c5f8f394c5..bfd554ad05 100644 --- a/components/ledger/libs/bun/bunpaginate/pagination_offset_test.go +++ b/components/ledger/libs/bun/bunpaginate/pagination_offset_test.go @@ -2,10 +2,11 @@ package bunpaginate_test import ( "context" + "testing" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" bunpaginate2 "github.com/formancehq/stack/libs/go-libs/bun/bunpaginate" "github.com/formancehq/stack/libs/go-libs/logging" - "testing" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/stretchr/testify/require" diff --git a/components/ledger/libs/httpserver/serverport.go b/components/ledger/libs/httpserver/serverport.go index 8a7aef4bbe..4d01bbf60c 100644 --- a/components/ledger/libs/httpserver/serverport.go +++ b/components/ledger/libs/httpserver/serverport.go @@ -6,6 +6,7 @@ import ( "net" "net/http" "strconv" + "time" "github.com/formancehq/stack/libs/go-libs/logging" @@ -80,7 +81,8 @@ func (s *server) StartServer(ctx context.Context, handler http.Handler, options StartedServer(ctx, s.listener) srv := &http.Server{ - Handler: handler, + Handler: handler, + ReadHeaderTimeout: 10 * time.Second, } for _, option := range options { option(srv) diff --git a/components/ledger/libs/migrations/migrator.go b/components/ledger/libs/migrations/migrator.go index 054cc839f4..9e080a984c 100644 --- a/components/ledger/libs/migrations/migrator.go +++ b/components/ledger/libs/migrations/migrator.go @@ -4,9 +4,10 @@ import ( "context" "database/sql" "fmt" - "github.com/lib/pq" "time" + "github.com/lib/pq" + "github.com/pkg/errors" "github.com/uptrace/bun" ) diff --git a/components/ledger/libs/pgtesting/postgres.go b/components/ledger/libs/pgtesting/postgres.go index 4804b0d592..268e8ca016 100644 --- a/components/ledger/libs/pgtesting/postgres.go +++ b/components/ledger/libs/pgtesting/postgres.go @@ -4,13 +4,14 @@ import ( "context" "database/sql" "fmt" - "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" "os" "strconv" "sync" "testing" "time" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" + "github.com/google/uuid" _ "github.com/lib/pq" "github.com/ory/dockertest/v3" diff --git a/components/ledger/libs/publish/messages.go b/components/ledger/libs/publish/messages.go index 4f08e3817c..7343b2e59d 100644 --- a/components/ledger/libs/publish/messages.go +++ b/components/ledger/libs/publish/messages.go @@ -3,10 +3,11 @@ package publish import ( "context" "encoding/json" + "time" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" - "time" "github.com/ThreeDotsLabs/watermill/message" "github.com/google/uuid" diff --git a/components/ledger/libs/publish/module.go b/components/ledger/libs/publish/module.go index b46145b2d0..2811c4ae44 100644 --- a/components/ledger/libs/publish/module.go +++ b/components/ledger/libs/publish/module.go @@ -2,6 +2,7 @@ package publish import ( "context" + "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill/message" "github.com/ThreeDotsLabs/watermill/pubsub/gochannel" diff --git a/components/ledger/libs/publish/module_test.go b/components/ledger/libs/publish/module_test.go index 536cd2ebdc..6bdf0bbbfe 100644 --- a/components/ledger/libs/publish/module_test.go +++ b/components/ledger/libs/publish/module_test.go @@ -3,15 +3,16 @@ package publish import ( "context" "fmt" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/propagation" - tracesdk "go.opentelemetry.io/otel/sdk/trace" - "go.opentelemetry.io/otel/trace" "io" "os" "testing" "time" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" + "github.com/IBM/sarama" "github.com/ThreeDotsLabs/watermill/message" "github.com/formancehq/stack/libs/go-libs/logging" diff --git a/components/ledger/libs/service/app.go b/components/ledger/libs/service/app.go index c52751c0a6..5fd5201749 100644 --- a/components/ledger/libs/service/app.go +++ b/components/ledger/libs/service/app.go @@ -2,12 +2,13 @@ package service import ( "context" + "io" + "os" + "github.com/formancehq/stack/libs/go-libs/logging" "github.com/spf13/cobra" "github.com/spf13/viper" "go.uber.org/fx" - "io" - "os" ) const DebugFlag = "debug" diff --git a/libs/events/.golangci.yml b/libs/events/.golangci.yml new file mode 100644 index 0000000000..5cafc8d3d0 --- /dev/null +++ b/libs/events/.golangci.yml @@ -0,0 +1,43 @@ +allow-parallel-runners: true +run: + timeout: 5m +linters: + disable-all: true + enable: + - gofmt + - goimports + - unused + - gosec +linters-settings: + gosec: + # To select a subset of rules to run. + # Available rules: https://github.com/securego/gosec#available-rules + includes: + - G103 # Audit the use of unsafe block + - G104 # Audit errors not checked + - G106 # Audit the use of ssh.InsecureIgnoreHostKey + - G108 # Profiling endpoint automatically exposed on /debug/pprof + - G109 # Potential Integer overflow made by strconv.Atoi result conversion to int16/32 + - G110 # Potential DoS vulnerability via decompression bomb + - G111 # Potential directory traversal + - G112 # Potential slowloris attack +# - G113 # Usage of Rat.SetString in math/big with an overflow (CVE-2022-23772) + - G201 # SQL query construction using format string + - G202 # SQL query construction using string concatenation + - G203 # Use of unescaped data in HTML templates + - G204 # Audit use of command execution + - G301 # Poor file permissions used when creating a directory + - G302 # Poor file permissions used with chmod + - G303 # Creating tempfile using a predictable path + - G304 # File path provided as taint input + - G305 # File traversal when extracting zip/tar archive + - G306 # Poor file permissions used when writing to a new file + - G307 # Poor file permissions used when creating a file with os.Create + - G401 # Detect the usage of DES, RC4, MD5 or SHA1 + - G403 # Ensure minimum RSA key length of 2048 bits + - G501 # Import blocklist: crypto/md5 + - G502 # Import blocklist: crypto/des + - G503 # Import blocklist: crypto/rc4 + - G504 # Import blocklist: net/http/cgi + - G505 # Import blocklist: crypto/sha1 + - G602 # Slice access out of bounds \ No newline at end of file diff --git a/libs/events/Earthfile b/libs/events/Earthfile new file mode 100644 index 0000000000..ae8f5da3ba --- /dev/null +++ b/libs/events/Earthfile @@ -0,0 +1,46 @@ +VERSION --arg-scope-and-set --pass-args --use-function-keyword 0.7 + +ARG core=github.com/formancehq/earthly:v0.11.1 +IMPORT $core AS core +IMPORT ../.. AS stack + +FROM core+base-image + +go-sources: + COPY events.go go.* base.yaml /src/ + COPY --dir services /src/ + WORKDIR /src + SAVE ARTIFACT /src + +tidy: + FROM core+builder-image + COPY (+go-sources/*) /src + WORKDIR /src + DO --pass-args stack+GO_TIDY + SAVE ARTIFACT go.* AS LOCAL ./ + +lint: + FROM core+builder-image + COPY (+go-sources/*) /src + WORKDIR /src + COPY --pass-args +tidy/go.* . + DO --pass-args stack+GO_LINT + SAVE ARTIFACT * AS LOCAL ./ + +generate: + RUN apk add nodejs npm + WORKDIR /src + COPY package* . + RUN npm install + RUN mkdir generated + COPY index.js base.yaml . + COPY --dir services . + RUN node index.js + SAVE ARTIFACT generated AS LOCAL ./generated + +pre-commit: + WAIT + BUILD --pass-args +tidy + END + BUILD --pass-args +lint + BUILD --pass-args +generate \ No newline at end of file diff --git a/libs/events/events.go b/libs/events/events.go index c40411abe0..312b88a178 100644 --- a/libs/events/events.go +++ b/libs/events/events.go @@ -1,10 +1,11 @@ package events import ( - "embed" "fmt" "path/filepath" + "embed" + "github.com/pkg/errors" "github.com/xeipuuv/gojsonschema" "golang.org/x/mod/semver" diff --git a/libs/events/generated/ledger-v2.0.0-COMMITTED_TRANSACTIONS.yaml.json b/libs/events/generated/ledger-v2.0.0-COMMITTED_TRANSACTIONS.yaml.json index 3e4ff79ef9..142b775bd8 100644 --- a/libs/events/generated/ledger-v2.0.0-COMMITTED_TRANSACTIONS.yaml.json +++ b/libs/events/generated/ledger-v2.0.0-COMMITTED_TRANSACTIONS.yaml.json @@ -1 +1 @@ -{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","properties":{"ledger":{"type":"string"},"transactions":{"type":"array","items":{"type":"object","properties":{"postings":{"type":"array","items":{"type":"object","properties":{"source":{"type":"string"},"destination":{"type":"string"},"amount":{"type":"number"},"asset":{"type":"string"}},"required":["source","destination","amount","asset"]}},"reference":{"type":"string"},"metadata":{"type":"object","properties":{},"required":[]},"id":{"type":"number"},"timestamp":{"type":"string"},"reverted":{"type":"boolean"}},"required":["postings","reference","metadata","id","timestamp","reverted"]}}},"required":["ledger","transactions"]}},"required":["date","app","version","type","payload"]} \ No newline at end of file +{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","properties":{"ledger":{"type":"string"},"transactions":{"type":"array","items":{"type":"object","properties":{"postings":{"type":"array","items":{"type":"object","properties":{"source":{"type":"string"},"destination":{"type":"string"},"amount":{"type":"number"},"asset":{"type":"string"}},"required":["source","destination","amount","asset"]}},"reference":{"type":"string"},"metadata":{"type":"object","properties":{},"required":[]},"id":{"type":"number"},"timestamp":{"type":"string"},"reverted":{"type":"boolean"}},"required":["postings","metadata","id","timestamp","reverted"]}}},"required":["ledger","transactions"]}},"required":["date","app","version","type","payload"]} \ No newline at end of file diff --git a/libs/events/generated/ledger-v2.0.0-REVERTED_TRANSACTION.yaml.json b/libs/events/generated/ledger-v2.0.0-REVERTED_TRANSACTION.yaml.json index eb551f2924..3741d89705 100644 --- a/libs/events/generated/ledger-v2.0.0-REVERTED_TRANSACTION.yaml.json +++ b/libs/events/generated/ledger-v2.0.0-REVERTED_TRANSACTION.yaml.json @@ -1 +1 @@ -{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","properties":{"ledger":{"type":"string"},"revertedTransaction":{"type":"object","properties":{"postings":{"type":"array","items":{"type":"object","properties":{"source":{"type":"string"},"destination":{"type":"string"},"amount":{"type":"number"},"asset":{"type":"string"}},"required":["source","destination","amount","asset"]}},"reference":{"type":"string"},"metadata":{"type":"object","properties":{},"required":[]},"id":{"type":"number"},"timestamp":{"type":"string"},"reverted":{"type":"boolean"}},"required":["postings","reference","metadata","id","timestamp","reverted"]},"revertTransaction":{"type":"object","properties":{"postings":{"type":"array","items":{"type":"object","properties":{"source":{"type":"string"},"destination":{"type":"string"},"amount":{"type":"number"},"asset":{"type":"string"}},"required":["source","destination","amount","asset"]}},"reference":{"type":"string"},"metadata":{"type":"object","properties":{},"required":[]},"id":{"type":"number"},"timestamp":{"type":"string"}},"required":["postings","reference","metadata","id","timestamp"]}},"required":["ledger","revertedTransaction","revertTransaction"]}},"required":["date","app","version","type","payload"]} \ No newline at end of file +{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","properties":{"ledger":{"type":"string"},"revertedTransaction":{"type":"object","properties":{"postings":{"type":"array","items":{"type":"object","properties":{"source":{"type":"string"},"destination":{"type":"string"},"amount":{"type":"number"},"asset":{"type":"string"}},"required":["source","destination","amount","asset"]}},"reference":{"type":"string"},"metadata":{"type":"object","properties":{},"required":[]},"id":{"type":"number"},"timestamp":{"type":"string"},"reverted":{"type":"boolean"}},"required":["postings","metadata","id","timestamp","reverted"]},"revertTransaction":{"type":"object","properties":{"postings":{"type":"array","items":{"type":"object","properties":{"source":{"type":"string"},"destination":{"type":"string"},"amount":{"type":"number"},"asset":{"type":"string"}},"required":["source","destination","amount","asset"]}},"reference":{"type":"string"},"metadata":{"type":"object","properties":{},"required":[]},"id":{"type":"number"},"timestamp":{"type":"string"}},"required":["postings","metadata","id","timestamp"]}},"required":["ledger","revertedTransaction","revertTransaction"]}},"required":["date","app","version","type","payload"]} \ No newline at end of file diff --git a/libs/events/generated/payments-v1.0.0-CONNECTOR_RESET.yaml.json b/libs/events/generated/payments-v1.0.0-CONNECTOR_RESET.yaml.json index ee60cdfeae..dfeef51ec1 100644 --- a/libs/events/generated/payments-v1.0.0-CONNECTOR_RESET.yaml.json +++ b/libs/events/generated/payments-v1.0.0-CONNECTOR_RESET.yaml.json @@ -1 +1 @@ -{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","properties":{"createdAt":{"type":"string"},"connector":{"type":"string"}},"required":["createdAt","connector"]}},"required":["date","app","version","type","payload"]} \ No newline at end of file +{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","properties":{"createdAt":{"type":"string"},"connector":{"type":"string"}},"required":["createdAt","connectorId"]}},"required":["date","app","version","type","payload"]} \ No newline at end of file diff --git a/libs/events/generated/payments-v1.0.0-DELETED_POOL.yaml.json b/libs/events/generated/payments-v1.0.0-DELETED_POOL.yaml.json new file mode 100644 index 0000000000..1ce33750a7 --- /dev/null +++ b/libs/events/generated/payments-v1.0.0-DELETED_POOL.yaml.json @@ -0,0 +1 @@ +{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","required":["id","createdAt"],"properties":{"id":{"type":"string"},"createdAt":{"type":"string","format":"date-time"}}}},"required":["date","app","version","type","payload"]} \ No newline at end of file diff --git a/libs/events/generated/payments-v1.0.0-DELETED_TRANSFER_INITIATION.yaml.json b/libs/events/generated/payments-v1.0.0-DELETED_TRANSFER_INITIATION.yaml.json new file mode 100644 index 0000000000..1ce33750a7 --- /dev/null +++ b/libs/events/generated/payments-v1.0.0-DELETED_TRANSFER_INITIATION.yaml.json @@ -0,0 +1 @@ +{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","required":["id","createdAt"],"properties":{"id":{"type":"string"},"createdAt":{"type":"string","format":"date-time"}}}},"required":["date","app","version","type","payload"]} \ No newline at end of file diff --git a/libs/events/generated/payments-v1.0.0-SAVED_ACCOUNT.yaml.json b/libs/events/generated/payments-v1.0.0-SAVED_ACCOUNT.yaml.json new file mode 100644 index 0000000000..3b11a8f4e1 --- /dev/null +++ b/libs/events/generated/payments-v1.0.0-SAVED_ACCOUNT.yaml.json @@ -0,0 +1 @@ +{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","required":["id","reference","createdAt","connectorId","provider","defaultAsset","accountName","type"],"properties":{"id":{"type":"string"},"reference":{"type":"string"},"createdAt":{"type":"string","format":"date-time"},"connectorId":{"type":"string"},"provider":{"type":"string"},"defaultAsset":{"type":"string"},"accountName":{"type":"string"},"type":{"type":"string","enum":["UNKNOWN","INTERNAL","EXTERNAL"]}}}},"required":["date","app","version","type","payload"]} \ No newline at end of file diff --git a/libs/events/generated/payments-v1.0.0-SAVED_BALANCE.yaml.json b/libs/events/generated/payments-v1.0.0-SAVED_BALANCE.yaml.json new file mode 100644 index 0000000000..6350d7a16e --- /dev/null +++ b/libs/events/generated/payments-v1.0.0-SAVED_BALANCE.yaml.json @@ -0,0 +1 @@ +{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","required":["accountID","connectorId","createdAt","asset","balance"],"properties":{"accountID":{"type":"string"},"connectorId":{"type":"string"},"createdAt":{"type":"string","format":"date-time"},"asset":{"type":"string"},"balance":{"type":"number","format":"bigint"}}}},"required":["date","app","version","type","payload"]} \ No newline at end of file diff --git a/libs/events/generated/payments-v1.0.0-SAVED_BANK_ACCOUNT.yaml.json b/libs/events/generated/payments-v1.0.0-SAVED_BANK_ACCOUNT.yaml.json new file mode 100644 index 0000000000..58cc015c34 --- /dev/null +++ b/libs/events/generated/payments-v1.0.0-SAVED_BANK_ACCOUNT.yaml.json @@ -0,0 +1 @@ +{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","required":["id","createdAt","name","country"],"properties":{"id":{"type":"string"},"createdAt":{"type":"string","format":"date-time"},"name":{"type":"string"},"country":{"type":"string"},"accountNumber":{"type":"string"},"iban":{"type":"string"},"swiftBicCode":{"type":"string"},"adjustments":{"type":["array","null"],"items":{"type":"object","required":["id","createdAt","accountID","connectorID","provider"],"properties":{"id":{"type":"string"},"createdAt":{"type":"string","format":"date-time"},"accountID":{"type":"string"},"connectorID":{"type":"string"},"provider":{"type":"string"}}}}}}},"required":["date","app","version","type","payload"]} \ No newline at end of file diff --git a/libs/events/generated/payments-v1.0.0-SAVED_PAYMENT.yaml.json b/libs/events/generated/payments-v1.0.0-SAVED_PAYMENT.yaml.json index 1334ed60ab..8fc954bca5 100644 --- a/libs/events/generated/payments-v1.0.0-SAVED_PAYMENT.yaml.json +++ b/libs/events/generated/payments-v1.0.0-SAVED_PAYMENT.yaml.json @@ -1 +1 @@ -{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","properties":{"provider":{"type":"string"},"reference":{"type":"string"},"scheme":{"type":"string","enum":["unknown","other","visa","mastercard","amex","diners","discover","jcb","unionpay","sepa debit","sepa credit","sepa","apple pay","google pay","a2a","ach debit","ach","rtp"]},"type":{"type":"string","enum":["PAY-IN","PAYOUT","TRANSFER","OTHER"]},"status":{"type":"string"},"id":{"type":"string"},"initialAmount":{"type":"number"},"createdAt":{"type":"string","format":"date-time"}}}},"required":["date","app","version","type","payload"]} \ No newline at end of file +{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","required":["id","reference","createdAt","connectorId","provider","type","status","scheme","asset","amount","initialAmount"],"properties":{"id":{"type":"string"},"reference":{"type":"string"},"createdAt":{"type":"string","format":"date-time"},"connectorId":{"type":"string"},"provider":{"type":"string"},"type":{"type":"string","enum":["PAY-IN","PAYOUT","TRANSFER","OTHER"]},"status":{"type":"string"},"scheme":{"type":"string","enum":["unknown","other","visa","mastercard","amex","diners","discover","jcb","unionpay","alipay","cup","sepa debit","sepa credit","sepa","apple pay","google pay","doku","dragonpay","maestro","molpay","a2a","ach debit","ach","rtp"]},"asset":{"type":"string"},"amount":{"type":"number","format":"bigint"},"initialAmount":{"type":"number","format":"bigint"},"sourceAccountId":{"type":"string"},"destinationAccountId":{"type":"string"},"links":{"type":["array","null"],"items":{"type":"object","properties":{"name":{"type":"string"},"uri":{"type":"string"}}}},"rawData":{"type":"object"},"metadata":{"type":"object","additionalProperties":{"type":"string"}}}}},"required":["date","app","version","type","payload"]} \ No newline at end of file diff --git a/libs/events/generated/payments-v1.0.0-SAVED_POOL.yaml.json b/libs/events/generated/payments-v1.0.0-SAVED_POOL.yaml.json new file mode 100644 index 0000000000..cc3dac4d4f --- /dev/null +++ b/libs/events/generated/payments-v1.0.0-SAVED_POOL.yaml.json @@ -0,0 +1 @@ +{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","required":["id","name","createdAt","accountIDs"],"properties":{"id":{"type":"string"},"name":{"type":"string"},"createdAt":{"type":"string","format":"date-time"},"accountIDs":{"type":"array","items":{"type":"string"}}}}},"required":["date","app","version","type","payload"]} \ No newline at end of file diff --git a/libs/events/generated/payments-v1.0.0-SAVED_TRANSFER_INITIATION.yaml.json b/libs/events/generated/payments-v1.0.0-SAVED_TRANSFER_INITIATION.yaml.json new file mode 100644 index 0000000000..26ac592bff --- /dev/null +++ b/libs/events/generated/payments-v1.0.0-SAVED_TRANSFER_INITIATION.yaml.json @@ -0,0 +1 @@ +{"type":"object","properties":{"app":{"type":"string"},"version":{"type":"string"},"date":{"type":"string","format":"date-time"},"type":{"type":"string"},"ledger":{"type":"string"},"payload":{"type":"object","required":["id","createdAt","scheduledAt","connectorId","provider","description","type","destinationAccountId","amount","asset","attempts","status","error"],"properties":{"id":{"type":"string"},"createdAt":{"type":"string","format":"date-time"},"scheduledAt":{"type":"string","format":"date-time"},"connectorId":{"type":"string"},"provider":{"type":"string"},"description":{"type":"string"},"type":{"type":"string","enum":["TRANSFER","PAYOUT"]},"sourceAccountId":{"type":"string"},"destinationAccountId":{"type":"string"},"amount":{"type":"number","format":"bigint"},"asset":{"type":"string"},"attempts":{"type":"number"},"status":{"type":"string","enum":["WAITING_FOR_VALIDATION","PROCESSING","PROCESSED","FAILED","REJECTED","VALIDATED","ASK_RETRIED","ASK_REVERSED","REVERSE_PROCESSING","REVERSE_FAILED","PARTIALLY_REVERSED","REVERSED"]},"error":{"type":"string"},"relatedPayments":{"type":["array","null"],"items":{"type":"object","required":["transferInitiationId","paymentId","createdAt","status","error"],"properties":{"transferInitiationId":{"type":"string"},"paymentId":{"type":"string"},"createdAt":{"type":"string","format":"date-time"},"status":{"type":"string"},"error":{"type":"string"}}}}}}},"required":["date","app","version","type","payload"]} \ No newline at end of file diff --git a/libs/events/index.js b/libs/events/index.js index 79f76b8489..499911ed0f 100644 --- a/libs/events/index.js +++ b/libs/events/index.js @@ -1,6 +1,5 @@ const fs = require("fs/promises"); -const yaml = require('yaml') -const JsonSchemaStaticDocs = require("json-schema-static-docs"); +const yaml = require('yaml'); (async () => { diff --git a/libs/go-libs/.golangci.yml b/libs/go-libs/.golangci.yml new file mode 100644 index 0000000000..5cafc8d3d0 --- /dev/null +++ b/libs/go-libs/.golangci.yml @@ -0,0 +1,43 @@ +allow-parallel-runners: true +run: + timeout: 5m +linters: + disable-all: true + enable: + - gofmt + - goimports + - unused + - gosec +linters-settings: + gosec: + # To select a subset of rules to run. + # Available rules: https://github.com/securego/gosec#available-rules + includes: + - G103 # Audit the use of unsafe block + - G104 # Audit errors not checked + - G106 # Audit the use of ssh.InsecureIgnoreHostKey + - G108 # Profiling endpoint automatically exposed on /debug/pprof + - G109 # Potential Integer overflow made by strconv.Atoi result conversion to int16/32 + - G110 # Potential DoS vulnerability via decompression bomb + - G111 # Potential directory traversal + - G112 # Potential slowloris attack +# - G113 # Usage of Rat.SetString in math/big with an overflow (CVE-2022-23772) + - G201 # SQL query construction using format string + - G202 # SQL query construction using string concatenation + - G203 # Use of unescaped data in HTML templates + - G204 # Audit use of command execution + - G301 # Poor file permissions used when creating a directory + - G302 # Poor file permissions used with chmod + - G303 # Creating tempfile using a predictable path + - G304 # File path provided as taint input + - G305 # File traversal when extracting zip/tar archive + - G306 # Poor file permissions used when writing to a new file + - G307 # Poor file permissions used when creating a file with os.Create + - G401 # Detect the usage of DES, RC4, MD5 or SHA1 + - G403 # Ensure minimum RSA key length of 2048 bits + - G501 # Import blocklist: crypto/md5 + - G502 # Import blocklist: crypto/des + - G503 # Import blocklist: crypto/rc4 + - G504 # Import blocklist: net/http/cgi + - G505 # Import blocklist: crypto/sha1 + - G602 # Slice access out of bounds \ No newline at end of file diff --git a/libs/go-libs/Earthfile b/libs/go-libs/Earthfile index e1df0cd8ad..2b6579d6f2 100644 --- a/libs/go-libs/Earthfile +++ b/libs/go-libs/Earthfile @@ -2,6 +2,7 @@ VERSION --arg-scope-and-set --pass-args --use-function-keyword 0.7 ARG core=github.com/formancehq/earthly IMPORT $core AS core +IMPORT ../.. AS stack FROM core+base-image @@ -15,4 +16,18 @@ tidy: COPY (+sources/*) /src WORKDIR /src DO --pass-args stack+GO_TIDY - SAVE ARTIFACT go.* AS LOCAL ./ \ No newline at end of file + SAVE ARTIFACT go.* AS LOCAL ./ + +lint: + FROM core+builder-image + COPY (+sources/*) /src + WORKDIR /src + COPY --pass-args +tidy/go.* . + DO --pass-args stack+GO_LINT + SAVE ARTIFACT * AS LOCAL ./ + +pre-commit: + WAIT + BUILD --pass-args +tidy + END + BUILD --pass-args +lint \ No newline at end of file diff --git a/libs/go-libs/aws/iam/load.go b/libs/go-libs/aws/iam/load.go index 332207e7d0..86aa4cab89 100644 --- a/libs/go-libs/aws/iam/load.go +++ b/libs/go-libs/aws/iam/load.go @@ -2,6 +2,7 @@ package iam import ( "context" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/spf13/pflag" diff --git a/libs/go-libs/bun/bunconnect/flags.go b/libs/go-libs/bun/bunconnect/flags.go index 5fc94cf3e4..973bce9d00 100644 --- a/libs/go-libs/bun/bunconnect/flags.go +++ b/libs/go-libs/bun/bunconnect/flags.go @@ -3,6 +3,8 @@ package bunconnect import ( "context" "database/sql/driver" + "time" + "github.com/aws/aws-sdk-go-v2/config" "github.com/formancehq/stack/libs/go-libs/aws/iam" "github.com/formancehq/stack/libs/go-libs/logging" @@ -10,7 +12,6 @@ import ( "github.com/lib/pq" "github.com/spf13/pflag" "github.com/spf13/viper" - "time" ) const ( diff --git a/libs/go-libs/bun/bunconnect/iam.go b/libs/go-libs/bun/bunconnect/iam.go index f7f90acd11..48f4667fe2 100644 --- a/libs/go-libs/bun/bunconnect/iam.go +++ b/libs/go-libs/bun/bunconnect/iam.go @@ -4,6 +4,7 @@ import ( "context" "database/sql/driver" "fmt" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/feature/rds/auth" "github.com/formancehq/stack/libs/go-libs/logging" diff --git a/libs/go-libs/bun/bunconnect/module.go b/libs/go-libs/bun/bunconnect/module.go index 664e1df353..8a403e877a 100644 --- a/libs/go-libs/bun/bunconnect/module.go +++ b/libs/go-libs/bun/bunconnect/module.go @@ -2,6 +2,7 @@ package bunconnect import ( "context" + "github.com/formancehq/stack/libs/go-libs/logging" "github.com/uptrace/bun" "go.uber.org/fx" diff --git a/libs/go-libs/bun/bunmigrate/command.go b/libs/go-libs/bun/bunmigrate/command.go index 82e6ccd5b3..0a083f42e0 100644 --- a/libs/go-libs/bun/bunmigrate/command.go +++ b/libs/go-libs/bun/bunmigrate/command.go @@ -4,6 +4,7 @@ import ( "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" "github.com/spf13/cobra" "github.com/uptrace/bun" + // Import the postgres driver. _ "github.com/lib/pq" ) diff --git a/libs/go-libs/bun/bunmigrate/run.go b/libs/go-libs/bun/bunmigrate/run.go index b0fa2a61c8..ef937ee178 100644 --- a/libs/go-libs/bun/bunmigrate/run.go +++ b/libs/go-libs/bun/bunmigrate/run.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "fmt" + "io" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" sharedlogging "github.com/formancehq/stack/libs/go-libs/logging" "github.com/formancehq/stack/libs/go-libs/pointer" @@ -14,7 +16,6 @@ import ( "github.com/uptrace/bun" "github.com/uptrace/bun/extra/bundebug" "github.com/xo/dburl" - "io" ) func isDatabaseExists(ctx context.Context, db *bun.DB, name string) (bool, error) { diff --git a/libs/go-libs/bun/bunmigrate/run_test.go b/libs/go-libs/bun/bunmigrate/run_test.go index afc794ddde..8de8732abf 100644 --- a/libs/go-libs/bun/bunmigrate/run_test.go +++ b/libs/go-libs/bun/bunmigrate/run_test.go @@ -1,13 +1,14 @@ package bunmigrate import ( + "os" + "testing" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/stretchr/testify/require" "github.com/uptrace/bun" - "os" - "testing" ) func TestRunMigrate(t *testing.T) { @@ -19,7 +20,6 @@ func TestRunMigrate(t *testing.T) { connectionOptions := &bunconnect.ConnectionOptions{ DatabaseSourceName: pgtesting.Server().GetDatabaseDSN("testing"), Debug: testing.Verbose(), - Writer: os.Stdout, } executor := func(args []string, db *bun.DB) error { return nil diff --git a/libs/go-libs/bun/bunpaginate/pagination_column_test.go b/libs/go-libs/bun/bunpaginate/pagination_column_test.go index e0801a781c..f76766b9f8 100644 --- a/libs/go-libs/bun/bunpaginate/pagination_column_test.go +++ b/libs/go-libs/bun/bunpaginate/pagination_column_test.go @@ -2,11 +2,12 @@ package bunpaginate_test import ( "context" + "math/big" + "testing" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" bunpaginate2 "github.com/formancehq/stack/libs/go-libs/bun/bunpaginate" "github.com/formancehq/stack/libs/go-libs/logging" - "math/big" - "testing" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/stretchr/testify/require" diff --git a/libs/go-libs/bun/bunpaginate/pagination_offset_test.go b/libs/go-libs/bun/bunpaginate/pagination_offset_test.go index c5f8f394c5..bfd554ad05 100644 --- a/libs/go-libs/bun/bunpaginate/pagination_offset_test.go +++ b/libs/go-libs/bun/bunpaginate/pagination_offset_test.go @@ -2,10 +2,11 @@ package bunpaginate_test import ( "context" + "testing" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" bunpaginate2 "github.com/formancehq/stack/libs/go-libs/bun/bunpaginate" "github.com/formancehq/stack/libs/go-libs/logging" - "testing" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/stretchr/testify/require" diff --git a/libs/go-libs/httpserver/serverport.go b/libs/go-libs/httpserver/serverport.go index 8a7aef4bbe..4d01bbf60c 100644 --- a/libs/go-libs/httpserver/serverport.go +++ b/libs/go-libs/httpserver/serverport.go @@ -6,6 +6,7 @@ import ( "net" "net/http" "strconv" + "time" "github.com/formancehq/stack/libs/go-libs/logging" @@ -80,7 +81,8 @@ func (s *server) StartServer(ctx context.Context, handler http.Handler, options StartedServer(ctx, s.listener) srv := &http.Server{ - Handler: handler, + Handler: handler, + ReadHeaderTimeout: 10 * time.Second, } for _, option := range options { option(srv) diff --git a/libs/go-libs/migrations/migrator.go b/libs/go-libs/migrations/migrator.go index 054cc839f4..9e080a984c 100644 --- a/libs/go-libs/migrations/migrator.go +++ b/libs/go-libs/migrations/migrator.go @@ -4,9 +4,10 @@ import ( "context" "database/sql" "fmt" - "github.com/lib/pq" "time" + "github.com/lib/pq" + "github.com/pkg/errors" "github.com/uptrace/bun" ) diff --git a/libs/go-libs/pgtesting/postgres.go b/libs/go-libs/pgtesting/postgres.go index 4804b0d592..268e8ca016 100644 --- a/libs/go-libs/pgtesting/postgres.go +++ b/libs/go-libs/pgtesting/postgres.go @@ -4,13 +4,14 @@ import ( "context" "database/sql" "fmt" - "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" "os" "strconv" "sync" "testing" "time" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" + "github.com/google/uuid" _ "github.com/lib/pq" "github.com/ory/dockertest/v3" diff --git a/libs/go-libs/publish/messages.go b/libs/go-libs/publish/messages.go index 4f08e3817c..7343b2e59d 100644 --- a/libs/go-libs/publish/messages.go +++ b/libs/go-libs/publish/messages.go @@ -3,10 +3,11 @@ package publish import ( "context" "encoding/json" + "time" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" - "time" "github.com/ThreeDotsLabs/watermill/message" "github.com/google/uuid" diff --git a/libs/go-libs/publish/module.go b/libs/go-libs/publish/module.go index b46145b2d0..2811c4ae44 100644 --- a/libs/go-libs/publish/module.go +++ b/libs/go-libs/publish/module.go @@ -2,6 +2,7 @@ package publish import ( "context" + "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill/message" "github.com/ThreeDotsLabs/watermill/pubsub/gochannel" diff --git a/libs/go-libs/publish/module_test.go b/libs/go-libs/publish/module_test.go index 536cd2ebdc..6bdf0bbbfe 100644 --- a/libs/go-libs/publish/module_test.go +++ b/libs/go-libs/publish/module_test.go @@ -3,15 +3,16 @@ package publish import ( "context" "fmt" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/propagation" - tracesdk "go.opentelemetry.io/otel/sdk/trace" - "go.opentelemetry.io/otel/trace" "io" "os" "testing" "time" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/trace" + "github.com/IBM/sarama" "github.com/ThreeDotsLabs/watermill/message" "github.com/formancehq/stack/libs/go-libs/logging" diff --git a/libs/go-libs/service/app.go b/libs/go-libs/service/app.go index c52751c0a6..5fd5201749 100644 --- a/libs/go-libs/service/app.go +++ b/libs/go-libs/service/app.go @@ -2,12 +2,13 @@ package service import ( "context" + "io" + "os" + "github.com/formancehq/stack/libs/go-libs/logging" "github.com/spf13/cobra" "github.com/spf13/viper" "go.uber.org/fx" - "io" - "os" ) const DebugFlag = "debug"