Merge branch 'main' into fix/broken-installation-link

formancehq · Jan 6, 2025 · 8b857a3 · 8b857a3
2 parents 8e30ebf + 58854fe
commit 8b857a3
Show file tree

Hide file tree

Showing 366 changed files with 15,181 additions and 9,087 deletions.
diff --git a/.envrc b/.envrc
@@ -0,0 +1,2 @@
+use flake --impure
+dotenv
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -4,7 +4,7 @@ on:
   push:
     branches:
       - main
-      - releases/*
+      - release/*
   pull_request:
     types: [ assigned, opened, synchronize, reopened, labeled ]
 
@@ -35,11 +35,8 @@ jobs:
         with:
           token: ${{ secrets.NUMARY_GITHUB_TOKEN }}
       - run: >
-          earthly
-          --allow-privileged
-          --secret SPEAKEASY_API_KEY=$SPEAKEASY_API_KEY
-          ${{ contains(github.event.pull_request.labels.*.name, 'no-cache') && '--no-cache' || '' }}
-          +pre-commit
+          /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes"
+          develop --impure --command just pre-commit
         env:
           SPEAKEASY_API_KEY: ${{ secrets.SPEAKEASY_API_KEY }}
       - name: Get changed files
@@ -65,11 +62,8 @@ jobs:
         with:
           token: ${{ secrets.NUMARY_GITHUB_TOKEN }}
       - run: >
-          earthly
-          --allow-privileged
-          --secret SPEAKEASY_API_KEY=$SPEAKEASY_API_KEY
-          ${{ contains(github.event.pull_request.labels.*.name, 'no-cache') && '--no-cache' || '' }}
-          +tests --coverage=true
+          /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes"
+          develop --impure --command just tests
         env:
           SPEAKEASY_API_KEY: ${{ secrets.SPEAKEASY_API_KEY }}
       - name: Upload coverage reports to Codecov with GitHub Action
@@ -101,15 +95,8 @@ jobs:
           username: "NumaryBot"
           password: ${{ secrets.NUMARY_GITHUB_TOKEN }}
       - run: >
-          earthly
-          --no-output
-          --allow-privileged
-          --secret SPEAKEASY_API_KEY=$SPEAKEASY_API_KEY
-          --secret GITHUB_TOKEN=$GITHUB_TOKEN
-          --secret FURY_TOKEN=$FURY_TOKEN
-          --secret GORELEASER_KEY=$GORELEASER_KEY
-          ${{ contains(github.event.pull_request.labels.*.name, 'no-cache') && '--no-cache' || '' }}
-          +release --mode=ci
+          /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes" 
+          develop --impure --command just release-ci
         env:
           GITHUB_TOKEN: ${{ secrets.NUMARY_GITHUB_TOKEN }}
           SPEAKEASY_API_KEY: ${{ secrets.SPEAKEASY_API_KEY }}
@@ -131,12 +118,6 @@ jobs:
       - uses: 'actions/checkout@v4'
         with:
           fetch-depth: 0
-      - name: Tailscale
-        uses: tailscale/github-action@v2
-        with:
-          oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }}
-          oauth-secret: ${{ secrets.TS_OAUTH_SECRET }}
-          tags: tag:ci
       - name: "Deploy in staging"
         env:
           TAG: ${{ github.sha }}

diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml
@@ -29,15 +29,8 @@ jobs:
           username: "NumaryBot"
           password: ${{ secrets.NUMARY_GITHUB_TOKEN }}
       - run: >
-          earthly
-          --no-output
-          --allow-privileged
-          --secret SPEAKEASY_API_KEY=$SPEAKEASY_API_KEY
-          --secret GITHUB_TOKEN=$GITHUB_TOKEN
-          --secret FURY_TOKEN=$FURY_TOKEN
-          --secret GORELEASER_KEY=$GORELEASER_KEY
-          ${{ contains(github.event.pull_request.labels.*.name, 'no-cache') && '--no-cache' || '' }}
-          +release --mode=release
+          /nix/var/nix/profiles/default/bin/nix --extra-experimental-features "nix-command" --extra-experimental-features "flakes" 
+          develop --impure --command just release
         env:
           GITHUB_TOKEN: ${{ secrets.NUMARY_GITHUB_TOKEN }}
           SPEAKEASY_API_KEY: ${{ secrets.SPEAKEASY_API_KEY }}

diff --git a/.gitignore b/.gitignore
@@ -6,3 +6,5 @@ dist
 vendor
 worktrees
 dumps
+.env
+!.envrc
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -382,10 +382,12 @@ That's because, if we would do that, we would have frequent serialization errors
 
 Tests are split in different scopes :
 * Unit tests: as any go app, you will find unit test along the source code in _test.go files over the app.
-* [e2e](./test/e2e) : End to end test. Those tests are mainly api tests, and app lifecycle tests. It checks than the ledger endpoint works as expected.
-* [migrations](./test/migrations) : Migrations tests. Tests inside this package allow to import an existing database to apply current code migrations on it.
-* [performance](./test/performance) : Performance tests. Tests inside this package test performance of the ledger.
-* [stress](./test/stress) : Stress tests. Tests inside this package ensure than ledger state stay consistent under high concurrency.
+* Integration tests: tests that involve the ledger and external services, like the database.
+  * [e2e](./test/e2e) : End to end test. Those tests are mainly api tests, and app lifecycle tests. It checks than the ledger endpoint works as expected.
+  * [migrations](./test/migrations) : Migrations tests. Tests inside this package allow to import an existing database to apply current code migrations on it.
+  * [performance](./test/performance) : Performance tests. Tests inside this package test performance of the ledger.
+  * [stress](./test/stress) : Stress tests. Tests inside this package ensure than ledger state stay consistent under high concurrency.
+  * [rolling-upgrades](./test/rolling-upgrades) : Rolling upgrade test under K8S 
 
 ## API changes
 

diff --git a/Earthfile b/Earthfile
@@ -1,13 +1,7 @@
-VERSION 0.8
+VERSION --wildcard-builds 0.8
 PROJECT FormanceHQ/ledger
 
-IMPORT github.com/formancehq/earthly:tags/v0.17.1 AS core
-
-FROM core+base-image
-
-CACHE --sharing=shared --id go-mod-cache /go/pkg/mod
-CACHE --sharing=shared --id golangci-cache /root/.cache/golangci-lint
-CACHE --sharing=shared --id go-cache /root/.cache/go-build
+IMPORT github.com/formancehq/earthly:tags/v0.19.0 AS core
 
 postgres:
     FROM postgres:15-alpine
@@ -22,29 +16,12 @@ sources:
     WORKDIR /src
     COPY go.mod go.sum ./
     RUN go mod download
-    COPY --dir internal pkg cmd tools .
+    COPY --dir internal pkg cmd .
     COPY main.go .
     SAVE ARTIFACT /src
 
-generate:
-    FROM core+builder-image
-    CACHE --id go-mod-cache /go/pkg/mod
-    CACHE --id go-cache /root/.cache/go-build
-    RUN apk update && apk add openjdk11
-    RUN go install go.uber.org/mock/[email protected]
-    RUN go install github.com/princjef/gomarkdoc/cmd/gomarkdoc@latest
-    COPY (+sources/*) /src
-    WORKDIR /src
-    RUN go generate ./...
-    SAVE ARTIFACT internal AS LOCAL internal
-    SAVE ARTIFACT pkg AS LOCAL pkg
-    SAVE ARTIFACT cmd AS LOCAL cmd
-
 compile:
-    FROM +sources
-    CACHE --id go-mod-cache /go/pkg/mod
-    CACHE --id go-cache /root/.cache/go-build
-    WORKDIR /src
+    LOCALLY
     ARG VERSION=latest
     RUN go build -o main -ldflags="-X ${GIT_PATH}/cmd.Version=${VERSION} \
         -X ${GIT_PATH}/cmd.BuildDate=$(date +%s) \
@@ -60,49 +37,6 @@ build-image:
     ARG tag=latest
     DO --pass-args core+SAVE_IMAGE --COMPONENT=ledger --REPOSITORY=${REPOSITORY} --TAG=$tag
 
-tests:
-    FROM +tidy
-    CACHE --id go-mod-cache /go/pkg/mod
-    CACHE --id go-cache /root/.cache/go-build
-    RUN go install github.com/onsi/ginkgo/v2/ginkgo@latest
-    COPY --dir --pass-args (+generate/*) .
-
-    ARG includeIntegrationTests="true"
-    ARG includeEnd2EndTests="true"
-    ARG coverage=""
-    ARG debug=false
-
-    ENV DEBUG=$debug
-    ENV CGO_ENABLED=1 # required for -race
-    RUN apk add gcc musl-dev
-
-    LET goFlags="-race"
-    IF [ "$coverage" = "true" ]
-        SET goFlags="$goFlags -covermode=atomic"
-        SET goFlags="$goFlags -coverpkg=github.com/formancehq/ledger/internal/..."
-        SET goFlags="$goFlags,github.com/formancehq/ledger/pkg/events/..."
-        SET goFlags="$goFlags,github.com/formancehq/ledger/pkg/accounts/..."
-        SET goFlags="$goFlags,github.com/formancehq/ledger/pkg/assets/..."
-        SET goFlags="$goFlags,github.com/formancehq/ledger/cmd/..."
-        SET goFlags="$goFlags -coverprofile coverage.txt"
-    END
-
-    IF [ "$includeIntegrationTests" = "true" ]
-        SET goFlags="$goFlags -tags it"
-        WITH DOCKER --load=postgres:15-alpine=+postgres
-            RUN go test $goFlags ./...
-        END
-    ELSE
-        RUN go test $goFlags ./...
-    END
-    IF [ "$coverage" = "true" ]
-        # as special case, exclude files suffixed by debug.go
-        # toremovelater: exclude machine code as it will be updated soon
-        RUN cat coverage.txt | grep -v debug.go | grep -v "/machine/" > coverage2.txt
-        RUN mv coverage2.txt coverage.txt
-        SAVE ARTIFACT coverage.txt AS LOCAL coverage.txt
-    END
-
 deploy:
     COPY (+sources/*) /src
     LET tag=$(tar cf - /src | sha1sum | awk '{print $1}')
@@ -115,32 +49,10 @@ deploy:
 deploy-staging:
     BUILD --pass-args core+deploy-staging
 
-lint:
-    #todo: get config from core
-    FROM +tidy
-    CACHE --id go-mod-cache /go/pkg/mod
-    CACHE --id go-cache /root/.cache/go-build
-    CACHE --id golangci-cache /root/.cache/golangci-lint
-
-    RUN golangci-lint run --fix --build-tags it --timeout 5m
-
-    SAVE ARTIFACT cmd AS LOCAL cmd
-    SAVE ARTIFACT internal AS LOCAL internal
-    SAVE ARTIFACT pkg AS LOCAL pkg
-    SAVE ARTIFACT test AS LOCAL test
-    SAVE ARTIFACT tools AS LOCAL tools
-    SAVE ARTIFACT main.go AS LOCAL main.go
-
 pre-commit:
-    WAIT
-        BUILD +tidy
-        BUILD +lint
-        BUILD +openapi
-        BUILD +openapi-markdown
-    END
-    BUILD +generate
+    BUILD +openapi
+    BUILD +openapi-markdown
     BUILD +generate-client
-    BUILD +export-docs-events
 
 openapi:
     FROM node:20-alpine
@@ -159,20 +71,6 @@ openapi-markdown:
     RUN widdershins openapi.yaml -o README.md --search false --language_tabs 'http:HTTP' --summary --omitHeader
     SAVE ARTIFACT README.md AS LOCAL docs/api/README.md
 
-tidy:
-    FROM +sources
-    CACHE --id go-mod-cache /go/pkg/mod
-    CACHE --id go-cache /root/.cache/go-build
-    WORKDIR /src
-    COPY --dir test .
-    RUN go mod tidy
-
-release:
-    FROM core+builder-image
-    ARG mode=local
-    COPY --dir . /src
-    DO core+GORELEASER --mode=$mode
-
 generate-client:
     FROM node:20-alpine
     RUN apk update && apk add yq jq
@@ -189,18 +87,23 @@ generate-client:
 export-database-schema:
     FROM +sources
     RUN go install github.com/roerohan/wait-for-it@latest
-    COPY --dir scripts scripts
     WITH DOCKER --load=postgres:15-alpine=+postgres --pull schemaspy/schemaspy:6.2.4
-        RUN ./scripts/export-database-schema.sh
+        RUN bash -c '
+            echo "Creating PG server...";
+            postgresContainerID=$(docker run -d --rm -e POSTGRES_USER=root -e POSTGRES_PASSWORD=root -e POSTGRES_DB=formance --net=host postgres:15-alpine);
+            wait-for-it -w 127.0.0.1:5432;
+
+            echo "Creating bucket...";
+            go run main.go buckets upgrade _default --postgres-uri "postgres://root:[email protected]:5432/formance?sslmode=disable";
+
+            echo "Exporting schemas...";
+            docker run --rm -u root \
+              -v ./docs/database:/output \
+              --net=host \
+              schemaspy/schemaspy:6.2.4 -u root -db formance -t pgsql11 -host 127.0.0.1 -port 5432 -p root -schemas _system,_default;
+
+            docker kill "$postgresContainerID";
+        '
     END
     SAVE ARTIFACT docs/database/_system/diagrams AS LOCAL docs/database/_system/diagrams
     SAVE ARTIFACT docs/database/_default/diagrams AS LOCAL docs/database/_default/diagrams
-
-export-docs-events:
-    FROM +tidy
-    CACHE --id go-mod-cache /go/pkg/mod
-    CACHE --id go-cache /root/.cache/go-build
-
-    RUN go run tools/docs/events/main.go --write-dir docs/events
-
-    SAVE ARTIFACT docs/events AS LOCAL docs/events
diff --git a/Justfile b/Justfile
@@ -0,0 +1,47 @@
+set dotenv-load
+
+default:
+  @just --list
+
+pre-commit: generate earthly tidy lint export-docs-events
+
+earthly:
+  @earthly --no-output +pre-commit
+
+lint:
+  @golangci-lint run --fix --build-tags it --timeout 5m
+  @cd {{justfile_directory()}}/tools/generator && golangci-lint run --fix --build-tags it --timeout 5m
+  @cd {{justfile_directory()}}/deployments/pulumi && golangci-lint run --fix --build-tags it --timeout 5m
+
+tidy:
+  @go mod tidy
+  @cd {{justfile_directory()}}/tools/generator && go mod tidy
+  @cd {{justfile_directory()}}/deployments/pulumi && go mod tidy
+
+generate:
+  @go generate ./...
+
+export-docs-events:
+  @go run . docs events --write-dir docs/events
+
+tests:
+  @go test -race -covermode=atomic \
+    -coverpkg=github.com/formancehq/ledger/internal/... \
+    -coverpkg=github.com/formancehq/ledger/pkg/events/... \
+    -coverpkg=github.com/formancehq/ledger/pkg/accounts/... \
+    -coverpkg=github.com/formancehq/ledger/pkg/assets/... \
+    -coverpkg=github.com/formancehq/ledger/cmd/... \
+    -coverprofile coverage.txt \
+    -tags it \
+    ./...
+  @cat coverage.txt | grep -v debug.go | grep -v "/machine/" > coverage2.txt
+  @mv coverage2.txt coverage.txt
+
+release-local:
+  @goreleaser release --nightly --skip=publish --clean
+
+release-ci:
+  @goreleaser release --nightly --clean
+
+release:
+  @goreleaser release --clean
-Original file line number
+Diff line change
@@ Expand Up / @@ -6,3 +6,5 @@ dist @@
     vendor
     worktrees
     dumps
+    .env
+    !.envrc