feat: remove earthly usage for benchmarks

.github/workflows/benchmark.yml

@@ -11,33 +11,44 @@ concurrency:
 jobs:
   Benchmark:
     runs-on: "github-001"
-    if: contains(github.event.pull_request.labels.*.name, 'benchmarks') || github.ref == 'refs/heads/main'
+    if: github.ref == 'refs/heads/main'
     steps:
       - uses: 'actions/checkout@v4'
         with:
           fetch-depth: 0
-      - run: go build -o /tmp/ledger ./
-      - run: echo "running actions as ${USER}"
+      - uses: extractions/setup-just@v2
       - run: >
-          /tmp/ledger serve 
-          --postgres-uri=postgres://formance:[email protected]/ledger 
-          --postgres-conn-max-idle-time=120s 
-          --postgres-max-open-conns=500 
-          --postgres-max-idle-conns=100 
-          --experimental-features 
-          --otel-metrics-keep-in-memory &
+          just
+          --justfile ./test/performance/justfile
+          --working-directory ./test/performance 
+          run . 5 10s 1
       - run: >
-          earthly
-          --allow-privileged
-          ${{ contains(github.event.pull_request.labels.*.name, 'no-cache') && '--no-cache' || '' }}
-          ./test/performance+run --args="-benchtime 10s --ledger.url=http://localhost:3068 --parallelism=5" --locally=yes
+          just 
+          --justfile ./test/performance/justfile 
+          --working-directory ./test/performance 
+          graphs
+      - uses: actions/upload-artifact@v4
+        with:
+          name: graphs
+          path: test/performance/report
+  BenchmarkCompare:
+    runs-on: "github-001"
+    if: contains(github.event.pull_request.labels.*.name, 'benchmarks')
+    steps:
+      - uses: 'actions/checkout@v4'
+        with:
+          fetch-depth: 0
+      - uses: extractions/setup-just@v2
+      - run: >
+          just
+          --justfile ./test/performance/justfile
+          --working-directory ./test/performance 
+          compare . 5 10s 5
       - run: >
-          earthly
-          --allow-privileged
-          ${{ contains(github.event.pull_request.labels.*.name, 'no-cache') && '--no-cache' || '' }}
-          ./test/performance+generate-graphs
-      - run: kill -9 $(ps aux | grep "ledger serve"| grep -v "grep" | awk '{print $2}')
-        if: always()
+          just 
+          --justfile ./test/performance/justfile 
+          --working-directory ./test/performance 
+          graphs
       - uses: actions/upload-artifact@v4
         with:
           name: graphs

flake.nix

@@ -101,6 +101,7 @@
               just
               nodejs_22
               self.packages.${system}.speakeasy
+              goperf
             ];
           };
         }

test/performance/README.md

@@ -10,25 +10,25 @@ Scripts can be found in directory [scripts](./scripts).
 ## Run locally
 
 ```shell
-earthly +run
+just run
 ```
 
 You can pass additional arguments (the underlying command is a standard `go test -bench=.`) using the flag `--args`.
 For example:
 ```shell
-earthly +run --args="-benchtime 10s"
+just run "-benchtime 10s"
 ```
 
 ## Run on a remote stack
 
 ```shell
-earthly +run --args="--stack.url=XXX --client.id=XXX --client.secret=XXX"
+just run "--stack.url=XXX --client.id=XXX --client.secret=XXX"
 ```
 
 ## Run on a remote ledger
 
 ```shell
-earthly +run --args="--ledger.url=XXX --auth.url=XXX --client.id=XXX --client.secret=XXX"
+just run "--ledger.url=XXX --auth.url=XXX --client.id=XXX --client.secret=XXX"
 ```
 
 ## Results
@@ -37,7 +37,7 @@ TPS is included as a benchmark metrics.
 
 You can generate some graphs using the command: 
 ```
-earthly +generate-graphs
+just graphs
 ```
 
 See generated files in `report` directory.

test/performance/charts/.gitignore

@@ -0,0 +1 @@
+*.js

test/performance/charts/index.ts

@@ -6,15 +6,15 @@ const main = async () => {
     let buffer = fs.readFileSync('../report/report.json', 'utf-8');
     let reports = JSON.parse(buffer);
     await exportTPSGraph({
-        output: 'tps.png',
+        output: '../report/tps.png',
     }, reports);
 
-    await exportDatabaseStats('database_connections.png', reports);
+    await exportDatabaseStats('../report/database_connections.png', reports);
 
     const ps: (keyof MetricsTime)[] = ['P99', 'P95', 'P75', 'Avg']
     for (let p of ps) {
         await exportLatencyGraph({
-            output: p.toLowerCase() + '.png'
+            output: '../report/' + p.toLowerCase() + '.png'
         }, p, reports);
     }
 }