revert #332

.github/workflows/bench.yml

@@ -1,77 +1,38 @@
 name: "Run benchmark"
+
 on:
   pull_request_target:
     types: [assigned, opened, synchronize, reopened, edited]
   push:
     branches:
       - main
+
 permissions:
   contents: write
   pull-requests: write
   issues: write
+
 jobs:
-  benchmark:
+  build:
     runs-on: benchmarking-runner
+    if: github.event.head_commit.message != 'Update performance results in README.md'
     env:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    strategy:
-      matrix:
-        service:
-          [
-            apollo_server,
-            caliban,
-            netflix_dgs,
-            gqlgen,
-            tailcall,
-            async_graphql,
-            hasura,
-            graphql_jit,
-          ]
     steps:
       - name: Checkout (GitHub)
         uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }}
 
-      - name: Build devcontainer and run setup and benchmark
+      - name: Build devcontainer and run benchmarks
         uses: devcontainers/[email protected]
         with:
           imageName: graphql-benchmarks
           push: never
           runCmd: |
-            bash ./graphql/${{ matrix.service }}/setup.sh
-            bash run_benchmarks.sh ${{ matrix.service }}
-
-      - name: List benchmark files
-        run: |
-          ls -la bench*.txt || echo "No matching files found"
-
-      - name: Upload benchmark results
-        uses: actions/upload-artifact@v3
-        with:
-          name: benchmark-results
-          path: bench*.txt
-
-  analyze:
-    needs: benchmark
-    runs-on: benchmarking-runner
-    steps:
-      - name: Checkout (GitHub)
-        uses: actions/checkout@v4
-
-      - name: Download all benchmark results
-        uses: actions/download-artifact@v3
-        with:
-          name: benchmark-results
-          path: .
-
-      - name: List downloaded artifacts
-        run: ls -la bench*.txt || echo "No matching files found"
-
-      - name: Analyze results
-        run: |
-          bash run_analyze_script.sh
+            bash ./setup.sh
+            bash ./run_benchmarks.sh
 
       - name: Print benchmark results
         run: cat ./results.md
@@ -90,4 +51,4 @@ jobs:
         with:
           branch: main
           commit_author: Author <[email protected]>
-          commit_message: "[ci skip] update performance results in README.md"
+          commit_message: "Update performance results in README.md"

README.md

@@ -28,7 +28,7 @@ Explore and compare the performance of the fastest GraphQL frameworks through ou
 
 ## Introduction
 
-This document presents a comparative analysis of several renowned GraphQL solutions. Dive deep into the performance metrics, and get insights into their throughput and latency.
+This document presents a comparative analysis of several renowned GraphQL frameworks. Dive deep into the performance metrics, and get insights into their throughput and latency.
 
 > **NOTE:** This is a work in progress suite of benchmarks, and we would appreciate help from the community to add more frameworks or tune the existing ones for better performance.
 
@@ -51,32 +51,32 @@ Get started with the benchmarks:
 | Query | Server | Requests/sec | Latency (ms) | Relative |
 |-------:|--------:|--------------:|--------------:|---------:|
 | 1 | `{ posts { id userId title user { id name email }}}` |
-|| [async-graphql] | `10,412.00` | `147.99` | `17.58x` |
-|| [Netflix DGS] | `9,907.41` | `175.64` | `16.72x` |
-|| [GraphQL JIT] | `9,880.01` | `242.40` | `16.68x` |
-|| [Gqlgen] | `1,448.80` | `79.74` | `2.45x` |
-|| [Hasura] | `1,268.31` | `87.46` | `2.14x` |
-|| [Apollo GraphQL] | `1,248.48` | `166.23` | `2.11x` |
-|| [Caliban] | `757.51` | `257.01` | `1.28x` |
-|| [Tailcall] | `592.42` | `227.60` | `1.00x` |
+|| [Tailcall] | `29,267.60` | `3.40` | `267.10x` |
+|| [async-graphql] | `1,740.18` | `57.69` | `15.88x` |
+|| [Caliban] | `1,569.42` | `63.36` | `14.32x` |
+|| [GraphQL JIT] | `1,336.41` | `74.51` | `12.20x` |
+|| [Gqlgen] | `747.49` | `132.68` | `6.82x` |
+|| [Netflix DGS] | `352.99` | `194.51` | `3.22x` |
+|| [Apollo GraphQL] | `266.96` | `368.47` | `2.44x` |
+|| [Hasura] | `109.58` | `549.59` | `1.00x` |
 | 2 | `{ posts { title }}` |
-|| [async-graphql] | `23,335.10` | `24.29` | `18.33x` |
-|| [Netflix DGS] | `20,600.60` | `44.12` | `16.18x` |
-|| [GraphQL JIT] | `20,451.20` | `62.80` | `16.06x` |
-|| [Gqlgen] | `7,036.34` | `23.19` | `5.53x` |
-|| [Apollo GraphQL] | `6,829.44` | `27.47` | `5.36x` |
-|| [Hasura] | `4,453.10` | `44.00` | `3.50x` |
-|| [Caliban] | `1,462.79` | `78.87` | `1.15x` |
-|| [Tailcall] | `1,273.29` | `86.31` | `1.00x` |
+|| [Tailcall] | `58,892.60` | `1.69` | `67.80x` |
+|| [Caliban] | `9,095.25` | `11.36` | `10.47x` |
+|| [async-graphql] | `9,013.56` | `11.22` | `10.38x` |
+|| [Gqlgen] | `2,150.46` | `48.27` | `2.48x` |
+|| [Apollo GraphQL] | `1,790.51` | `55.78` | `2.06x` |
+|| [Netflix DGS] | `1,577.33` | `70.02` | `1.82x` |
+|| [GraphQL JIT] | `1,355.90` | `73.65` | `1.56x` |
+|| [Hasura] | `868.64` | `114.88` | `1.00x` |
 | 3 | `{ greet }` |
-|| [Gqlgen] | `53,539.90` | `2.78` | `10.09x` |
-|| [Apollo GraphQL] | `40,331.30` | `5.49` | `7.60x` |
-|| [Hasura] | `39,528.60` | `8.54` | `7.45x` |
-|| [async-graphql] | `38,009.10` | `5.71` | `7.16x` |
-|| [Netflix DGS] | `25,211.90` | `9.83` | `4.75x` |
-|| [GraphQL JIT] | `23,492.70` | `18.38` | `4.43x` |
-|| [Caliban] | `18,230.80` | `20.44` | `3.43x` |
-|| [Tailcall] | `5,307.93` | `24.38` | `1.00x` |
+|| [Caliban] | `67,030.40` | `1.08` | `25.98x` |
+|| [Tailcall] | `58,359.30` | `1.73` | `22.61x` |
+|| [async-graphql] | `47,894.60` | `2.12` | `18.56x` |
+|| [Gqlgen] | `47,153.70` | `5.18` | `18.27x` |
+|| [Netflix DGS] | `8,188.33` | `15.04` | `3.17x` |
+|| [Apollo GraphQL] | `8,021.30` | `12.65` | `3.11x` |
+|| [GraphQL JIT] | `5,168.90` | `19.32` | `2.00x` |
+|| [Hasura] | `2,580.56` | `38.79` | `1.00x` |
 
 <!-- PERFORMANCE_RESULTS_END -->
 

analyze.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-# Install gnuplot
-sudo apt-get update && sudo apt-get install -y gnuplot
-
 function extractMetric() {
   local file="$1"
   local metric="$2"
@@ -164,4 +161,4 @@ mv $latencyHistogramFile assets/
 # Delete the result TXT files
 for file in "${resultFiles[@]}"; do
   rm "$file"
-done
+done

results.md

@@ -3,31 +3,31 @@
 | Query | Server | Requests/sec | Latency (ms) | Relative |
 |-------:|--------:|--------------:|--------------:|---------:|
 | 1 | `{ posts { id userId title user { id name email }}}` |
-|| [async-graphql] | `10,412.00` | `147.99` | `17.58x` |
-|| [Netflix DGS] | `9,907.41` | `175.64` | `16.72x` |
-|| [GraphQL JIT] | `9,880.01` | `242.40` | `16.68x` |
-|| [Gqlgen] | `1,448.80` | `79.74` | `2.45x` |
-|| [Hasura] | `1,268.31` | `87.46` | `2.14x` |
-|| [Apollo GraphQL] | `1,248.48` | `166.23` | `2.11x` |
-|| [Caliban] | `757.51` | `257.01` | `1.28x` |
-|| [Tailcall] | `592.42` | `227.60` | `1.00x` |
+|| [Tailcall] | `29,267.60` | `3.40` | `267.10x` |
+|| [async-graphql] | `1,740.18` | `57.69` | `15.88x` |
+|| [Caliban] | `1,569.42` | `63.36` | `14.32x` |
+|| [GraphQL JIT] | `1,336.41` | `74.51` | `12.20x` |
+|| [Gqlgen] | `747.49` | `132.68` | `6.82x` |
+|| [Netflix DGS] | `352.99` | `194.51` | `3.22x` |
+|| [Apollo GraphQL] | `266.96` | `368.47` | `2.44x` |
+|| [Hasura] | `109.58` | `549.59` | `1.00x` |
 | 2 | `{ posts { title }}` |
-|| [async-graphql] | `23,335.10` | `24.29` | `18.33x` |
-|| [Netflix DGS] | `20,600.60` | `44.12` | `16.18x` |
-|| [GraphQL JIT] | `20,451.20` | `62.80` | `16.06x` |
-|| [Gqlgen] | `7,036.34` | `23.19` | `5.53x` |
-|| [Apollo GraphQL] | `6,829.44` | `27.47` | `5.36x` |
-|| [Hasura] | `4,453.10` | `44.00` | `3.50x` |
-|| [Caliban] | `1,462.79` | `78.87` | `1.15x` |
-|| [Tailcall] | `1,273.29` | `86.31` | `1.00x` |
+|| [Tailcall] | `58,892.60` | `1.69` | `67.80x` |
+|| [Caliban] | `9,095.25` | `11.36` | `10.47x` |
+|| [async-graphql] | `9,013.56` | `11.22` | `10.38x` |
+|| [Gqlgen] | `2,150.46` | `48.27` | `2.48x` |
+|| [Apollo GraphQL] | `1,790.51` | `55.78` | `2.06x` |
+|| [Netflix DGS] | `1,577.33` | `70.02` | `1.82x` |
+|| [GraphQL JIT] | `1,355.90` | `73.65` | `1.56x` |
+|| [Hasura] | `868.64` | `114.88` | `1.00x` |
 | 3 | `{ greet }` |
-|| [Gqlgen] | `53,539.90` | `2.78` | `10.09x` |
-|| [Apollo GraphQL] | `40,331.30` | `5.49` | `7.60x` |
-|| [Hasura] | `39,528.60` | `8.54` | `7.45x` |
-|| [async-graphql] | `38,009.10` | `5.71` | `7.16x` |
-|| [Netflix DGS] | `25,211.90` | `9.83` | `4.75x` |
-|| [GraphQL JIT] | `23,492.70` | `18.38` | `4.43x` |
-|| [Caliban] | `18,230.80` | `20.44` | `3.43x` |
-|| [Tailcall] | `5,307.93` | `24.38` | `1.00x` |
+|| [Caliban] | `67,030.40` | `1.08` | `25.98x` |
+|| [Tailcall] | `58,359.30` | `1.73` | `22.61x` |
+|| [async-graphql] | `47,894.60` | `2.12` | `18.56x` |
+|| [Gqlgen] | `47,153.70` | `5.18` | `18.27x` |
+|| [Netflix DGS] | `8,188.33` | `15.04` | `3.17x` |
+|| [Apollo GraphQL] | `8,021.30` | `12.65` | `3.11x` |
+|| [GraphQL JIT] | `5,168.90` | `19.32` | `2.00x` |
+|| [Hasura] | `2,580.56` | `38.79` | `1.00x` |
 
 <!-- PERFORMANCE_RESULTS_END -->

run_benchmarks.sh

@@ -12,7 +12,6 @@ function killServerOnPort() {
     echo "No process found running on port $port"
   fi
 }
-
 bench1Results=()
 bench2Results=()
 bench3Results=()
@@ -73,29 +72,17 @@ function runBenchmark() {
 
 rm "results.md"
 
-# Main script
-if [ $# -eq 0 ]; then
-    echo "Usage: $0 <service_name>"
-    echo "Available services: apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit"
-    exit 1
-fi  
-
-service="$1"
-valid_services=("apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit")
-
-if [[ ! " ${valid_services[@]} " =~ " ${service} " ]]; then
-    echo "Invalid service name. Available services: ${valid_services[*]}"
-    exit 1
-fi
-
-
-
-runBenchmark "graphql/${service}/run.sh"
-
-if [ "$service" == "apollo_server" ]; then
+for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit"; do
+  runBenchmark "graphql/${service}/run.sh"
+  if [ "$service" == "apollo_server" ]; then
     cd graphql/apollo_server/
     npm stop
     cd ../../
-elif [ "$service" == "hasura" ]; then
+  elif [ "$service" == "hasura" ]; then
     bash "graphql/hasura/kill.sh"
-fi
+  fi
+done
+
+bash analyze.sh "${bench1Results[@]}"
+bash analyze.sh "${bench2Results[@]}"
+bash analyze.sh "${bench3Results[@]}"