diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index e77549c8..fa4bc97a 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,32 +1,23 @@ name: "Run benchmark" + on: pull_request_target: types: [assigned, opened, synchronize, reopened, edited] push: branches: - main + permissions: contents: write pull-requests: write issues: write + jobs: - benchmark: + build: runs-on: benchmarking-runner + if: github.event.head_commit.message != 'Update performance results in README.md' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - strategy: - matrix: - service: - [ - apollo_server, - caliban, - netflix_dgs, - gqlgen, - tailcall, - async_graphql, - hasura, - graphql_jit, - ] steps: - name: Checkout (GitHub) uses: actions/checkout@v4 @@ -34,44 +25,14 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} - - name: Build devcontainer and run setup and benchmark + - name: Build devcontainer and run benchmarks uses: devcontainers/ci@v0.3 with: imageName: graphql-benchmarks push: never runCmd: | - bash ./graphql/${{ matrix.service }}/setup.sh - bash run_benchmarks.sh ${{ matrix.service }} - - - name: List benchmark files - run: | - ls -la bench*.txt || echo "No matching files found" - - - name: Upload benchmark results - uses: actions/upload-artifact@v3 - with: - name: benchmark-results - path: bench*.txt - - analyze: - needs: benchmark - runs-on: benchmarking-runner - steps: - - name: Checkout (GitHub) - uses: actions/checkout@v4 - - - name: Download all benchmark results - uses: actions/download-artifact@v3 - with: - name: benchmark-results - path: . - - - name: List downloaded artifacts - run: ls -la bench*.txt || echo "No matching files found" - - - name: Analyze results - run: | - bash run_analyze_script.sh + bash ./setup.sh + bash ./run_benchmarks.sh - name: Print benchmark results run: cat ./results.md @@ -90,4 +51,4 @@ jobs: with: branch: main commit_author: Author - commit_message: "[ci skip] update performance results in README.md" + commit_message: "Update performance results in README.md" diff --git a/README.md b/README.md index d26d7a7f..b86b4aec 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Explore and compare the performance of the fastest GraphQL frameworks through ou ## Introduction -This document presents a comparative analysis of several renowned GraphQL solutions. Dive deep into the performance metrics, and get insights into their throughput and latency. +This document presents a comparative analysis of several renowned GraphQL frameworks. Dive deep into the performance metrics, and get insights into their throughput and latency. > **NOTE:** This is a work in progress suite of benchmarks, and we would appreciate help from the community to add more frameworks or tune the existing ones for better performance. @@ -51,32 +51,32 @@ Get started with the benchmarks: | Query | Server | Requests/sec | Latency (ms) | Relative | |-------:|--------:|--------------:|--------------:|---------:| | 1 | `{ posts { id userId title user { id name email }}}` | -|| [async-graphql] | `10,412.00` | `147.99` | `17.58x` | -|| [Netflix DGS] | `9,907.41` | `175.64` | `16.72x` | -|| [GraphQL JIT] | `9,880.01` | `242.40` | `16.68x` | -|| [Gqlgen] | `1,448.80` | `79.74` | `2.45x` | -|| [Hasura] | `1,268.31` | `87.46` | `2.14x` | -|| [Apollo GraphQL] | `1,248.48` | `166.23` | `2.11x` | -|| [Caliban] | `757.51` | `257.01` | `1.28x` | -|| [Tailcall] | `592.42` | `227.60` | `1.00x` | +|| [Tailcall] | `29,267.60` | `3.40` | `267.10x` | +|| [async-graphql] | `1,740.18` | `57.69` | `15.88x` | +|| [Caliban] | `1,569.42` | `63.36` | `14.32x` | +|| [GraphQL JIT] | `1,336.41` | `74.51` | `12.20x` | +|| [Gqlgen] | `747.49` | `132.68` | `6.82x` | +|| [Netflix DGS] | `352.99` | `194.51` | `3.22x` | +|| [Apollo GraphQL] | `266.96` | `368.47` | `2.44x` | +|| [Hasura] | `109.58` | `549.59` | `1.00x` | | 2 | `{ posts { title }}` | -|| [async-graphql] | `23,335.10` | `24.29` | `18.33x` | -|| [Netflix DGS] | `20,600.60` | `44.12` | `16.18x` | -|| [GraphQL JIT] | `20,451.20` | `62.80` | `16.06x` | -|| [Gqlgen] | `7,036.34` | `23.19` | `5.53x` | -|| [Apollo GraphQL] | `6,829.44` | `27.47` | `5.36x` | -|| [Hasura] | `4,453.10` | `44.00` | `3.50x` | -|| [Caliban] | `1,462.79` | `78.87` | `1.15x` | -|| [Tailcall] | `1,273.29` | `86.31` | `1.00x` | +|| [Tailcall] | `58,892.60` | `1.69` | `67.80x` | +|| [Caliban] | `9,095.25` | `11.36` | `10.47x` | +|| [async-graphql] | `9,013.56` | `11.22` | `10.38x` | +|| [Gqlgen] | `2,150.46` | `48.27` | `2.48x` | +|| [Apollo GraphQL] | `1,790.51` | `55.78` | `2.06x` | +|| [Netflix DGS] | `1,577.33` | `70.02` | `1.82x` | +|| [GraphQL JIT] | `1,355.90` | `73.65` | `1.56x` | +|| [Hasura] | `868.64` | `114.88` | `1.00x` | | 3 | `{ greet }` | -|| [Gqlgen] | `53,539.90` | `2.78` | `10.09x` | -|| [Apollo GraphQL] | `40,331.30` | `5.49` | `7.60x` | -|| [Hasura] | `39,528.60` | `8.54` | `7.45x` | -|| [async-graphql] | `38,009.10` | `5.71` | `7.16x` | -|| [Netflix DGS] | `25,211.90` | `9.83` | `4.75x` | -|| [GraphQL JIT] | `23,492.70` | `18.38` | `4.43x` | -|| [Caliban] | `18,230.80` | `20.44` | `3.43x` | -|| [Tailcall] | `5,307.93` | `24.38` | `1.00x` | +|| [Caliban] | `67,030.40` | `1.08` | `25.98x` | +|| [Tailcall] | `58,359.30` | `1.73` | `22.61x` | +|| [async-graphql] | `47,894.60` | `2.12` | `18.56x` | +|| [Gqlgen] | `47,153.70` | `5.18` | `18.27x` | +|| [Netflix DGS] | `8,188.33` | `15.04` | `3.17x` | +|| [Apollo GraphQL] | `8,021.30` | `12.65` | `3.11x` | +|| [GraphQL JIT] | `5,168.90` | `19.32` | `2.00x` | +|| [Hasura] | `2,580.56` | `38.79` | `1.00x` | diff --git a/analyze.sh b/analyze.sh index a52e8471..17bbbce1 100755 --- a/analyze.sh +++ b/analyze.sh @@ -1,8 +1,5 @@ #!/bin/bash -# Install gnuplot -sudo apt-get update && sudo apt-get install -y gnuplot - function extractMetric() { local file="$1" local metric="$2" @@ -164,4 +161,4 @@ mv $latencyHistogramFile assets/ # Delete the result TXT files for file in "${resultFiles[@]}"; do rm "$file" -done \ No newline at end of file +done diff --git a/assets/latency_histogram1.png b/assets/latency_histogram1.png index bbc179c9..027197d4 100644 Binary files a/assets/latency_histogram1.png and b/assets/latency_histogram1.png differ diff --git a/assets/latency_histogram2.png b/assets/latency_histogram2.png index f4f859bf..975a8be0 100644 Binary files a/assets/latency_histogram2.png and b/assets/latency_histogram2.png differ diff --git a/assets/latency_histogram3.png b/assets/latency_histogram3.png index 0516c4cc..bd5b4411 100644 Binary files a/assets/latency_histogram3.png and b/assets/latency_histogram3.png differ diff --git a/assets/req_sec_histogram1.png b/assets/req_sec_histogram1.png index 83e80c85..a6e161fe 100644 Binary files a/assets/req_sec_histogram1.png and b/assets/req_sec_histogram1.png differ diff --git a/assets/req_sec_histogram2.png b/assets/req_sec_histogram2.png index b7be5ae0..f4f6b831 100644 Binary files a/assets/req_sec_histogram2.png and b/assets/req_sec_histogram2.png differ diff --git a/assets/req_sec_histogram3.png b/assets/req_sec_histogram3.png index fec26217..ed84ffcb 100644 Binary files a/assets/req_sec_histogram3.png and b/assets/req_sec_histogram3.png differ diff --git a/results.md b/results.md index 55561364..dc6582c8 100644 --- a/results.md +++ b/results.md @@ -3,31 +3,31 @@ | Query | Server | Requests/sec | Latency (ms) | Relative | |-------:|--------:|--------------:|--------------:|---------:| | 1 | `{ posts { id userId title user { id name email }}}` | -|| [async-graphql] | `10,412.00` | `147.99` | `17.58x` | -|| [Netflix DGS] | `9,907.41` | `175.64` | `16.72x` | -|| [GraphQL JIT] | `9,880.01` | `242.40` | `16.68x` | -|| [Gqlgen] | `1,448.80` | `79.74` | `2.45x` | -|| [Hasura] | `1,268.31` | `87.46` | `2.14x` | -|| [Apollo GraphQL] | `1,248.48` | `166.23` | `2.11x` | -|| [Caliban] | `757.51` | `257.01` | `1.28x` | -|| [Tailcall] | `592.42` | `227.60` | `1.00x` | +|| [Tailcall] | `29,267.60` | `3.40` | `267.10x` | +|| [async-graphql] | `1,740.18` | `57.69` | `15.88x` | +|| [Caliban] | `1,569.42` | `63.36` | `14.32x` | +|| [GraphQL JIT] | `1,336.41` | `74.51` | `12.20x` | +|| [Gqlgen] | `747.49` | `132.68` | `6.82x` | +|| [Netflix DGS] | `352.99` | `194.51` | `3.22x` | +|| [Apollo GraphQL] | `266.96` | `368.47` | `2.44x` | +|| [Hasura] | `109.58` | `549.59` | `1.00x` | | 2 | `{ posts { title }}` | -|| [async-graphql] | `23,335.10` | `24.29` | `18.33x` | -|| [Netflix DGS] | `20,600.60` | `44.12` | `16.18x` | -|| [GraphQL JIT] | `20,451.20` | `62.80` | `16.06x` | -|| [Gqlgen] | `7,036.34` | `23.19` | `5.53x` | -|| [Apollo GraphQL] | `6,829.44` | `27.47` | `5.36x` | -|| [Hasura] | `4,453.10` | `44.00` | `3.50x` | -|| [Caliban] | `1,462.79` | `78.87` | `1.15x` | -|| [Tailcall] | `1,273.29` | `86.31` | `1.00x` | +|| [Tailcall] | `58,892.60` | `1.69` | `67.80x` | +|| [Caliban] | `9,095.25` | `11.36` | `10.47x` | +|| [async-graphql] | `9,013.56` | `11.22` | `10.38x` | +|| [Gqlgen] | `2,150.46` | `48.27` | `2.48x` | +|| [Apollo GraphQL] | `1,790.51` | `55.78` | `2.06x` | +|| [Netflix DGS] | `1,577.33` | `70.02` | `1.82x` | +|| [GraphQL JIT] | `1,355.90` | `73.65` | `1.56x` | +|| [Hasura] | `868.64` | `114.88` | `1.00x` | | 3 | `{ greet }` | -|| [Gqlgen] | `53,539.90` | `2.78` | `10.09x` | -|| [Apollo GraphQL] | `40,331.30` | `5.49` | `7.60x` | -|| [Hasura] | `39,528.60` | `8.54` | `7.45x` | -|| [async-graphql] | `38,009.10` | `5.71` | `7.16x` | -|| [Netflix DGS] | `25,211.90` | `9.83` | `4.75x` | -|| [GraphQL JIT] | `23,492.70` | `18.38` | `4.43x` | -|| [Caliban] | `18,230.80` | `20.44` | `3.43x` | -|| [Tailcall] | `5,307.93` | `24.38` | `1.00x` | +|| [Caliban] | `67,030.40` | `1.08` | `25.98x` | +|| [Tailcall] | `58,359.30` | `1.73` | `22.61x` | +|| [async-graphql] | `47,894.60` | `2.12` | `18.56x` | +|| [Gqlgen] | `47,153.70` | `5.18` | `18.27x` | +|| [Netflix DGS] | `8,188.33` | `15.04` | `3.17x` | +|| [Apollo GraphQL] | `8,021.30` | `12.65` | `3.11x` | +|| [GraphQL JIT] | `5,168.90` | `19.32` | `2.00x` | +|| [Hasura] | `2,580.56` | `38.79` | `1.00x` | diff --git a/run_benchmarks.sh b/run_benchmarks.sh index 9c1b1aa8..37280277 100755 --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -12,7 +12,6 @@ function killServerOnPort() { echo "No process found running on port $port" fi } - bench1Results=() bench2Results=() bench3Results=() @@ -73,29 +72,17 @@ function runBenchmark() { rm "results.md" -# Main script -if [ $# -eq 0 ]; then - echo "Usage: $0 " - echo "Available services: apollo_server, caliban, netflix_dgs, gqlgen, tailcall, async_graphql, hasura, graphql_jit" - exit 1 -fi - -service="$1" -valid_services=("apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit") - -if [[ ! " ${valid_services[@]} " =~ " ${service} " ]]; then - echo "Invalid service name. Available services: ${valid_services[*]}" - exit 1 -fi - - - -runBenchmark "graphql/${service}/run.sh" - -if [ "$service" == "apollo_server" ]; then +for service in "apollo_server" "caliban" "netflix_dgs" "gqlgen" "tailcall" "async_graphql" "hasura" "graphql_jit"; do + runBenchmark "graphql/${service}/run.sh" + if [ "$service" == "apollo_server" ]; then cd graphql/apollo_server/ npm stop cd ../../ -elif [ "$service" == "hasura" ]; then + elif [ "$service" == "hasura" ]; then bash "graphql/hasura/kill.sh" -fi \ No newline at end of file + fi +done + +bash analyze.sh "${bench1Results[@]}" +bash analyze.sh "${bench2Results[@]}" +bash analyze.sh "${bench3Results[@]}"