Update Benchmark set up and persist benchmarks

We have recently updated our toolchain version to 5.9, but have never
updated the benchmark suite to use 5.9 or newer. Also, it does not look
like we ever commited a baseline to this repository.

While at it, wiring up GitHub actions to run the benchmarks for every
pull request, as well as on a merge into `main`. Re-uses a lot of the
heavy lifting that was defined in `apple/swift-nio` already.

Note that `nightly-main` appears broken at the moment. The Benchmark
tool gets stuck, and initial debugging point to it never finishing
`_sendAndAcknowledgeMessages` when producing a set of messages before
every benchmark.

.github/workflows/benchmarks.yml

@@ -0,0 +1,46 @@
+name: Benchmarks
+
+on:
+  workflow_call:
+    inputs:
+      benchmark_package_path:
+        type: string
+        description: "Path to the directory containing the benchmarking package. Defaults to ."
+        default: "."
+      swift_package_arguments:
+        type: string
+        description: "Arguments to the switch package command invocation e.g. `--disable-sandbox`"
+      linux_5_9_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 5.9 Swift version matrix job. Defaults to true."
+        default: true
+      linux_5_10_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 5.10 Swift version matrix job. Defaults to true."
+        default: true
+      linux_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      linux_nightly_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux nightly 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      linux_nightly_main_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux nightly main Swift version matrix job. Defaults to true."
+        default: true
+
+jobs:
+  benchmarks:
+    name: Benchmarks
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/swift_matrix.yml
+    with:
+      name: "Benchmarks"
+      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q libjemalloc-dev && apt-get -y install libsasl2-dev && swift package --package-path ${{ inputs.benchmark_package_path }} ${{ inputs.swift_package_arguments }} benchmark baseline check --check-absolute-path ${{ inputs.benchmark_package_path }}/Thresholds/${SWIFT_VERSION}/"
+      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
+      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
+      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
+      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
+      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}

.github/workflows/cxx_interop.yml

@@ -0,0 +1,39 @@
+name: Cxx interop
+
+on:
+  workflow_call:
+    inputs:
+      linux_5_9_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 5.9 Swift version matrix job. Defaults to true."
+        default: true
+      linux_5_10_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 5.10 Swift version matrix job. Defaults to true."
+        default: true
+      linux_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      linux_nightly_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux nightly 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      linux_nightly_main_enabled:
+        type: boolean
+        description: "Boolean to enable the Linux nightly main Swift version matrix job. Defaults to true."
+        default: true
+
+jobs:
+  cxx-interop:
+    name: Cxx interop
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/swift_matrix.yml
+    with:
+      name: "Cxx interop"
+      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q jq && apt-get -y install libsasl2-dev && curl -s  https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-cxx-interop-compatibility.sh | bash"
+      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
+      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
+      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
+      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
+      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}

.github/workflows/main.yml

@@ -9,10 +9,23 @@ on:
 jobs:
   unit-tests:
     name: Unit tests
+    # Workaround https://github.com/nektos/act/issues/1875
     uses: ./.github/workflows/unit_tests.yml
     with:
       linux_5_9_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -strict-concurrency=complete"
       linux_5_10_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -strict-concurrency=complete"
       linux_6_0_arguments_override: "--explicit-target-dependency-import-check error"
       linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error"
       linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error"
+
+  benchmarks:
+    name: Benchmarks
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/benchmarks.yml@main
+    with:
+      benchmark_package_path: "Benchmarks"
+
+  cxx-interop:
+    name: Cxx interop
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/cxx_interop.yml

.github/workflows/pull_request.yml

@@ -23,17 +23,14 @@ jobs:
       linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error"
       linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error"
 
+  benchmarks:
+    name: Benchmarks
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/benchmarks.yml
+    with:
+      benchmark_package_path: "Benchmarks"
+
   cxx-interop:
     name: Cxx interop
-    uses: apple/swift-nio/.github/workflows/swift_matrix.yml@main
-    with:
-      name: "Cxx interop"
-      matrix_linux_command: "apt-get update -y -q && apt-get install -y -q jq && apt-get -y install libsasl2-dev && curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-cxx-interop-compatibility.sh | bash"
-      matrix_linux_5_9_enabled: true
-      matrix_linux_5_10_enabled: true
-      matrix_linux_6_0_enabled: true
-      matrix_linux_nightly_6_0_enabled: true
-      matrix_linux_nightly_main_enabled: true
-      matrix_windows_6_0_enabled: false
-      matrix_windows_nightly_6_0_enabled: false
-      matrix_windows_nightly_main_enabled: false
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/cxx_interop.yml

.github/workflows/swift_matrix.yml

@@ -0,0 +1,133 @@
+name: Matrix
+
+on:
+  workflow_call:
+    inputs:
+      name:
+        type: string
+        description: "The name of the workflow used for the concurrency group."
+        required: true
+      matrix_linux_command:
+        type: string
+        description: "The command of the current Swift version linux matrix job to execute."
+        required: true
+      matrix_linux_5_9_enabled:
+        type: boolean
+        description: "Boolean to enable the 5.9 Swift version matrix job. Defaults to true."
+        default: true
+      matrix_linux_5_9_container_image:
+        type: string
+        description: "Container image for the 5.9 Swift version matrix job. Defaults to matching Swift Ubuntu image."
+        default: "swift:5.9-jammy"
+      matrix_linux_5_9_command_override:
+        type: string
+        description: "The command of the 5.9 Swift version linux matrix job to execute."
+      matrix_linux_5_10_enabled:
+        type: boolean
+        description: "Boolean to enable the 5.10 Swift version matrix job. Defaults to true."
+        default: true
+      matrix_linux_5_10_container_image:
+        type: string
+        description: "Container image for the 5.10 Swift version matrix job. Defaults to matching Swift Ubuntu image."
+        default: "swift:5.10-jammy"
+      matrix_linux_5_10_command_override:
+        type: string
+        description: "The command of the 5.10 Swift version linux matrix job to execute."
+      matrix_linux_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      matrix_linux_6_0_container_image:
+        type: string
+        description: "Container image for the 6.0 Swift version matrix job. Defaults to matching Swift Ubuntu image."
+        default: "swift:6.0-jammy"
+      matrix_linux_6_0_command_override:
+        type: string
+        description: "The command of the 6.0 Swift version linux matrix job to execute."
+      matrix_linux_nightly_6_0_enabled:
+        type: boolean
+        description: "Boolean to enable the nightly 6.0 Swift version matrix job. Defaults to true."
+        default: true
+      matrix_linux_nightly_6_0_container_image:
+        type: string
+        description: "Container image for the nightly 6.0 Swift version matrix job. Defaults to matching Swift Ubuntu image."
+        default: "swiftlang/swift:nightly-6.0-jammy"
+      matrix_linux_nightly_6_0_command_override:
+        type: string
+        description: "The command of the nightly 6.0 Swift version linux matrix job to execute."
+      matrix_linux_nightly_main_enabled:
+        type: boolean
+        description: "Boolean to enable the nightly main Swift version matrix job. Defaults to true."
+        default: true
+      matrix_linux_nightly_main_container_image:
+        type: string
+        description: "Container image for the nightly main Swift version matrix job. Defaults to matching Swift Ubuntu image."
+        default: "swiftlang/swift:nightly-main-jammy"
+      matrix_linux_nightly_main_command_override:
+        type: string
+        description: "The command of the nightly main Swift version linux matrix job to execute."
+
+# We are cancelling previously triggered workflow runs
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.name }}
+  cancel-in-progress: true
+
+jobs:
+  linux:
+    name: Linux (${{ matrix.swift.swift_version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        # We are specifying only the major and minor of the docker images to automatically pick up the latest patch release
+        swift:
+          - image: ${{ inputs.matrix_linux_5_9_container_image }}
+            swift_version: "5.9"
+            enabled: ${{ inputs.matrix_linux_5_9_enabled }}
+          - image: ${{ inputs.matrix_linux_5_10_container_image }}
+            swift_version: "5.10"
+            enabled: ${{ inputs.matrix_linux_5_10_enabled }}
+          - image: ${{ inputs.matrix_linux_6_0_container_image }}
+            swift_version: "6.0"
+            enabled: ${{ inputs.matrix_linux_6_0_enabled }}
+          - image: ${{ inputs.matrix_linux_nightly_6_0_container_image }}
+            swift_version: "nightly-6.0"
+            enabled: ${{ inputs.matrix_linux_nightly_6_0_enabled }}
+          - image: ${{ inputs.matrix_linux_nightly_main_container_image }}
+            swift_version: "nightly-main"
+            enabled: ${{ inputs.matrix_linux_nightly_main_enabled }}
+    container:
+      image: ${{ matrix.swift.image }}
+    steps:
+      - name: Checkout repository
+        if: ${{ matrix.swift.enabled }}
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+          submodules: true
+      - name: Mark the workspace as safe
+        if: ${{ matrix.swift.enabled }}
+        # https://github.com/actions/checkout/issues/766
+        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
+      - name: Run matrix job
+        if: ${{ matrix.swift.enabled }}
+        env:
+          SWIFT_VERSION: ${{ matrix.swift.swift_version }}
+          COMMAND: ${{ inputs.matrix_linux_command }}
+          COMMAND_OVERRIDE_5_9: ${{ inputs.matrix_linux_5_9_command_override }}
+          COMMAND_OVERRIDE_5_10: ${{ inputs.matrix_linux_5_10_command_override }}
+          COMMAND_OVERRIDE_6_0: ${{ inputs.matrix_linux_6_0_command_override }}
+          COMMAND_OVERRIDE_NIGHTLY_6_0: ${{ inputs.matrix_linux_nightly_6_0_command_override }}
+          COMMAND_OVERRIDE_NIGHTLY_MAIN: ${{ inputs.matrix_linux_nightly_main_command_override }}
+        run: |
+          apt-get -qq update && apt-get -qq -y install curl && apt-get -y install libsasl2-dev
+          curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.sh | bash
+    services:
+      zookeeper:
+        image: ubuntu/zookeeper
+      kafka:
+        image: ubuntu/kafka
+        env:
+          ZOOKEEPER_HOST: zookeeper
+    env:
+      KAFKA_HOST: kafka

.github/workflows/unit_tests.yml

@@ -47,59 +47,18 @@ on:
 jobs:
   unit-tests:
     name: Unit tests
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        # We are specifying only the major and minor of the docker images to automatically pick up the latest patch release
-        swift:
-          - image: "swift:5.9-jammy"
-            swift_version: "5.9"
-            enabled: ${{ inputs.linux_5_9_enabled }}
-          - image: "swift:5.10-jammy"
-            swift_version: "5.10"
-            enabled: ${{ inputs.linux_5_10_enabled }}
-          - image: "swift:6.0-jammy"
-            swift_version: "6.0"
-            enabled: ${{ inputs.linux_6_0_enabled }}
-          - image: "swiftlang/swift:nightly-6.0-jammy"
-            swift_version: "nightly-6.0"
-            enabled: ${{ inputs.linux_nightly_6_0_enabled }}
-          - image: "swiftlang/swift:nightly-main-jammy"
-            swift_version: "nightly-main"
-            enabled: ${{ inputs.linux_nightly_main_enabled }}
-    steps:
-      - name: Checkout repository
-        if: ${{ matrix.swift.enabled }}
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-          submodules: true
-      - name: Mark the workspace as safe
-        if: ${{ matrix.swift.enabled }}
-        # https://github.com/actions/checkout/issues/766
-        run: git config --global --add safe.directory ${GITHUB_WORKSPACE}
-      - name: Run matrix job
-        if: ${{ matrix.swift.enabled }}
-        env:
-          SWIFT_VERSION: ${{ matrix.swift.swift_version }}
-          COMMAND: "swift test"
-          COMMAND_OVERRIDE_5_9: "swift test ${{ inputs.linux_5_9_arguments_override }}"
-          COMMAND_OVERRIDE_5_10: "swift test ${{ inputs.linux_5_10_arguments_override }}"
-          COMMAND_OVERRIDE_6_0: "swift test ${{ inputs.linux_6_0_arguments_override }}"
-          COMMAND_OVERRIDE_NIGHTLY_6_0: "swift test ${{ inputs.linux_nightly_6_0_arguments_override }}"
-          COMMAND_OVERRIDE_NIGHTLY_MAIN: "swift test ${{ inputs.linux_nightly_main_arguments_override }}"
-        run: |
-          apt-get -qq update && apt-get -qq -y install curl && apt-get -y install libsasl2-dev
-          curl -s https://raw.githubusercontent.com/apple/swift-nio/main/scripts/check-matrix-job.sh | bash
-    container:
-      image: ${{ matrix.swift.image }}
-    services:
-      zookeeper:
-        image: ubuntu/zookeeper
-      kafka:
-        image: ubuntu/kafka
-        env:
-          ZOOKEEPER_HOST: zookeeper
-    env:
-      KAFKA_HOST: kafka
+    # Workaround https://github.com/nektos/act/issues/1875
+    uses: ./.github/workflows/swift_matrix.yml
+    with:
+      name: "Unit tests"
+      matrix_linux_command: "swift test"
+      matrix_linux_5_9_enabled: ${{ inputs.linux_5_9_enabled }}
+      matrix_linux_5_9_command_override: "swift test ${{ inputs.linux_5_9_arguments_override }}"
+      matrix_linux_5_10_enabled: ${{ inputs.linux_5_10_enabled }}
+      matrix_linux_5_10_command_override: "swift test ${{ inputs.linux_5_10_arguments_override }}"
+      matrix_linux_6_0_enabled: ${{ inputs.linux_6_0_enabled }}
+      matrix_linux_6_0_command_override: "swift test ${{ inputs.linux_6_0_arguments_override }}"
+      matrix_linux_nightly_6_0_enabled: ${{ inputs.linux_nightly_6_0_enabled }}
+      matrix_linux_nightly_6_0_command_override: "swift test ${{ inputs.linux_nightly_6_0_arguments_override }}"
+      matrix_linux_nightly_main_enabled: ${{ inputs.linux_nightly_main_enabled }}
+      matrix_linux_nightly_main_command_override: "swift test ${{ inputs.linux_nightly_main_arguments_override }}"

Benchmarks/Package.swift

@@ -1,4 +1,4 @@
-// swift-tools-version: 5.7
+// swift-tools-version: 5.9
 //===----------------------------------------------------------------------===//
 //
 // This source file is part of the swift-kafka-client open source project

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_basic_consumer_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 77266944,
+  "cpuTotal" : 200000000,
+  "objectAllocCount" : 5549,
+  "releaseCount" : 15168,
+  "retainCount" : 7108,
+  "retainReleaseDelta" : 2511,
+  "throughput" : 2,
+  "wallClock" : 695307500
+}

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.SwiftKafkaConsumer_with_offset_commit_messages_1000.p90.json

@@ -0,0 +1,3 @@
+{
+
+}

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_basic_consumer_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 47382528,
+  "cpuTotal" : 10000000,
+  "objectAllocCount" : 16,
+  "releaseCount" : 48,
+  "retainCount" : 2,
+  "retainReleaseDelta" : 30,
+  "throughput" : 2,
+  "wallClock" : 640572501
+}

Benchmarks/Thresholds/5.10/SwiftKafkaConsumerBenchmarks.librdkafka_with_offset_commit_messages_1000.p90.json

@@ -0,0 +1,10 @@
+{
+  "allocatedResidentMemory" : 44204031,
+  "cpuTotal" : 80000000,
+  "objectAllocCount" : 16,
+  "releaseCount" : 48,
+  "retainCount" : 2,
+  "retainReleaseDelta" : 30,
+  "throughput" : 3,
+  "wallClock" : 320339967
+}