diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml new file mode 100644 index 0000000..5893d44 --- /dev/null +++ b/.github/workflows/benchmarks.yaml @@ -0,0 +1,94 @@ +name: Benchmark PR vs main + +on: + workflow_dispatch: + pull_request: + branches: + - main + paths: + - '**.swift' + - '**.yml' + +jobs: + benchmark-delta: + + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + continue-on-error: true + + strategy: + matrix: + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Homebrew Mac + if: ${{ runner.os == 'Macos' }} + run: | + echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH + brew install jemalloc + + - name: Ubuntu deps + if: ${{ runner.os == 'Linux' }} + run: | + sudo apt-get install -y libjemalloc-dev + + - name: Git URL token override and misc + run: | + #git config --global url."https://ordo-ci:${{ secrets.CI_MACHINE_PAT }}@github.com".insteadOf "https://github.com" + #/usr/bin/ordo-performance + [ -d Benchmarks ] && echo "hasBenchmark=1" >> $GITHUB_ENV + echo "/opt/homebrew/bin:/usr/local/bin" >> $GITHUB_PATH + - name: Run benchmarks for PR branch + if: ${{ env.hasBenchmark == '1' }} + run: | + cd Benchmarks + swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request + - name: Switch to branch 'main' + if: ${{ env.hasBenchmark == '1' }} + run: | + git stash + git checkout main + - name: Run benchmarks for branch 'main' + if: ${{ env.hasBenchmark == '1' }} + run: | + cd Benchmarks + swift package --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update main + - name: Compare PR and main + if: ${{ env.hasBenchmark == '1' }} + id: benchmark + run: | + echo $(date) >> $GITHUB_STEP_SUMMARY + echo "exitStatus=1" >> $GITHUB_ENV + cd Benchmarks + swift package benchmark baseline check main pull_request --format markdown >> $GITHUB_STEP_SUMMARY + echo "exitStatus=0" >> $GITHUB_ENV + continue-on-error: true + - if: ${{ env.exitStatus == '0' }} + name: Pull request comment text success + id: prtestsuccess + run: | + echo 'PRTEST<> $GITHUB_ENV + echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/swift-extras/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + - if: ${{ env.exitStatus == '1' }} + name: Pull request comment text failure + id: prtestfailure + run: | + echo 'PRTEST<> $GITHUB_ENV + echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/swift-extras/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV + echo "_Pull request had performance regressions_" >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + - name: Comment PR + if: ${{ env.hasBenchmark == '1' }} + uses: thollander/actions-comment-pull-request@v2 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + message: ${{ env.PRTEST }} + comment_includes: "Pull request benchmark comparison [${{ matrix.os }}] with" + - name: Exit with correct status + run: | + exit ${{ env.exitStatus }} \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f8c56f7..b723996 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -58,27 +58,6 @@ jobs: - name: Build & run run: swift run -c release - "tuxOS-Integration-Tests": - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - images: - - swift:5.10 - container: - image: ${{ matrix.images }} - env: - MAX_ALLOCS_ALLOWED_base64_decoding: 1000 - MAX_ALLOCS_ALLOWED_base64_encoding: 1000 - MAX_ALLOCS_ALLOWED_base32_decoding: 1000 - MAX_ALLOCS_ALLOWED_base32_encoding: 1000 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Test - run: ./run-tests.sh - working-directory: ./IntegrationTests - "macOS-Tests": runs-on: macOS-14 strategy: diff --git a/Benchmarks/.gitignore b/Benchmarks/.gitignore new file mode 100644 index 0000000..8760ad6 --- /dev/null +++ b/Benchmarks/.gitignore @@ -0,0 +1,2 @@ +.benchmarkBaselines +Package.resolved \ No newline at end of file diff --git a/Benchmarks/Benchmarks/BaseN/BaseN.swift b/Benchmarks/Benchmarks/BaseN/BaseN.swift new file mode 100644 index 0000000..87bb2d8 --- /dev/null +++ b/Benchmarks/Benchmarks/BaseN/BaseN.swift @@ -0,0 +1,56 @@ +import Benchmark +import ExtrasBase64 + +let benchmarks = { + Benchmark.defaultConfiguration = .init( + metrics: [ + .cpuTotal, + .throughput, + .mallocCountTotal, + ], + warmupIterations: 10, + scalingFactor: .kilo + ) + + Benchmark("Base32.encode") { benchmark in + let bytes = Array(UInt8(0) ... UInt8(255)) + + benchmark.startMeasurement() + + for _ in benchmark.scaledIterations { + blackHole(Base32.encodeToString(bytes: bytes)) + } + } + + Benchmark("Base32.decode") { benchmark in + let bytes = Array(UInt8(0) ... UInt8(255)) + let base32 = Base32.encodeToString(bytes: bytes) + + benchmark.startMeasurement() + + for _ in benchmark.scaledIterations { + try blackHole(Base32.decode(string: base32)) + } + } + + Benchmark("Base64.encode") { benchmark in + let bytes = Array(UInt8(0) ... UInt8(255)) + + benchmark.startMeasurement() + + for _ in benchmark.scaledIterations { + blackHole(Base64.encodeToString(bytes: bytes)) + } + } + + Benchmark("Base64.decode") { benchmark in + let bytes = Array(UInt8(0) ... UInt8(255)) + let base64 = Base64.encodeToString(bytes: bytes) + + benchmark.startMeasurement() + + for _ in benchmark.scaledIterations { + try blackHole(Base64.decode(string: base64)) + } + } +} diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift new file mode 100644 index 0000000..f520fe5 --- /dev/null +++ b/Benchmarks/Package.swift @@ -0,0 +1,26 @@ +// swift-tools-version: 5.9 +// The swift-tools-version declares the minimum version of Swift required to build this package. + +import PackageDescription + +let package = Package( + name: "Benchmarks", + platforms: [.macOS(.v14)], + dependencies: [ + .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.0.0"), + .package(name: "swift-extras-base64", path: ".."), + ], + targets: [ + .executableTarget( + name: "BaseN", + dependencies: [ + .product(name: "ExtrasBase64", package: "swift-extras-base64"), + .product(name: "Benchmark", package: "package-benchmark"), + ], + path: "Benchmarks/BaseN", + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark"), + ] + ), + ] +) diff --git a/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh b/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh deleted file mode 100755 index fe76fd4..0000000 --- a/IntegrationTests/allocation-counter-tests-framework/run-allocation-counter.sh +++ /dev/null @@ -1,292 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftNIO open source project -## -## Copyright (c) 2019 Apple Inc. and the SwiftNIO project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftNIO project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -set -eu -here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -build_opts=( -c release ) - -function die() { - echo >&2 "ERROR: $*" - exit 1 -} - -function make_git_commit_all() { - git init > /dev/null - git checkout -b main - if [[ "$(git config user.email)" == "" ]]; then - git config --local user.email does@really-not.matter - git config --local user.name 'Does Not Matter' - fi - git add . > /dev/null - git commit -m 'everything' > /dev/null -} - -# -function hooked_package_swift_start() { - local extra_dependencies_file=$1 - local swiftpm_pkg_name=$2 - shift 2 - - cat <<"EOF" -// swift-tools-version:5.0 -import PackageDescription - -let package = Package( - name: "allocation-counter-tests", - products: [ -EOF - for f in "$@"; do - local module - module=$(module_name_from_path "$f") - echo ".executable(name: \"$module\", targets: [\"bootstrap_$module\"])," - done - cat < -function hooked_package_swift_target() { - local target_name="$1" - shift - local deps="" - for dep in "$@"; do - deps="$deps \"$dep\"," - done - cat < Package.swift <