From 1abee4a9b819f120d5f19927eee72e587ba7cd0b Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Tue, 3 Sep 2024 21:54:24 -0400 Subject: [PATCH] Add macOS toolchain build and test --- .github/workflows/sdks.yml | 309 ++++++++++++++++++++-------- get-packages-and-swift-source.swift | 34 ++- 2 files changed, 256 insertions(+), 87 deletions(-) diff --git a/.github/workflows/sdks.yml b/.github/workflows/sdks.yml index 4ed3d6d..06cf32b 100644 --- a/.github/workflows/sdks.yml +++ b/.github/workflows/sdks.yml @@ -3,6 +3,7 @@ on: pull_request: schedule: - cron: '0 9 * * *' + jobs: get-latest-toolchain: runs-on: ubuntu-latest @@ -33,15 +34,22 @@ jobs: echo "trunk-tag=$SWIFT_TAG" >> $GITHUB_OUTPUT fi echo "latest-tag=$SWIFT_TAG" >> $GITHUB_OUTPUT - - name: Get cached toolchain - id: cache-toolchain + - name: Get cached Ubuntu toolchain + id: cache-toolchain-ubuntu uses: actions/cache@v4 with: path: ~/${{ steps.check.outputs.latest-tag }}-ubuntu22.04.tar.gz - key: ${{ steps.check.outputs.latest-tag }}-toolchain - - name: Get latest toolchain if not cached - if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' }} + key: swift-ubuntu22.04-${{ steps.check.outputs.latest-tag }}-toolchain + - name: Get cached macOS toolchain + id: cache-toolchain-macos + uses: actions/cache@v4 + with: + path: ~/${{ steps.check.outputs.latest-tag }}-osx.pkg + key: swift-macos-${{ steps.check.outputs.latest-tag }}-toolchain + - name: Download toolchain + if: ${{ steps.cache-toolchain-ubuntu.outputs.cache-hit != 'true' || steps.cache-toolchain-macos.outputs.cache-hit != 'true' }} run: | + SWIFT_TAG="${{ steps.check.outputs.latest-tag }}" if [ ${{ matrix.version }} = 'release' ]; then SWIFT_BRANCH="swift-$(echo ${{ steps.check.outputs.release-tag }} | cut -d- -f2)-release" elif [ ${{ matrix.version }} = 'devel' ]; then @@ -50,17 +58,23 @@ jobs: SWIFT_BRANCH="development" fi cd - SWIFT_TAG="${{ steps.check.outputs.latest-tag }}" - wget -q https://download.swift.org/$SWIFT_BRANCH/ubuntu2204/$SWIFT_TAG/$SWIFT_TAG-ubuntu22.04.tar.gz - echo "got latest toolchain: ${SWIFT_TAG}" + if [[ "${{ steps.cache-toolchain-macos.outputs.cache-hit }}" != 'true' ]]; then + wget -q https://download.swift.org/$SWIFT_BRANCH/xcode/$SWIFT_TAG/$SWIFT_TAG-osx.pkg + echo "downloaded latest macOS toolchain: ${SWIFT_TAG}" + fi + if [[ "${{ steps.cache-toolchain-ubuntu.outputs.cache-hit }}" != 'true' ]]; then + wget -q https://download.swift.org/$SWIFT_BRANCH/ubuntu2204/$SWIFT_TAG/$SWIFT_TAG-ubuntu22.04.tar.gz + echo "downloaded latest Ubuntu toolchain: ${SWIFT_TAG}" + fi build-sdk-and-tests: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} needs: get-latest-toolchain strategy: fail-fast: false matrix: version: [release, devel, trunk] - arch: [aarch64, x86_64, armv7] + arch: [x86_64, aarch64, armv7] + os: [ubuntu-22.04, macos-13] env: ANDROID_API_LEVEL: 24 steps: @@ -78,12 +92,16 @@ jobs: echo "latest=$(echo $TAG | cut -d- -f4-6)" >> $GITHUB_OUTPUT fi echo "tag=$TAG" >> $GITHUB_OUTPUT - - name: Get cached Swift ${{ matrix.version }} toolchain - id: cache-toolchain - uses: actions/cache@v4 - with: - path: ~/${{ steps.version.outputs.tag }}-ubuntu22.04.tar.gz - key: ${{ steps.version.outputs.tag }}-toolchain + - name: Setup OS Environment + id: setup-os + run: | + if [[ ${{ matrix.os }} = 'macos'* ]]; then + # the inplace "-i" flag to sed requires an empty arg on macOS + echo 'sed -i "" "${@}"' > ~/sedinplace.sh + else + echo 'sed -i "${@}"' > ~/sedinplace.sh + fi + chmod +x ~/sedinplace.sh - name: Get cached SDK id: cache-sdk uses: actions/cache@v4 @@ -94,30 +112,73 @@ jobs: uses: actions/checkout@v4 with: path: sdk-config + - name: Get cached Swift Ubuntu ${{ matrix.version }} toolchain + if: ${{ matrix.os == 'ubuntu-22.04' }} + id: cache-toolchain-ubuntu + uses: actions/cache@v4 + with: + path: ~/${{ steps.version.outputs.tag }}-ubuntu22.04.tar.gz + key: swift-ubuntu22.04-${{ steps.version.outputs.tag }}-toolchain + - name: Get cached Swift macOS ${{ matrix.version }} toolchain + if: ${{ matrix.os == 'macos-13' }} + id: cache-toolchain-macos + uses: actions/cache@v4 + with: + path: ~/${{ steps.version.outputs.tag }}-osx.pkg + key: swift-macos-${{ steps.version.outputs.tag }}-toolchain + - name: Setup Swift Toolchain + env: + SWIFT_TAG: ${{ steps.version.outputs.tag }} + run: | + set -x + cd sdk-config + if [[ ${{ matrix.os }} = 'macos'* ]]; then + mkdir ${SWIFT_TAG} + pushd ${SWIFT_TAG} + xar -x -C . -f ~/${SWIFT_TAG}-osx.pkg + tar xz -C . -f ${SWIFT_TAG}-osx-package.pkg/Payload + TOOLCHAIN=${PWD}/usr + popd + elif [[ ${{ matrix.os }} = 'ubuntu'* ]]; then + tar xf ~/$SWIFT_TAG-ubuntu22.04.tar.gz + TOOLCHAIN=${PWD}/$SWIFT_TAG-ubuntu22.04/usr + fi + + echo "TOOLCHAIN=${TOOLCHAIN}" >> $GITHUB_ENV + ${TOOLCHAIN}/bin/swift --version + - name: Build Swift ${{ matrix.version }} Android SDK if not the latest - if: ${{ steps.cache-sdk.outputs.cache-hit != 'true' }} + # build-script currently only works on ubuntu + if: ${{ (steps.cache-sdk.outputs.cache-hit != 'true') && (matrix.os == 'ubuntu-22.04') }} env: SWIFT_TAG: ${{ steps.version.outputs.tag }} ANDROID_ARCH: ${{ matrix.arch }} run: | + set -x + + if [[ ${{ matrix.os }} = 'macos'* ]]; then + brew install ninja patchelf + elif [[ ${{ matrix.os }} = 'ubuntu'* ]]; then + sudo apt install ninja-build + fi + cd sdk-config - sudo apt install ninja-build - BUILD_SWIFT_PM=1 swift get-packages-and-swift-source.swift - SDK_NAME=$(ls | grep swift-$(echo ${{ matrix.version }} | sed "s/-2[5-9][a-e]\?//")-android-${{ matrix.arch }}) + ${TOOLCHAIN}/bin/swift --version + BUILD_SWIFT_PM=1 ${TOOLCHAIN}/bin/swift get-packages-and-swift-source.swift + + SDK_NAME=$(ls | grep swift-${{ matrix.version }}-android-${{ matrix.arch }}) SDK=`pwd`/$SDK_NAME - tar xf ~/$SWIFT_TAG-ubuntu22.04.tar.gz - ./$SWIFT_TAG-ubuntu22.04/usr/bin/swift --version git apply swift-android-ci.patch git apply -C1 swift-android.patch git apply -C0 swift-android-foundation-ndk26.patch if [[ ${{ matrix.version }} = 'release' ]]; then - sed -i "s%strsignal(signal).map%String(cString: strsignal(signal)) //%" swift-driver/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift + ~/sedinplace.sh "s%strsignal(signal).map%String(cString: strsignal(signal)) //%" swift-driver/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift git apply swift-android-release-stdlib.patch STUPID_FILE_RENAMING=Tool else - sed -i "s%r26%ndk/27%" swift/stdlib/cmake/modules/AddSwiftStdlib.cmake + ~/sedinplace.sh "s%r26%ndk/27%" swift/stdlib/cmake/modules/AddSwiftStdlib.cmake if [ ${{ matrix.version }} = 'devel' ]; then git apply android-overlay/import-android-devel.patch else @@ -127,10 +188,10 @@ jobs: STUPID_FILE_RENAMING=Command fi - sed -i "s%/data/data/com.termux/files%$SDK%" $SDK/usr/lib/pkgconfig/sqlite3.pc - sed -i "s%String(cString: getpass%\"fake\" //%" swiftpm/Sources/PackageRegistry$STUPID_FILE_RENAMING/PackageRegistry$STUPID_FILE_RENAMING+Auth.swift + ~/sedinplace.sh "s%/data/data/com.termux/files%$SDK%" $SDK/usr/lib/pkgconfig/sqlite3.pc + ~/sedinplace.sh "s%String(cString: getpass%\"fake\" //%" swiftpm/Sources/PackageRegistry$STUPID_FILE_RENAMING/PackageRegistry$STUPID_FILE_RENAMING+Auth.swift - ./swift/utils/build-script -RA --skip-build-cmark --build-llvm=0 --android --android-ndk $ANDROID_NDK --android-arch ${{ matrix.arch }} --android-api-level $ANDROID_API_LEVEL --build-swift-tools=0 --native-swift-tools-path=`pwd`/$SWIFT_TAG-ubuntu22.04/usr/bin --native-clang-tools-path=`pwd`/$SWIFT_TAG-ubuntu22.04/usr/bin --cross-compile-hosts=android-${{ matrix.arch }} --cross-compile-deps-path=$SDK --skip-local-build --build-swift-static-stdlib --xctest --skip-early-swift-driver --install-swift --install-libdispatch --install-foundation --install-xctest --install-destdir=$SDK --swift-install-components='clang-resource-dir-symlink;license;stdlib;sdk-overlay' --cross-compile-append-host-target-to-destdir=False -b -p --install-llbuild --sourcekit-lsp --skip-early-swiftsyntax + ./swift/utils/build-script -RA --skip-build-cmark --build-llvm=0 --android --android-ndk $ANDROID_NDK --android-arch ${{ matrix.arch }} --android-api-level $ANDROID_API_LEVEL --build-swift-tools=0 --native-swift-tools-path=${TOOLCHAIN}/bin --native-clang-tools-path=${TOOLCHAIN}/bin --cross-compile-hosts=android-${{ matrix.arch }} --cross-compile-deps-path=$SDK --skip-local-build --build-swift-static-stdlib --xctest --skip-early-swift-driver --install-swift --install-libdispatch --install-foundation --install-xctest --install-destdir=$SDK --swift-install-components='clang-resource-dir-symlink;license;stdlib;sdk-overlay' --cross-compile-append-host-target-to-destdir=False -b -p --install-llbuild --sourcekit-lsp --skip-early-swiftsyntax cp $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/$(echo ${{ matrix.arch }} | sed "s/v7//")-linux-android*/libc++_shared.so $SDK/usr/lib patchelf --set-rpath \$ORIGIN $SDK/usr/lib/swift/android/libdispatch.so @@ -141,28 +202,88 @@ jobs: - name: Upload SDK uses: actions/upload-artifact@v4 with: - name: sdk-${{ matrix.version }}-${{ matrix.arch }} + name: swift-android-sdk-${{ matrix.version }}-${{ matrix.arch }}-host-${{ matrix.os }} path: ~/swift-${{ matrix.version }}-android-${{ matrix.arch }}*-sdk.tar.xz - - name: Unpack ${{ matrix.version }} toolchain and SDK - id: sdk-unpack + - name: "Install Android NDK" + id: setup-ndk + uses: nttld/setup-ndk@v1 + with: + ndk-version: r27 + add-to-path: false + - name: Setup Swift ${{ matrix.version }} Android SDK + id: sdk-setup run: | + set -x cd sdk-config + tar xf ~/swift-${{ matrix.version }}-android-${{ matrix.arch }}*-sdk.tar.xz + pushd swift-*-sdk + SDK_PATH=${PWD} + popd + + if [[ ${{ matrix.os }} = 'ubuntu'* ]]; then + ARCH_JSON=${SDK_PATH}/usr/swiftpm-android-${{ matrix.arch }}.json + cat ${ARCH_JSON} + elif [[ ${{ matrix.os }} = 'macos'* ]]; then + # fix the clang symlink + ls -la ${SDK_PATH}/usr/lib/swift/clang + rm -vf ${SDK_PATH}/usr/lib/swift/clang + # Swift 5.10: usr/lib/clang/15.0.0 Trunk and Devel: usr/lib/clang/17 + ${TOOLCHAIN}/bin/clang --version + ln -sfv ${TOOLCHAIN}/lib/clang/* ${SDK_PATH}/usr/lib/swift/clang + ls -d ${SDK_PATH}/usr/lib/swift/clang/include + + # the toolchain destination JSON in the SDK is for ubuntu, + # so for macOS we need to manually build it + ARCH=${{ matrix.arch }} + ARCH_JSON="$PWD/android-${ARCH}.json" + ARCH_TARGET="${ARCH}-unknown-linux-android24" + ARCHID="${ARCH}-linux-android" + if [ "$ARCH" == "armv7" ]; then + ARCHID="arm-linux-androideabi" + fi - if [ ! -d ${{ steps.version.outputs.tag }}-ubuntu22.04 ]; then - tar xf ~/${{ steps.version.outputs.tag }}-ubuntu22.04.tar.gz + # current default on macos is 26.3.11579264, + # so use the path from the nttld/setup-ndk action for r27 + ANDROID_NDK=${{ steps.setup-ndk.outputs.ndk-path }} + NDK_PREBUILT=${ANDROID_NDK}/toolchains/llvm/prebuilt/darwin-x86_64 + + # With the full SDK, the libs are all located in: + #"-L", "${SDK}/usr/lib/${ARCHID}" + # But with the partial SDK, they are divided between: + #"-L", "${SDK_PATH}/usr/lib", + #"-L", "${SDK_PATH}/usr/lib/swift/android", + cat > ${ARCH_JSON} << EOF + { + "version": 1, + "target": "${ARCH_TARGET}", + "toolchain-bin-dir": "${TOOLCHAIN}/bin", + "sdk": "${NDK_PREBUILT}/sysroot", + "extra-cc-flags": [ "-fPIC" ], + "extra-cpp-flags": [ "-lstdc++" ], + "extra-swiftc-flags": [ + "-resource-dir", "${SDK_PATH}/usr/lib/swift", + "-tools-directory", "${NDK_PREBUILT}/bin", + "-L", "${SDK_PATH}/usr/lib" + ] + } + EOF + + cat ${ARCH_JSON} fi - ./${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift --version - tar xf ~/swift-${{ matrix.version }}-android-${{ matrix.arch }}*-sdk.tar.xz + echo "SDK_PATH=${SDK_PATH}" >> $GITHUB_ENV + echo "ARCH_JSON=${ARCH_JSON}" >> $GITHUB_ENV - name: Get Swift Argument Parser package uses: actions/checkout@v4 with: repository: apple/swift-argument-parser path: swift-argument-parser - name: Build Swift Argument Parser package + # macOS error: ld.lld: error: --defsym:1: symbol not found: changelog_authors_main + if: ${{ matrix.os != 'macos-13' }} run: | cd swift-argument-parser - ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift build --build-tests --destination ../sdk-config/swift-*-sdk/usr/swiftpm-android-${{ matrix.arch }}.json -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android + ${TOOLCHAIN}/bin/swift build --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android - name: Get Swift crypto package uses: actions/checkout@v4 with: @@ -171,9 +292,9 @@ jobs: - name: Build Swift crypto package run: | cd swift-crypto - sed -i "s%\\\\(testsDirectory)/.*Vectors%/data/local/tmp/pack/crypto-vectors%" Tests/CryptoTests/Utils/RFCVector.swift Tests/CryptoTests/Utils/Wycheproof.swift Tests/_CryptoExtrasTests/Utils/Wycheproof.swift - sed -i 's%#file%"/data/local/tmp/pack/crypto-vectors"%;s%../_CryptoExtrasVectors/%%' Tests/_CryptoExtrasTests/TestRSABlindSigning.swift - ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift build --build-tests --destination ../sdk-config/swift-*-sdk/usr/swiftpm-android-${{ matrix.arch }}.json -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android + ~/sedinplace.sh "s%\\\\(testsDirectory)/.*Vectors%/data/local/tmp/pack/crypto-vectors%" Tests/{Crypto,_CryptoExtras}Tests/Utils/{RFCVector,Wycheproof}.swift + ~/sedinplace.sh 's%#file%"/data/local/tmp/pack/crypto-vectors"%;s%../_CryptoExtrasVectors/%%' Tests/_CryptoExtrasTests/TestRSABlindSigning.swift + ${TOOLCHAIN}/bin/swift build --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android - name: Get Swift NIO package uses: actions/checkout@v4 with: @@ -183,11 +304,11 @@ jobs: run: | cd swift-nio git apply ../sdk-config/swift-nio-disable-ecn-tests.patch ../sdk-config/swift-nio-filesystem-both-ndks.patch ../sdk-config/swift-nio-ndk27.patch - ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift package update + ${TOOLCHAIN}/bin/swift package update cd .build/checkouts/ patch -p1 < ../../../sdk-config/android-overlay/swift-system.patch cd ../.. - ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift build --build-tests --destination ../sdk-config/swift-*-sdk/usr/swiftpm-android-${{ matrix.arch }}.json -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android + ${TOOLCHAIN}/bin/swift build --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android - name: Get Swift Numerics package uses: actions/checkout@v4 with: @@ -196,7 +317,7 @@ jobs: - name: Build Swift Numerics package run: | cd swift-numerics - ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift build --build-tests --destination ../sdk-config/swift-*-sdk/usr/swiftpm-android-${{ matrix.arch }}.json -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android + ${TOOLCHAIN}/bin/swift build --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android - name: Get Swift System package uses: actions/checkout@v4 with: @@ -205,7 +326,7 @@ jobs: - name: Build Swift System package run: | cd swift-system - ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift build --build-tests --destination ../sdk-config/swift-*-sdk/usr/swiftpm-android-${{ matrix.arch }}.json -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android + ${TOOLCHAIN}/bin/swift build --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android - name: Get Swift Collections package uses: actions/checkout@v4 with: @@ -214,7 +335,7 @@ jobs: - name: Build Swift Collections package run: | cd swift-collections - ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift build -j 1 --build-tests --destination ../sdk-config/swift-*-sdk/usr/swiftpm-android-${{ matrix.arch }}.json -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android + ${TOOLCHAIN}/bin/swift build -j 1 --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android - name: Get Swift Atomics package uses: actions/checkout@v4 with: @@ -229,8 +350,8 @@ jobs: - name: Build Swift NIO SSH package run: | cd sns - sed -i "s%url: .*swift-\(\w\+\)\.git.*$%path: \"../swift-\1\"),%g" Package.swift - ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift build --build-tests --destination ../sdk-config/swift-*-sdk/usr/swiftpm-android-${{ matrix.arch }}.json -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android + ~/sedinplace.sh "s%url: .*swift-\([a-z]*\)\.git.*$%path: \"../swift-\1\"),%g" Package.swift + ${TOOLCHAIN}/bin/swift build --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android - name: Get Swift NIO SSL package uses: actions/checkout@v4 with: @@ -239,7 +360,7 @@ jobs: - name: Build Swift NIO SSL package run: | cd snl - SWIFTCI_USE_LOCAL_DEPS=1 ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift build --build-tests --destination ../sdk-config/swift-*-sdk/usr/swiftpm-android-${{ matrix.arch }}.json -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android + SWIFTCI_USE_LOCAL_DEPS=1 ${TOOLCHAIN}/bin/swift build --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android - name: Get Yams package uses: actions/checkout@v4 with: @@ -248,8 +369,8 @@ jobs: - name: Build Yams package run: | cd yams - sed -i "s% fixturesDirectory + \"/SourceKitten#289% \"/data/local/tmp/pack%" Tests/YamsTests/PerformanceTests.swift - ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift build --build-tests --destination ../sdk-config/swift-*-sdk/usr/swiftpm-android-${{ matrix.arch }}.json -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android + ~/sedinplace.sh "s% fixturesDirectory + \"/SourceKitten#289% \"/data/local/tmp/pack%" Tests/YamsTests/PerformanceTests.swift + ${TOOLCHAIN}/bin/swift build --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android - name: Get Swift NIO HTTP/2 package uses: actions/checkout@v4 with: @@ -259,8 +380,8 @@ jobs: if: ${{ matrix.arch != 'armv7' }} run: | cd snh - sed -i "s%url: .*swift-\(\w\+\)\.git.*$%path: \"../swift-\1\"),%g" Package.swift - ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift build --build-tests --destination ../sdk-config/swift-*-sdk/usr/swiftpm-android-${{ matrix.arch }}.json -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android + ~/sedinplace.sh "s%url: .*swift-\([a-z]*\)\.git.*$%path: \"../swift-\1\"),%g" Package.swift + ${TOOLCHAIN}/bin/swift build --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android - name: Get Swift Algorithms package uses: actions/checkout@v4 with: @@ -269,8 +390,8 @@ jobs: - name: Build Swift Algorithms package run: | cd sa - sed -i "s%url: .*$%path: \"../swift-numerics\"),%" Package.swift - ../sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04/usr/bin/swift build --build-tests --destination ../sdk-config/swift-*-sdk/usr/swiftpm-android-${{ matrix.arch }}.json -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android + ~/sedinplace.sh "s%url: .*$%path: \"../swift-numerics\"),%" Package.swift + ${TOOLCHAIN}/bin/swift build --build-tests --destination ${ARCH_JSON} -Xlinker -rpath -Xlinker \$ORIGIN/lib/swift/android - name: Get cached Termux debug app for NIO tests if: ${{ matrix.arch == 'x86_64' }} id: cache-termux @@ -279,14 +400,33 @@ jobs: path: ~/termux-debug.apk key: termux-app - name: Get Termux debug app if not cached - if: ${{ matrix.arch == 'x86_64' && steps.cache-termux.outputs.cache-hit != 'true'}} + if: ${{ (matrix.arch == 'x86_64') && (steps.cache-termux.outputs.cache-hit != 'true') }} run: wget -O ~/termux-debug.apk https://github.com/termux/termux-app/releases/download/v0.118.0/termux-app_v0.118.0+github-debug_x86_64.apk - - name: Put x86_64 tests in one directory to push to Android x86_64 emulator and set KVM permissions + - name: Prepare Android emulator test package if: ${{ matrix.arch == 'x86_64' }} run: | + set -x + # create the test runner script + cat > ~/test-toolchain.sh << EOF + adb install ~/termux-debug.apk + adb push pack /data/local/tmp + adb shell "run-as com.termux mkdir /data/data/com.termux/pack" + adb shell "run-as com.termux cp /data/local/tmp/pack/{swift-nioPackageTests.xctest,FileHandleTests.swift} /data/data/com.termux/pack" + adb shell "run-as com.termux cp -r /data/local/tmp/pack/lib /data/data/com.termux/pack" + adb shell "run-as com.termux cp -r /data/local/tmp/pack/Test\ Data /data/data/com.termux/pack" + adb shell "run-as com.termux ln -s README.md /data/data/com.termux/pack/Test\ Data/README.md.symlink" + adb shell "run-as com.termux ln -s Foo /data/data/com.termux/pack/Test\ Data/Foo.symlink" + adb shell "run-as com.termux sh -c 'TMPDIR=/data/data/com.termux /data/data/com.termux/pack/swift-nioPackageTests.xctest'" + adb shell "TMPDIR=/data/local/tmp /data/local/tmp/pack/swift-systemPackageTests.xctest" + EOF + mkdir -p pack/lib/swift/android TARGET="x86_64-unknown-linux-android$ANDROID_API_LEVEL" - cp swift-argument-parser/.build/$TARGET/debug/{generate-manual,math,repeat,roll,swift-argument-parserPackageTests.xctest} pack + + if [[ ${{ matrix.os }} != 'macos'* ]]; then + cp swift-argument-parser/.build/$TARGET/debug/{generate-manual,math,repeat,roll,swift-argument-parserPackageTests.xctest} pack + echo 'adb shell /data/local/tmp/pack/swift-argument-parserPackageTests.xctest' >> ~/test-toolchain.sh + fi wget -q https://raw.githubusercontent.com/termux/termux-elf-cleaner/v1.10/termux-elf-cleaner.cpp wget -q https://raw.githubusercontent.com/termux/termux-elf-cleaner/v1.10/elf.h @@ -294,15 +434,31 @@ jobs: ./elf-cleaner pack/{generate-manual,math,repeat,roll} || true cp swift-crypto/.build/$TARGET/debug/swift-cryptoPackageTests.xctest pack + echo 'adb shell /data/local/tmp/pack/swift-cryptoPackageTests.xctest' >> ~/test-toolchain.sh + cp swift-nio/.build/$TARGET/debug/swift-nioPackageTests.xctest pack + cp swift-numerics/.build/$TARGET/debug/swift-numericsPackageTests.xctest pack + echo 'adb shell /data/local/tmp/pack/swift-numericsPackageTests.xctest' >> ~/test-toolchain.sh + cp swift-system/.build/$TARGET/debug/swift-systemPackageTests.xctest pack cp swift-collections/.build/$TARGET/debug/swift-collectionsPackageTests.xctest pack + echo 'adb shell /data/local/tmp/pack/swift-collectionsPackageTests.xctest' >> ~/test-toolchain.sh + cp sns/.build/$TARGET/debug/swift-nio-sshPackageTests.xctest pack + echo 'adb shell /data/local/tmp/pack/swift-nio-sshPackageTests.xctest' >> ~/test-toolchain.sh + cp snl/.build/$TARGET/debug/swift-nio-sslPackageTests.xctest pack - cp yams/.build/$TARGET/debug/YamsPackageTests.xctest pack + echo 'adb shell /data/local/tmp/pack/swift-nio-sslPackageTests.xctest' >> ~/test-toolchain.sh + cp snh/.build/$TARGET/debug/swift-nio-http2PackageTests.xctest pack + echo 'adb shell /data/local/tmp/pack/swift-nio-http2PackageTests.xctest' >> ~/test-toolchain.sh + + cp yams/.build/$TARGET/debug/YamsPackageTests.xctest pack + echo 'adb shell /data/local/tmp/pack/YamsPackageTests.xctest' >> ~/test-toolchain.sh + cp sa/.build/$TARGET/debug/swift-algorithmsPackageTests.xctest pack + echo 'adb shell /data/local/tmp/pack/swift-algorithmsPackageTests.xctest' >> ~/test-toolchain.sh mkdir pack/crypto-vectors pack/swift-crypto_CryptoTests.resources cp swift-crypto/Tests/Test\ Vectors/* swift-crypto/Tests/_CryptoExtrasVectors/* pack/crypto-vectors @@ -313,35 +469,26 @@ jobs: cp sdk-config/swift-*-android-x86_64-*${ANDROID_API_LEVEL}-sdk/usr/lib/lib*so pack/lib cp sdk-config/swift-*-android-x86_64-*${ANDROID_API_LEVEL}-sdk/usr/lib/swift/android/lib*so pack/lib/swift/android mv pack/lib/libc++_shared.so pack/lib/swift/android - rm -rf sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04 + # need to free up some space or else the emulator fails to launch: + # ERROR | Not enough space to create userdata partition. Available: 6086.191406 MB at /home/runner/.android/avd/../avd/test.avd, need 7372.800000 MB. + rm -rf sdk-config/${{ steps.version.outputs.tag }}-ubuntu22.04 */.build + + chmod +x ~/test-toolchain.sh + + echo "TEST SCRIPT:" + cat ~/test-toolchain.sh + + if [[ ${{ matrix.os }} != 'macos'* ]]; then + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + fi - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - name: Run tests on Android x86_64 emulator if: ${{ matrix.arch == 'x86_64' }} uses: reactivecircus/android-emulator-runner@v2 with: api-level: 24 arch: x86_64 - script: | - adb install ~/termux-debug.apk - adb push pack /data/local/tmp - - adb shell /data/local/tmp/pack/swift-argument-parserPackageTests.xctest - adb shell /data/local/tmp/pack/swift-cryptoPackageTests.xctest - adb shell "run-as com.termux mkdir /data/data/com.termux/pack" - adb shell "run-as com.termux cp /data/local/tmp/pack/{swift-nioPackageTests.xctest,FileHandleTests.swift} /data/data/com.termux/pack" - adb shell "run-as com.termux cp -r /data/local/tmp/pack/lib /data/data/com.termux/pack" - adb shell "run-as com.termux cp -r /data/local/tmp/pack/Test\ Data /data/data/com.termux/pack" - adb shell "run-as com.termux ln -s README.md /data/data/com.termux/pack/Test\ Data/README.md.symlink" - adb shell "run-as com.termux ln -s Foo /data/data/com.termux/pack/Test\ Data/Foo.symlink" - adb shell "run-as com.termux sh -c 'TMPDIR=/data/data/com.termux /data/data/com.termux/pack/swift-nioPackageTests.xctest'" - adb shell /data/local/tmp/pack/swift-numericsPackageTests.xctest - adb shell "TMPDIR=/data/local/tmp /data/local/tmp/pack/swift-systemPackageTests.xctest" - adb shell /data/local/tmp/pack/swift-collectionsPackageTests.xctest - adb shell /data/local/tmp/pack/swift-nio-sshPackageTests.xctest - adb shell /data/local/tmp/pack/swift-nio-sslPackageTests.xctest - adb shell /data/local/tmp/pack/swift-nio-http2PackageTests.xctest - adb shell /data/local/tmp/pack/swift-algorithmsPackageTests.xctest - adb shell /data/local/tmp/pack/YamsPackageTests.xctest + script: ~/test-toolchain.sh + diff --git a/get-packages-and-swift-source.swift b/get-packages-and-swift-source.swift index 1adc80d..d1d9449 100644 --- a/get-packages-and-swift-source.swift +++ b/get-packages-and-swift-source.swift @@ -36,13 +36,13 @@ let tagExtract = try NSRegularExpression(pattern: "swift-([5-9]\\.[0-9]+)?\\.?[1 if tagExtract.numberOfMatches(in: SWIFT_TAG, range: tagRange) == 1 { let match = tagExtract.firstMatch(in: SWIFT_TAG, range: tagRange) if match!.range(at: 1).location != NSNotFound { - swiftVersion = SWIFT_TAG.substring(with: match!.range(at: 1)) + swiftVersion = (SWIFT_TAG as NSString).substring(with: match!.range(at: 1)) } - swiftBranch = SWIFT_TAG.substring(with: match!.range(at: 2)) + swiftBranch = (SWIFT_TAG as NSString).substring(with: match!.range(at: 2)) if match!.range(at: 3).location != NSNotFound { - swiftSnapshotDate = SWIFT_TAG.substring(with: match!.range(at: 3)) + swiftSnapshotDate = (SWIFT_TAG as NSString).substring(with: match!.range(at: 3)) } } else { fatalError("Something went wrong with extracting data from the SWIFT_TAG environment variable: \(SWIFT_TAG)") @@ -98,6 +98,8 @@ func runCommand(_ name: String, with args: [String]) -> String { command.standardOutput = output command.standardError = error do { + print("running command: \(([command.executableURL!.path] + args).joined(separator: " "))") + fflush(stdout) try command.run() } catch { fatalError("couldn't run \(name) \(args) with error: \(error)") @@ -128,14 +130,30 @@ func runCommand(_ name: String, with args: [String]) -> String { print("Checking if needed system utilities are installed...") print(runCommand("cmake", with: ["--version"])) print("ninja \(runCommand("ninja", with: ["--version"]))") + +#if os(macOS) +print(runCommand("python3", with: ["--version"])) +#else print(runCommand("python", with: ["--version"])) +#endif print(runCommand("patchelf", with: ["--version"])) +#if !os(macOS) +// ar does not take a "--version" arg on macOS print(runCommand("ar", with: ["--version"])) +#endif print(runCommand("tar", with: ["--version"])) print(runCommand("xz", with: ["--version"])) print(runCommand("curl", with: ["--version"])) print(runCommand("gzip", with: ["--version"])) +#if os(macOS) +extension String { + func appendingPathComponent(_ path: String) -> String { + (self as NSString).appendingPathComponent(path) + } +} +#endif + let fmd = FileManager.default let cwd = fmd.currentDirectoryPath let termuxArchive = cwd.appendingPathComponent("termux") @@ -164,7 +182,7 @@ for termuxPackage in termuxPackages { print("Checking for \(packageName)") if !fmd.fileExists(atPath: termuxArchive.appendingPathComponent(String(packageName))) { print("Downloading \(packageName)") - _ = runCommand("curl", with: ["-o", "termux/\(packageName)", + _ = runCommand("curl", with: ["-f", "-o", "termux/\(packageName)", "\(termuxURL)/\(packagePath)"]) } @@ -181,7 +199,11 @@ for termuxPackage in termuxPackages { if !fmd.fileExists(atPath: cwd.appendingPathComponent(sdkDir)) { print("Unpacking \(packageName)") +#if os(macOS) + _ = runCommand("tar", with: ["xf", "\(termuxArchive.appendingPathComponent(String(packageName)))"]) +#else _ = runCommand("ar", with: ["x", "\(termuxArchive.appendingPathComponent(String(packageName)))"]) +#endif _ = runCommand("tar", with: ["xf", "data.tar.xz"]) } } @@ -240,7 +262,7 @@ for repo in swiftRepos { print("Checking for \(repo) source") if !fmd.fileExists(atPath: cwd.appendingPathComponent(repo)) { print("Downloading and extracting \(repo) source") - _ = runCommand("curl", with: ["-L", "-O", + _ = runCommand("curl", with: ["-f", "-L", "-O", "https://github.com/apple/\(repo)/archive/refs/tags/\(SWIFT_TAG).tar.gz"]) _ = runCommand("tar", with: ["xf", "\(SWIFT_TAG).tar.gz"]) try fmd.moveItem(atPath: cwd.appendingPathComponent("\(repo)-\(SWIFT_TAG)"), @@ -252,7 +274,7 @@ for repo in swiftRepos { if ProcessInfo.processInfo.environment["BUILD_SWIFT_PM"] != nil { for repo in extraSwiftRepos { let tag = repoTags[repo] ?? SWIFT_TAG - _ = runCommand("curl", with: ["-L", "-O", + _ = runCommand("curl", with: ["-f", "-L", "-O", "https://github.com/\(repo == "Yams" ? "jpsim" : "apple")/\(repo)/archive/refs/tags/\(tag).tar.gz"]) _ = runCommand("tar", with: ["xf", "\(tag).tar.gz"]) try fmd.moveItem(atPath: cwd.appendingPathComponent("\(repo)-\(tag)"),