diff --git a/Package.swift b/Package.swift index 8094ef8..5575be1 100644 --- a/Package.swift +++ b/Package.swift @@ -63,6 +63,7 @@ let package = Package( .product(name: "Crypto", package: "swift-crypto"), .product(name: "Logging", package: "swift-log"), .product(name: "SystemPackage", package: "swift-system"), + .product(name: "_NIOFileSystem", package: "swift-nio"), ], exclude: ["Vendor/README.md"], swiftSettings: [ diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/OpenReadableFile.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/OpenReadableFile.swift index 6965689..4291885 100644 --- a/Sources/Helpers/Vendor/_AsyncFileSystem/OpenReadableFile.swift +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/OpenReadableFile.swift @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +@preconcurrency import _NIOFileSystem + import class Dispatch.DispatchQueue import struct SystemPackage.FileDescriptor @@ -20,6 +22,7 @@ public struct OpenReadableFile: Sendable { /// Underlying storage for this file handle, dependent on the file system type that produced it. enum Storage { + case nio(ReadFileHandle) /// Operating system file descriptor and a queue used for reading from that file descriptor without blocking /// the Swift Concurrency thread pool. case real(FileDescriptor, DispatchQueue) @@ -30,11 +33,17 @@ public struct OpenReadableFile: Sendable { /// Concrete instance of underlying file storage. let fileHandle: Storage - /// Creates a readable ``AsyncSequence`` that can be iterated on to read from this file handle. /// - Returns: `ReadableFileStream` value conforming to ``AsyncSequence``, ready for asynchronous iteration. public func read() async throws -> ReadableFileStream { switch self.fileHandle { + case let .nio(fileDescriptor): + return ReadableFileStream.nio( + .init( + fileDescriptor: fileDescriptor, + readChunkSize: self.chunkSize + ) + ) case let .real(fileDescriptor, ioQueue): return ReadableFileStream.real( .init( @@ -43,7 +52,6 @@ public struct OpenReadableFile: Sendable { readChunkSize: self.chunkSize ) ) - case .mock(let array): return ReadableFileStream.mock(.init(bytes: array, chunkSize: self.chunkSize)) } diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/OpenWritableFile.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/OpenWritableFile.swift index 442f5d0..cbbbb27 100644 --- a/Sources/Helpers/Vendor/_AsyncFileSystem/OpenWritableFile.swift +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/OpenWritableFile.swift @@ -9,6 +9,7 @@ // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors // //===----------------------------------------------------------------------===// +@preconcurrency import _NIOFileSystem import class Dispatch.DispatchQueue @preconcurrency import struct SystemPackage.FileDescriptor @@ -18,6 +19,7 @@ import struct SystemPackage.FilePath public actor OpenWritableFile: WritableStream { /// Underlying storage for this file handle, dependent on the file system type that produced it. enum Storage { + case nio(WriteFileHandle) /// Operating system file descriptor and a queue used for reading from that file descriptor without blocking /// the Swift Concurrency thread pool. case real(FileDescriptor, DispatchQueue) @@ -48,6 +50,14 @@ public actor OpenWritableFile: WritableStream { public func write(_ bytes: some Collection & Sendable) async throws { assert(!isClosed) switch self.storage { + case let .nio(FileDescriptor): + var writer = FileDescriptor.bufferedWriter() + do { + let writtenBytesCount = try await writer.write(contentsOf: bytes) + assert(bytes.count == writtenBytesCount) + } catch { + throw error.attach(path) + } case let .real(fileDescriptor, queue): let path = self.path try await queue.scheduleOnQueue { diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/ReadableFileStream.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/ReadableFileStream.swift index 980ff2b..951d454 100644 --- a/Sources/Helpers/Vendor/_AsyncFileSystem/ReadableFileStream.swift +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/ReadableFileStream.swift @@ -10,23 +10,29 @@ // //===----------------------------------------------------------------------===// -import _Concurrency import SystemPackage +import _Concurrency +@preconcurrency import _NIOFileSystem + import class Dispatch.DispatchQueue /// Type-erasure wrapper over underlying file system readable streams. public enum ReadableFileStream: AsyncSequence { public typealias Element = ArraySlice + case nio(NIOReadableFileStream) case real(RealReadableFileStream) case mock(MockReadableFileStream) public enum Iterator: AsyncIteratorProtocol { + case nio(NIOReadableFileStream.Iterator) case real(RealReadableFileStream.Iterator) case mock(MockReadableFileStream.Iterator) public func next() async throws -> ArraySlice? { switch self { + case .nio(let local): + return try await local.next() case .real(let local): return try await local.next() case .mock(let virtual): @@ -37,6 +43,8 @@ public enum ReadableFileStream: AsyncSequence { public func makeAsyncIterator() -> Iterator { switch self { + case .nio(let real): + return .nio(real.makeAsyncIterator()) case .real(let real): return .real(real.makeAsyncIterator()) case .mock(let mock): @@ -45,6 +53,41 @@ public enum ReadableFileStream: AsyncSequence { } } +public struct NIOReadableFileStream: AsyncSequence { + + public typealias Element = ArraySlice + let fileDescriptor: ReadFileHandle + let readChunkSize: Int + + public final class Iterator: AsyncIteratorProtocol { + init(_ fileDescriptor: ReadFileHandle, readChunkSize: Int) { + self.chunkSize = readChunkSize + self.reader = fileDescriptor.bufferedReader() + } + private let chunkSize: Int + private var reader: BufferedReader + + public func next() async throws -> ArraySlice? { + let next = try await reader.read(.bytes(Int64(chunkSize))) + var buffer = [UInt8](repeating: 0, count: chunkSize) + guard next.writableBytes > 0 else { + return nil + } + buffer.withUnsafeMutableBytes { destBytes in + next.withUnsafeReadableBytes { srcBytes in + destBytes.copyBytes(from: srcBytes) + } + } + buffer.removeLast(chunkSize - next.writableBytes) + return buffer[...] + } + } + + public func makeAsyncIterator() -> Iterator { + Iterator(self.fileDescriptor, readChunkSize: self.readChunkSize) + } +} + /// A stream of file contents from the real file system provided by the OS. public struct RealReadableFileStream: AsyncSequence { public typealias Element = ArraySlice diff --git a/Sources/Helpers/Vendor/_AsyncFileSystem/SDKFileSystem.swift b/Sources/Helpers/Vendor/_AsyncFileSystem/SDKFileSystem.swift new file mode 100644 index 0000000..08a41ba --- /dev/null +++ b/Sources/Helpers/Vendor/_AsyncFileSystem/SDKFileSystem.swift @@ -0,0 +1,59 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift open source project +// +// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@preconcurrency import _NIOFileSystem + +public actor SDKFileSystem: AsyncFileSystem { + public init(readChunkSize: Int = defaultChunkSize) { + self.readChunkSize = readChunkSize + } + public static let defaultChunkSize = 512 * 1024 + let readChunkSize: Int + public func exists(_ path: SystemPackage.FilePath) async -> Bool { + do { + guard let _ = try await FileSystem.shared.info(forFileAt: path) else { + return false + } + return true + } catch { + return false + } + } + + public func withOpenReadableFile( + _ path: SystemPackage.FilePath, _ body: @Sendable (OpenReadableFile) async throws -> T + ) async throws -> T { + let fh = try await FileSystem.shared.openFile(forReadingAt: path) + do { + let result = try await body(OpenReadableFile(chunkSize: readChunkSize, fileHandle: .nio(fh))) + try await fh.close() + return result + } catch { + try await fh.close() + throw error.attach(path) + } + } + + public func withOpenWritableFile( + _ path: SystemPackage.FilePath, _ body: @Sendable (OpenWritableFile) async throws -> T + ) async throws -> T { + let fh = try await FileSystem.shared.openFile(forWritingAt: path, options: .newFile(replaceExisting: true)) + do { + let result = try await body(OpenWritableFile(storage:.nio(fh),path:path)) + try await fh.close() + return result + } catch { + try await fh.close() + throw error.attach(path) + } + } +} diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift index 41d9bd9..dd6c874 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift @@ -33,7 +33,7 @@ public extension Triple.Arch { public extension SwiftSDKGenerator { func run(recipe: SwiftSDKRecipe) async throws { - try await withQueryEngine(OSFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in + try await withQueryEngine(SDKFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in let httpClientType: HTTPClientProtocol.Type #if canImport(AsyncHTTPClient) httpClientType = HTTPClient.self