Use _NIOFileSystem for file system operations.

Motivation:
Add support for _NIOFileSystem to move away from OSFileSystem as per issue swiftlang#76.

Modifications:
Add SDKFileSystem that uses _NIOFileSystem. Change default AsyncFileSystem to SDKFileSystem

Result:

Using Swift NIO for file system operations.

Package.swift

@@ -63,6 +63,7 @@ let package = Package(
         .product(name: "Crypto", package: "swift-crypto"),
         .product(name: "Logging", package: "swift-log"),
         .product(name: "SystemPackage", package: "swift-system"),
+        .product(name: "_NIOFileSystem", package: "swift-nio"),
       ],
       exclude: ["Vendor/README.md"],
       swiftSettings: [

Sources/Helpers/Vendor/_AsyncFileSystem/OpenReadableFile.swift

@@ -10,6 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
+@preconcurrency import _NIOFileSystem
+
 import class Dispatch.DispatchQueue
 import struct SystemPackage.FileDescriptor
 
@@ -20,6 +22,7 @@ public struct OpenReadableFile: Sendable {
 
     /// Underlying storage for this file handle, dependent on the file system type that produced it.
     enum Storage {
+        case nio(ReadFileHandle)
         /// Operating system file descriptor and a queue used for reading from that file descriptor without blocking
         /// the Swift Concurrency thread pool.
         case real(FileDescriptor, DispatchQueue)
@@ -30,11 +33,17 @@ public struct OpenReadableFile: Sendable {
 
     /// Concrete instance of underlying file storage.
     let fileHandle: Storage
-
     /// Creates a readable ``AsyncSequence`` that can be iterated on to read from this file handle.
     /// - Returns: `ReadableFileStream` value conforming to ``AsyncSequence``, ready for asynchronous iteration.
     public func read() async throws -> ReadableFileStream {
         switch self.fileHandle {
+        case let .nio(fileDescriptor):
+            return ReadableFileStream.nio(
+                .init(
+                    fileDescriptor: fileDescriptor,
+                    readChunkSize: self.chunkSize
+                )
+            )
         case let .real(fileDescriptor, ioQueue):
             return ReadableFileStream.real(
                 .init(
@@ -43,7 +52,6 @@ public struct OpenReadableFile: Sendable {
                     readChunkSize: self.chunkSize
                 )
             )
-
         case .mock(let array):
             return ReadableFileStream.mock(.init(bytes: array, chunkSize: self.chunkSize))
         }

Sources/Helpers/Vendor/_AsyncFileSystem/OpenWritableFile.swift

@@ -9,6 +9,7 @@
 // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
 //
 //===----------------------------------------------------------------------===//
+@preconcurrency import _NIOFileSystem
 
 import class Dispatch.DispatchQueue
 @preconcurrency import struct SystemPackage.FileDescriptor
@@ -18,6 +19,7 @@ import struct SystemPackage.FilePath
 public actor OpenWritableFile: WritableStream {
     /// Underlying storage for this file handle, dependent on the file system type that produced it.
     enum Storage {
+        case nio(WriteFileHandle)
         /// Operating system file descriptor and a queue used for reading from that file descriptor without blocking
         /// the Swift Concurrency thread pool.
         case real(FileDescriptor, DispatchQueue)
@@ -48,6 +50,14 @@ public actor OpenWritableFile: WritableStream {
     public func write(_ bytes: some Collection<UInt8> & Sendable) async throws {
         assert(!isClosed)
         switch self.storage {
+        case let .nio(FileDescriptor):
+            var writer = FileDescriptor.bufferedWriter()
+            do {
+                let writtenBytesCount = try await writer.write(contentsOf: bytes)
+                assert(bytes.count == writtenBytesCount)
+            } catch {
+                throw error.attach(path)
+            }
         case let .real(fileDescriptor, queue):
             let path = self.path
             try await queue.scheduleOnQueue {

Sources/Helpers/Vendor/_AsyncFileSystem/ReadableFileStream.swift

@@ -10,23 +10,29 @@
 //
 //===----------------------------------------------------------------------===//
 
-import _Concurrency
 import SystemPackage
+import _Concurrency
+@preconcurrency import _NIOFileSystem
+
 import class Dispatch.DispatchQueue
 
 /// Type-erasure wrapper over underlying file system readable streams.
 public enum ReadableFileStream: AsyncSequence {
     public typealias Element = ArraySlice<UInt8>
 
+    case nio(NIOReadableFileStream)
     case real(RealReadableFileStream)
     case mock(MockReadableFileStream)
 
     public enum Iterator: AsyncIteratorProtocol {
+        case nio(NIOReadableFileStream.Iterator)
         case real(RealReadableFileStream.Iterator)
         case mock(MockReadableFileStream.Iterator)
 
         public func next() async throws -> ArraySlice<UInt8>? {
             switch self {
+            case .nio(let local):
+                return try await local.next()
             case .real(let local):
                 return try await local.next()
             case .mock(let virtual):
@@ -37,6 +43,8 @@ public enum ReadableFileStream: AsyncSequence {
 
     public func makeAsyncIterator() -> Iterator {
         switch self {
+        case .nio(let real):
+            return .nio(real.makeAsyncIterator())
         case .real(let real):
             return .real(real.makeAsyncIterator())
         case .mock(let mock):
@@ -45,6 +53,41 @@ public enum ReadableFileStream: AsyncSequence {
     }
 }
 
+public struct NIOReadableFileStream: AsyncSequence {
+
+    public typealias Element = ArraySlice<UInt8>
+    let fileDescriptor: ReadFileHandle
+    let readChunkSize: Int
+
+    public final class Iterator: AsyncIteratorProtocol {
+        init(_ fileDescriptor: ReadFileHandle, readChunkSize: Int) {
+            self.chunkSize = readChunkSize
+            self.reader = fileDescriptor.bufferedReader()
+        }
+        private let chunkSize: Int
+        private var reader: BufferedReader<ReadFileHandle>
+
+        public func next() async throws -> ArraySlice<UInt8>? {
+            let next = try await reader.read(.bytes(Int64(chunkSize)))
+            var buffer = [UInt8](repeating: 0, count: chunkSize)
+            guard next.writableBytes > 0 else {
+                    return nil
+            }
+            buffer.withUnsafeMutableBytes { destBytes in
+                    next.withUnsafeReadableBytes { srcBytes in
+                        destBytes.copyBytes(from: srcBytes)
+                    }
+                }
+            buffer.removeLast(chunkSize - next.writableBytes)
+            return buffer[...]
+        }
+    }
+
+    public func makeAsyncIterator() -> Iterator {
+        Iterator(self.fileDescriptor, readChunkSize: self.readChunkSize)
+    }
+}
+
 /// A stream of file contents from the real file system provided by the OS.
 public struct RealReadableFileStream: AsyncSequence {
     public typealias Element = ArraySlice<UInt8>

Sources/Helpers/Vendor/_AsyncFileSystem/SDKFileSystem.swift

@@ -0,0 +1,59 @@
+//===----------------------------------------------------------------------===//
+//
+// This source file is part of the Swift open source project
+//
+// Copyright (c) 2023-2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+@preconcurrency import _NIOFileSystem
+
+public actor SDKFileSystem: AsyncFileSystem {
+    public init(readChunkSize: Int = defaultChunkSize) {
+        self.readChunkSize = readChunkSize
+    }
+    public static let defaultChunkSize = 512 * 1024
+    let readChunkSize: Int
+    public func exists(_ path: SystemPackage.FilePath) async -> Bool {
+        do {
+            guard let _ = try await FileSystem.shared.info(forFileAt: path) else {
+                return false
+            }
+            return true
+        } catch {
+            return false
+        }
+    }
+
+    public func withOpenReadableFile<T: Sendable>(
+        _ path: SystemPackage.FilePath, _ body: @Sendable (OpenReadableFile) async throws -> T
+    ) async throws -> T {
+        let fh = try await FileSystem.shared.openFile(forReadingAt: path)
+        do {
+            let result = try await body(OpenReadableFile(chunkSize: readChunkSize, fileHandle: .nio(fh)))
+            try await fh.close()
+            return result
+        } catch {
+            try await fh.close()
+            throw error.attach(path)
+        }
+    }
+
+    public func withOpenWritableFile<T: Sendable>(
+        _ path: SystemPackage.FilePath, _ body: @Sendable (OpenWritableFile) async throws -> T
+    ) async throws -> T {
+        let fh = try await FileSystem.shared.openFile(forWritingAt: path, options: .newFile(replaceExisting: true))
+        do {
+            let result = try await body(OpenWritableFile(storage:.nio(fh),path:path))
+            try await fh.close()
+            return result
+        } catch {
+            try await fh.close()
+            throw error.attach(path)
+        }
+    }
+}

Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift

@@ -33,7 +33,7 @@ public extension Triple.Arch {
 
 public extension SwiftSDKGenerator {
   func run(recipe: SwiftSDKRecipe) async throws {
-    try await withQueryEngine(OSFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in
+    try await withQueryEngine(SDKFileSystem(), self.logger, cacheLocation: self.engineCachePath) { engine in
       let httpClientType: HTTPClientProtocol.Type
       #if canImport(AsyncHTTPClient)
       httpClientType = HTTPClient.self