From 9fcd3f2b5e7f38d37a0ddf433a3dc70484527a53 Mon Sep 17 00:00:00 2001 From: stevapple Date: Mon, 14 Sep 2020 00:30:45 +0800 Subject: [PATCH 01/24] Implement WindowsToolchain --- README.md | 2 +- Sources/SwiftDriver/CMakeLists.txt | 4 +- Sources/SwiftDriver/Driver/Driver.swift | 4 +- .../Jobs/CommandLineArguments.swift | 8 +- .../Jobs/Toolchain+InterpreterSupport.swift | 11 + .../Jobs/Toolchain+LinkerSupport.swift | 23 ++ .../Jobs/WindowsToolchain+LinkerSupport.swift | 220 ++++++++++++++++++ .../SwiftDriver/Toolchains/Toolchain.swift | 8 +- .../Toolchains/WindowsToolchain.swift | 94 ++++++++ 9 files changed, 363 insertions(+), 11 deletions(-) create mode 100644 Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift create mode 100644 Sources/SwiftDriver/Toolchains/WindowsToolchain.swift diff --git a/README.md b/README.md index d0d58d7d6..fa49d4e59 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ The goal of the new Swift driver is to provide a drop-in replacement for the exi * Platform support * [x] Teach the `DarwinToolchain` to also handle iOS, tvOS, watchOS * [x] Fill out the `GenericUnixToolchain` toolchain to get it working - * [ ] Implement a `WindowsToolchain` + * [x] Implement a `WindowsToolchain` * [x] Implement proper tokenization for response files * Compilation modes * [x] Batch mode diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt index 73a31a8f0..be7edfd67 100644 --- a/Sources/SwiftDriver/CMakeLists.txt +++ b/Sources/SwiftDriver/CMakeLists.txt @@ -1,6 +1,6 @@ # This source file is part of the Swift.org open source project # -# Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors +# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors # Licensed under Apache License v2.0 with Runtime Library Exception # # See http://swift.org/LICENSE.txt for license information @@ -57,10 +57,12 @@ add_library(SwiftDriver Jobs/Toolchain+InterpreterSupport.swift Jobs/Toolchain+LinkerSupport.swift Jobs/VerifyDebugInfoJob.swift + Jobs/WindowsToolchain+LinkerSupport.swift Toolchains/DarwinToolchain.swift Toolchains/GenericUnixToolchain.swift Toolchains/Toolchain.swift + Toolchains/WindowsToolchain.swift Utilities/Bits.swift Utilities/Bitstream.swift diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index 001648fe3..b2ed3695f 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -1702,7 +1702,7 @@ extension Triple { case .freeBSD, .haiku: return GenericUnixToolchain.self case .win32: - fatalError("Windows target not supported yet") + return WindowsToolchain.self default: diagnosticsEngine.emit(.error_unknown_target(triple)) throw Diagnostics.fatalError @@ -1715,7 +1715,7 @@ extension Driver { #if os(macOS) || os(iOS) || os(tvOS) || os(watchOS) static let defaultToolchainType: Toolchain.Type = DarwinToolchain.self #elseif os(Windows) - static let defaultToolchainType: Toolchain.Type = { fatalError("Windows target not supported yet") }() + static let defaultToolchainType: Toolchain.Type = WindowsToolchain.self #else static let defaultToolchainType: Toolchain.Type = GenericUnixToolchain.self #endif diff --git a/Sources/SwiftDriver/Jobs/CommandLineArguments.swift b/Sources/SwiftDriver/Jobs/CommandLineArguments.swift index 06e76a188..b560d4f8b 100644 --- a/Sources/SwiftDriver/Jobs/CommandLineArguments.swift +++ b/Sources/SwiftDriver/Jobs/CommandLineArguments.swift @@ -165,10 +165,10 @@ extension Array where Element == Job.ArgTemplate { var joinedArguments: String { return self.map { switch $0 { - case .flag(let string): - return string.spm_shellEscaped() - case .path(let path): - return path.name.spm_shellEscaped() + case .flag(let string): + return string.spm_shellEscaped() + case .path(let path): + return path.name.spm_shellEscaped() case .responseFilePath(let path): return "@\(path.name.spm_shellEscaped())" case let .joinedOptionAndPath(option, path): diff --git a/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift b/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift index fc9e9628c..ab860276e 100644 --- a/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift +++ b/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift @@ -81,3 +81,14 @@ extension GenericUnixToolchain { return envVars } } + +extension WindowsToolchain { + public func platformSpecificInterpreterEnvironmentVariables( + env: [String : String], + parsedOptions: inout ParsedOptions, + sdkPath: String?, + targetTriple: Triple) throws -> [String: String] { + // TODO: See whether Windows needs `platformSpecificInterpreterEnvironmentVariables` + return [:] + } +} diff --git a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift index a9ad52e12..6bdc469ba 100644 --- a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift @@ -30,6 +30,12 @@ extension Toolchain { resourceDirBase = sdkPath .appending(components: "usr", "lib", isShared ? "swift" : "swift_static") + } else if triple.isWindows, + let SDKROOT = env["SDKROOT"], + let sdkPath = try? AbsolutePath(validating: SDKROOT) { + resourceDirBase = sdkPath + .appending(components: "usr", "lib", + isShared ? "swift" : "swift_static") } else { resourceDirBase = try getToolPath(.swiftCompiler) .parentDirectory // remove /swift @@ -48,6 +54,21 @@ extension Toolchain { for triple: Triple, parsedOptions: inout ParsedOptions ) throws -> AbsolutePath { + /// Use the one in `VCTools` if exists on Windows. + if triple.isWindows, + let vctools = env["VCToolsInstallDir"], + let root = try? AbsolutePath(validating: vctools) { + let archName: String = { + switch triple.arch { + case .aarch64, .aarch64_32: return "arm64" + case .arm: return "arm" + case .x86: return "x86" + case nil, .x86_64: return "x64" + default: fatalError("unknown arch \(triple.archName) on Windows") + } + }() + return root.appending(components: "lib", archName) + } return try computeResourceDirPath(for: triple, parsedOptions: &parsedOptions, isShared: true) @@ -258,3 +279,5 @@ extension DarwinToolchain { } } + +// TODO: See whether Windows needs `addArgsToLinkStdlib`. diff --git a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift new file mode 100644 index 000000000..0f1d9ab12 --- /dev/null +++ b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift @@ -0,0 +1,220 @@ +//===---------------- WindowsToolchain+LinkerSupport.swift ----------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import TSCBasic +import SwiftOptions + +extension WindowsToolchain { + public func addPlatformSpecificLinkerArgs( + to commandLine: inout [Job.ArgTemplate], + parsedOptions: inout ParsedOptions, + linkerOutputType: LinkOutputType, + inputs: [TypedVirtualPath], + outputFile: VirtualPath, + shouldUseInputFileList: Bool, + sdkPath: String?, + sanitizers: Set, + targetInfo: FrontendTargetInfo + ) throws -> AbsolutePath { + let targetTriple = targetInfo.target.triple + switch linkerOutputType { + case .dynamicLibrary: + // Same options as an executable, just with '-shared' + commandLine.appendFlags("-parse-as-library", "-emit-library") + fallthrough + case .executable: + if !targetTriple.triple.isEmpty { + commandLine.appendFlag("-target") + commandLine.appendFlag(targetTriple.triple) + } + commandLine.appendFlag("-emit-executable") + default: + break + } + + switch linkerOutputType { + case .staticLibrary: + commandLine.append(.joinedOptionAndPath("-out:", outputFile)) + commandLine.append(contentsOf: inputs.map { .path($0.file) }) + return try getToolPath(.staticLinker) + default: + // Configure the toolchain. + // + // By default use the system `clang` to perform the link. We use `clang` for + // the driver here because we do not wish to select a particular C++ runtime. + // Furthermore, until C++ interop is enabled, we cannot have a dependency on + // C++ code from pure Swift code. If linked libraries are C++ based, they + // should properly link C++. In the case of static linking, the user can + // explicitly specify the C++ runtime to link against. This is particularly + // important for platforms like android where as it is a Linux platform, the + // default C++ runtime is `libstdc++` which is unsupported on the target but + // as the builds are usually cross-compiled from Linux, libstdc++ is going to + // be present. This results in linking the wrong version of libstdc++ + // generating invalid binaries. It is also possible to use different C++ + // runtimes than the default C++ runtime for the platform (e.g. libc++ on + // Windows rather than msvcprt). When C++ interop is enabled, we will need to + // surface this via a driver flag. For now, opt for the simpler approach of + // just using `clang` and avoid a dependency on the C++ runtime. + var clangPath = try getToolPath(.clang) + if let toolsDirPath = parsedOptions.getLastArgument(.toolsDirectory) { + // FIXME: What if this isn't an absolute path? + let toolsDir = try AbsolutePath(validating: toolsDirPath.asSingle) + + // If there is a clang in the toolchain folder, use that instead. + if let tool = lookupExecutablePath(filename: "clang.exe", searchPaths: [toolsDir]) { + clangPath = tool + } + + // Look for binutils in the toolchain folder. + commandLine.appendFlag("-B") + commandLine.appendPath(toolsDir) + } + + let staticStdlib = parsedOptions.hasFlag(positive: .staticStdlib, + negative: .noStaticStdlib, + default: false) + let staticExecutable = parsedOptions.hasFlag(positive: .staticExecutable, + negative: .noStaticExecutable, + default: false) + let hasRuntimeArgs = !(staticStdlib || staticExecutable) + + let runtimePaths = try runtimeLibraryPaths( + for: targetTriple, + parsedOptions: &parsedOptions, + sdkPath: sdkPath, + isShared: hasRuntimeArgs + ) + + if hasRuntimeArgs && targetTriple.environment != .android { + // FIXME: We probably shouldn't be adding an rpath here unless we know + // ahead of time the standard library won't be copied. + for path in runtimePaths { + commandLine.appendFlag(.Xlinker) + commandLine.appendFlag("-rpath") + commandLine.appendFlag(.Xlinker) + commandLine.appendPath(path) + } + } + + let sharedResourceDirPath = try computeResourceDirPath( + for: targetTriple, + parsedOptions: &parsedOptions, + isShared: true + ) + + let swiftrtPath = sharedResourceDirPath + .appending( + components: "x86_64", "swiftrt.o" + ) + commandLine.appendPath(swiftrtPath) + + let inputFiles: [Job.ArgTemplate] = inputs.compactMap { input in + // Autolink inputs are handled specially + if input.type == .autolink { + return .responseFilePath(input.file) + } else if input.type == .object { + return .path(input.file) + } else { + return nil + } + } + commandLine.append(contentsOf: inputFiles) + + let fSystemArgs = parsedOptions.arguments(for: .F, .Fsystem) + for opt in fSystemArgs { + if opt.option == .Fsystem { + commandLine.appendFlag("-iframework") + } else { + commandLine.appendFlag(.F) + } + commandLine.appendPath(try VirtualPath(path: opt.argument.asSingle)) + } + + // Add the runtime library link paths. + for path in runtimePaths { + commandLine.appendFlag(.L) + commandLine.appendPath(path) + } + + // Link the standard library. In two paths, we do this using a .lnk file + // if we're going that route, we'll set `linkFilePath` to the path to that + // file. + var linkFilePath: AbsolutePath? = try computeResourceDirPath( + for: targetTriple, + parsedOptions: &parsedOptions, + isShared: false + ) + + if staticExecutable { + linkFilePath = linkFilePath?.appending(component: "static-executable-args.lnk") + } else if staticStdlib { + linkFilePath = linkFilePath?.appending(component: "static-stdlib-args.lnk") + } else { + linkFilePath = nil + commandLine.appendFlag("-lswiftCore") + } + + if let linkFile = linkFilePath { + guard fileSystem.isFile(linkFile) else { + fatalError("\(linkFile.pathString) not found") + } + commandLine.append(.responseFilePath(.absolute(linkFile))) + } + + // Explicitly pass the target to the linker + commandLine.appendFlag("--target=\(targetTriple.triple)") + + // Delegate to Clang for sanitizers. It will figure out the correct linker + // options. + if linkerOutputType == .executable && !sanitizers.isEmpty { + let sanitizerNames = sanitizers + .map { $0.rawValue } + .sorted() // Sort so we get a stable, testable order + .joined(separator: ",") + commandLine.appendFlag("-fsanitize=\(sanitizerNames)") + + // The TSan runtime depends on the blocks runtime and libdispatch. + if sanitizers.contains(.thread) { + commandLine.appendFlag("-lBlocksRuntime") + commandLine.appendFlag("-ldispatch") + } + } + + if parsedOptions.hasArgument(.profileGenerate) { + let libProfile = sharedResourceDirPath + .parentDirectory // remove platform name + .appending(components: "clang", "lib", targetTriple.osName, + "libclangrt_profile-\(targetTriple.archName).a") + commandLine.appendPath(libProfile) + + // HACK: Hard-coded from llvm::getInstrProfRuntimeHookVarName() + commandLine.appendFlag("-u__llvm_profile_runtime") + } + + // Run clang++ in verbose mode if "-v" is set + try commandLine.appendLast(.v, from: &parsedOptions) + + // These custom arguments should be right before the object file at the + // end. + try commandLine.append( + contentsOf: parsedOptions.arguments(in: .linkerOption) + ) + try commandLine.appendAllArguments(.Xlinker, from: &parsedOptions) + try commandLine.appendAllArguments(.XclangLinker, from: &parsedOptions) + + // This should be the last option, for convenience in checking output. + commandLine.appendFlag(.o) + commandLine.appendPath(outputFile) + return clangPath + } + + } +} diff --git a/Sources/SwiftDriver/Toolchains/Toolchain.swift b/Sources/SwiftDriver/Toolchains/Toolchain.swift index 11ec15ab4..076d14f9f 100644 --- a/Sources/SwiftDriver/Toolchains/Toolchain.swift +++ b/Sources/SwiftDriver/Toolchains/Toolchain.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -135,7 +135,7 @@ extension Toolchain { return path } else if let path = try? xcrunFind(executable: executable) { return path - } else if !["swift-frontend", "swift"].contains(executable), + } else if !["swift-frontend", "swift", "swift-frontend.exe", "swift.exe"].contains(executable), let parentDirectory = try? getToolPath(.swiftCompiler).parentDirectory, parentDirectory != executableDir, let path = lookupExecutablePath(filename: executable, searchPaths: [parentDirectory]) { @@ -144,9 +144,11 @@ extension Toolchain { return path } else if let path = lookupExecutablePath(filename: executable, searchPaths: searchPaths) { return path + // Temporary shim: fall back to looking for "swift" before failing. } else if executable == "swift-frontend" { - // Temporary shim: fall back to looking for "swift" before failing. return try lookup(executable: "swift") + } else if executable == "swift-frontend.exe" { + return try lookup(executable: "swift.exe") } else if fallbackToExecutableDefaultPath { return AbsolutePath("/usr/bin/" + executable) } else { diff --git a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift new file mode 100644 index 000000000..d34309496 --- /dev/null +++ b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift @@ -0,0 +1,94 @@ +//===--------- WindowsToolchain.swift - Swift Windows Toolchain -----------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import TSCBasic + +/// Toolchain for Windows. +public final class WindowsToolchain: Toolchain { + public let env: [String: String] + + /// The executor used to run processes used to find tools and retrieve target info. + public let executor: DriverExecutor + + /// The file system to use for queries. + public let fileSystem: FileSystem + + /// Doubles as path cache and point for overriding normal lookup + private var toolPaths = [Tool: AbsolutePath]() + + public init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem = localFileSystem) { + self.env = env + self.executor = executor + self.fileSystem = fileSystem + } + + public func makeLinkerOutputFilename(moduleName: String, type: LinkOutputType) -> String { + switch type { + case .executable: return "\(moduleName).exe" + case .dynamicLibrary: return "\(moduleName).dll" + case .staticLibrary: return "\(moduleName).lib" + } + } + + /// Retrieve the absolute path for a given tool. + public func getToolPath(_ tool: Tool) throws -> AbsolutePath { + // Check the cache + if let toolPath = toolPaths[tool] { + return toolPath + } + let path = try lookupToolPath(tool) + // Cache the path + toolPaths[tool] = path + return path + } + + private func lookupToolPath(_ tool: Tool) throws -> AbsolutePath { + switch tool { + case .swiftCompiler: + return try lookup(executable: "swift-frontend.exe") + case .staticLinker: + return try lookup(executable: "llvm-lib.exe") + case .dynamicLinker: + // FIXME: This needs to look in the tools_directory first. + return try lookup(executable: "lld-link.exe") + case .clang: + return try lookup(executable: "clang.exe") + case .swiftAutolinkExtract: + return try lookup(executable: "swift-autolink-extract.exe") + case .dsymutil: + return try lookup(executable: "dsymutil.exe") + case .lldb: + return try lookup(executable: "lldb.exe") + case .dwarfdump: + return try lookup(executable: "llvm-dwarfdump.exe") + case .swiftHelp: + return try lookup(executable: "swift-help.exe") + } + } + + public func overrideToolPath(_ tool: Tool, path: AbsolutePath) { + toolPaths[tool] = path + } + + public func defaultSDKPath(_ target: Triple?) throws -> AbsolutePath? { + return nil + } + + public var shouldStoreInvocationInDebugInfo: Bool { false } + + public func runtimeLibraryName( + for sanitizer: Sanitizer, + targetTriple: Triple, + isShared: Bool + ) throws -> String { + return "clang_rt.\(sanitizer.libraryName)-\(targetTriple.archName).dll" + } +} From 9076ab2d5b55ba9f7f9fbb3a2fa3d27d0b91a066 Mon Sep 17 00:00:00 2001 From: stevapple Date: Mon, 14 Sep 2020 00:59:58 +0800 Subject: [PATCH 02/24] Fix swiftrtPath --- .../Jobs/WindowsToolchain+LinkerSupport.swift | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift index 0f1d9ab12..03714d22e 100644 --- a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift @@ -93,7 +93,7 @@ extension WindowsToolchain { isShared: hasRuntimeArgs ) - if hasRuntimeArgs && targetTriple.environment != .android { + if hasRuntimeArgs { // FIXME: We probably shouldn't be adding an rpath here unless we know // ahead of time the standard library won't be copied. for path in runtimePaths { @@ -110,10 +110,9 @@ extension WindowsToolchain { isShared: true ) - let swiftrtPath = sharedResourceDirPath - .appending( - components: "x86_64", "swiftrt.o" - ) + let swiftrtPath = sharedResourceDirPath.appending( + components: "windows", targetTriple.archName, "swiftrt.obj" + ) commandLine.appendPath(swiftrtPath) let inputFiles: [Job.ArgTemplate] = inputs.compactMap { input in From 2c9716b749f3925cb3047741a97097105fbc9743 Mon Sep 17 00:00:00 2001 From: stevapple Date: Mon, 14 Sep 2020 13:40:18 +0800 Subject: [PATCH 03/24] Fix various problems --- .../Jobs/Toolchain+LinkerSupport.swift | 4 +-- .../Jobs/WindowsToolchain+LinkerSupport.swift | 26 ++++++------------- .../Toolchains/WindowsToolchain.swift | 19 ++++++++++---- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift index 6bdc469ba..4dd95e0b9 100644 --- a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift @@ -60,7 +60,7 @@ extension Toolchain { let root = try? AbsolutePath(validating: vctools) { let archName: String = { switch triple.arch { - case .aarch64, .aarch64_32: return "arm64" + case .aarch64: return "arm64" case .arm: return "arm" case .x86: return "x86" case nil, .x86_64: return "x64" @@ -280,4 +280,4 @@ extension DarwinToolchain { } -// TODO: See whether Windows needs `addArgsToLinkStdlib`. +// TODO: Implement `addArgsToLinkStdlib` for `WindowsToolchain`. diff --git a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift index 03714d22e..d771479e5 100644 --- a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift @@ -27,24 +27,26 @@ extension WindowsToolchain { let targetTriple = targetInfo.target.triple switch linkerOutputType { case .dynamicLibrary: - // Same options as an executable, just with '-shared' commandLine.appendFlags("-parse-as-library", "-emit-library") - fallthrough case .executable: if !targetTriple.triple.isEmpty { commandLine.appendFlag("-target") commandLine.appendFlag(targetTriple.triple) } - commandLine.appendFlag("-emit-executable") + commandLine.appendFlag("-emit-executable") default: break } - + switch linkerOutputType { case .staticLibrary: - commandLine.append(.joinedOptionAndPath("-out:", outputFile)) - commandLine.append(contentsOf: inputs.map { .path($0.file) }) + commandLine.append(.joinedOptionAndPath("-out:", outputFile)) + commandLine.append(contentsOf: inputs.map { .path($0.file) }) + if commandLine.contains(.flag("-use-ld=lld")) { + return try lookup(executable: "llvm-lib.exe") + } return try getToolPath(.staticLinker) + // TODO: Check for `-use-ld=lld`. default: // Configure the toolchain. // @@ -93,17 +95,6 @@ extension WindowsToolchain { isShared: hasRuntimeArgs ) - if hasRuntimeArgs { - // FIXME: We probably shouldn't be adding an rpath here unless we know - // ahead of time the standard library won't be copied. - for path in runtimePaths { - commandLine.appendFlag(.Xlinker) - commandLine.appendFlag("-rpath") - commandLine.appendFlag(.Xlinker) - commandLine.appendPath(path) - } - } - let sharedResourceDirPath = try computeResourceDirPath( for: targetTriple, parsedOptions: &parsedOptions, @@ -214,6 +205,5 @@ extension WindowsToolchain { commandLine.appendPath(outputFile) return clangPath } - } } diff --git a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift index d34309496..d7f29c4b1 100644 --- a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift @@ -55,16 +55,16 @@ public final class WindowsToolchain: Toolchain { case .swiftCompiler: return try lookup(executable: "swift-frontend.exe") case .staticLinker: - return try lookup(executable: "llvm-lib.exe") + return try lookup(executable: "lib.exe") case .dynamicLinker: // FIXME: This needs to look in the tools_directory first. - return try lookup(executable: "lld-link.exe") + return try lookup(executable: "link.exe") case .clang: return try lookup(executable: "clang.exe") case .swiftAutolinkExtract: - return try lookup(executable: "swift-autolink-extract.exe") + fatalError("Trying to look up \"swift-autolink-extract\" on Windows") case .dsymutil: - return try lookup(executable: "dsymutil.exe") + fatalError("Trying to look up \"dsymutil\" on Windows") case .lldb: return try lookup(executable: "lldb.exe") case .dwarfdump: @@ -89,6 +89,15 @@ public final class WindowsToolchain: Toolchain { targetTriple: Triple, isShared: Bool ) throws -> String { - return "clang_rt.\(sanitizer.libraryName)-\(targetTriple.archName).dll" + let archName: String = { + switch targetTriple.arch { + case .aarch64: return "aarch64" + case .arm: return "armv7" + case .x86: return "i386" + case nil, .x86_64: return "x86_64" + default: fatalError("unknown arch \(targetTriple.archName) on Windows") + } + }() + return "clang_rt.\(sanitizer.libraryName)-\(archName).lib" } } From 8b7b0df2f859fd57fa9472e685a7714a702d9097 Mon Sep 17 00:00:00 2001 From: stevapple Date: Mon, 14 Sep 2020 15:10:07 +0800 Subject: [PATCH 04/24] Derive executable file name from Toolchain --- .../Toolchains/DarwinToolchain.swift | 9 ++++++--- .../Toolchains/GenericUnixToolchain.swift | 3 +++ Sources/SwiftDriver/Toolchains/Toolchain.swift | 15 ++++++++------- .../Toolchains/WindowsToolchain.swift | 17 ++++++++++------- 4 files changed, 27 insertions(+), 17 deletions(-) diff --git a/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift b/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift index e63ed010b..c6627b37e 100644 --- a/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift @@ -20,15 +20,18 @@ import SwiftOptions public final class DarwinToolchain: Toolchain { public let env: [String: String] - /// Doubles as path cache and point for overriding normal lookup - private var toolPaths = [Tool: AbsolutePath]() - /// The executor used to run processes used to find tools and retrieve target info. public let executor: DriverExecutor /// The file system to use for any file operations. public let fileSystem: FileSystem + /// The suffix of executable files. + public let executableSuffix = "" + + /// Doubles as path cache and point for overriding normal lookup + private var toolPaths = [Tool: AbsolutePath]() + public init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem = localFileSystem) { self.env = env self.executor = executor diff --git a/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift b/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift index 09ca4d00a..5a9bf124f 100644 --- a/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift @@ -21,6 +21,9 @@ public final class GenericUnixToolchain: Toolchain { /// The file system to use for queries. public let fileSystem: FileSystem + /// The suffix of executable files. + public let executableSuffix = "" + /// Doubles as path cache and point for overriding normal lookup private var toolPaths = [Tool: AbsolutePath]() diff --git a/Sources/SwiftDriver/Toolchains/Toolchain.swift b/Sources/SwiftDriver/Toolchains/Toolchain.swift index 076d14f9f..d1b434ae2 100644 --- a/Sources/SwiftDriver/Toolchains/Toolchain.swift +++ b/Sources/SwiftDriver/Toolchains/Toolchain.swift @@ -38,6 +38,8 @@ public protocol Toolchain { var executor: DriverExecutor { get } + var executableSuffix: String { get } + /// Retrieve the absolute path to a particular tool. func getToolPath(_ tool: Tool) throws -> AbsolutePath @@ -129,26 +131,25 @@ extension Toolchain { /// looks in the `executableDir`, `xcrunFind` or in the `searchPaths`. /// - Parameter executable: executable to look for [i.e. `swift`]. func lookup(executable: String) throws -> AbsolutePath { + let filename = executable + executableSuffix if let overrideString = envVar(forExecutable: executable) { return try AbsolutePath(validating: overrideString) - } else if let path = lookupExecutablePath(filename: executable, searchPaths: [executableDir]) { + } else if let path = lookupExecutablePath(filename: filename, searchPaths: [executableDir]) { return path } else if let path = try? xcrunFind(executable: executable) { return path - } else if !["swift-frontend", "swift", "swift-frontend.exe", "swift.exe"].contains(executable), + } else if !["swift-frontend", "swift"].contains(executable), let parentDirectory = try? getToolPath(.swiftCompiler).parentDirectory, parentDirectory != executableDir, - let path = lookupExecutablePath(filename: executable, searchPaths: [parentDirectory]) { + let path = lookupExecutablePath(filename: filename, searchPaths: [parentDirectory]) { // If the driver library's client and the frontend are in different directories, // try looking for tools next to the frontend. return path - } else if let path = lookupExecutablePath(filename: executable, searchPaths: searchPaths) { + } else if let path = lookupExecutablePath(filename: filename, searchPaths: searchPaths) { return path - // Temporary shim: fall back to looking for "swift" before failing. } else if executable == "swift-frontend" { + // Temporary shim: fall back to looking for "swift" before failing. return try lookup(executable: "swift") - } else if executable == "swift-frontend.exe" { - return try lookup(executable: "swift.exe") } else if fallbackToExecutableDefaultPath { return AbsolutePath("/usr/bin/" + executable) } else { diff --git a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift index d7f29c4b1..7ae4db01b 100644 --- a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift @@ -21,6 +21,9 @@ public final class WindowsToolchain: Toolchain { /// The file system to use for queries. public let fileSystem: FileSystem + /// The suffix of executable files. + public let executableSuffix = ".exe" + /// Doubles as path cache and point for overriding normal lookup private var toolPaths = [Tool: AbsolutePath]() @@ -53,24 +56,24 @@ public final class WindowsToolchain: Toolchain { private func lookupToolPath(_ tool: Tool) throws -> AbsolutePath { switch tool { case .swiftCompiler: - return try lookup(executable: "swift-frontend.exe") + return try lookup(executable: "swift-frontend") case .staticLinker: - return try lookup(executable: "lib.exe") + return try lookup(executable: "lib") case .dynamicLinker: // FIXME: This needs to look in the tools_directory first. - return try lookup(executable: "link.exe") + return try lookup(executable: "link") case .clang: - return try lookup(executable: "clang.exe") + return try lookup(executable: "clang") case .swiftAutolinkExtract: fatalError("Trying to look up \"swift-autolink-extract\" on Windows") case .dsymutil: fatalError("Trying to look up \"dsymutil\" on Windows") case .lldb: - return try lookup(executable: "lldb.exe") + return try lookup(executable: "lldb") case .dwarfdump: - return try lookup(executable: "llvm-dwarfdump.exe") + return try lookup(executable: "llvm-dwarfdump") case .swiftHelp: - return try lookup(executable: "swift-help.exe") + return try lookup(executable: "swift-help") } } From 3c0feb330778fcf501bdb07519959ed91371e174 Mon Sep 17 00:00:00 2001 From: stevapple Date: Tue, 15 Sep 2020 00:22:40 +0800 Subject: [PATCH 05/24] Fix addPlatformSpecificLinkerArgs for WindowsToolchain --- .../Jobs/Toolchain+LinkerSupport.swift | 15 - .../Jobs/WindowsToolchain+LinkerSupport.swift | 293 ++++++++---------- .../Toolchains/WindowsToolchain.swift | 23 +- 3 files changed, 145 insertions(+), 186 deletions(-) diff --git a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift index 4dd95e0b9..44702d3ff 100644 --- a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift @@ -54,21 +54,6 @@ extension Toolchain { for triple: Triple, parsedOptions: inout ParsedOptions ) throws -> AbsolutePath { - /// Use the one in `VCTools` if exists on Windows. - if triple.isWindows, - let vctools = env["VCToolsInstallDir"], - let root = try? AbsolutePath(validating: vctools) { - let archName: String = { - switch triple.arch { - case .aarch64: return "arm64" - case .arm: return "arm" - case .x86: return "x86" - case nil, .x86_64: return "x64" - default: fatalError("unknown arch \(triple.archName) on Windows") - } - }() - return root.appending(components: "lib", archName) - } return try computeResourceDirPath(for: triple, parsedOptions: &parsedOptions, isShared: true) diff --git a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift index d771479e5..ad520fd10 100644 --- a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift @@ -25,185 +25,158 @@ extension WindowsToolchain { targetInfo: FrontendTargetInfo ) throws -> AbsolutePath { let targetTriple = targetInfo.target.triple + switch linkerOutputType { case .dynamicLibrary: - commandLine.appendFlags("-parse-as-library", "-emit-library") + commandLine.appendFlags("-Xlinker", "-dll") + fallthrough case .executable: if !targetTriple.triple.isEmpty { commandLine.appendFlag("-target") commandLine.appendFlag(targetTriple.triple) } - commandLine.appendFlag("-emit-executable") - default: - break - } - - switch linkerOutputType { - case .staticLibrary: - commandLine.append(.joinedOptionAndPath("-out:", outputFile)) - commandLine.append(contentsOf: inputs.map { .path($0.file) }) - if commandLine.contains(.flag("-use-ld=lld")) { - return try lookup(executable: "llvm-lib.exe") - } - return try getToolPath(.staticLinker) - // TODO: Check for `-use-ld=lld`. - default: - // Configure the toolchain. - // - // By default use the system `clang` to perform the link. We use `clang` for - // the driver here because we do not wish to select a particular C++ runtime. - // Furthermore, until C++ interop is enabled, we cannot have a dependency on - // C++ code from pure Swift code. If linked libraries are C++ based, they - // should properly link C++. In the case of static linking, the user can - // explicitly specify the C++ runtime to link against. This is particularly - // important for platforms like android where as it is a Linux platform, the - // default C++ runtime is `libstdc++` which is unsupported on the target but - // as the builds are usually cross-compiled from Linux, libstdc++ is going to - // be present. This results in linking the wrong version of libstdc++ - // generating invalid binaries. It is also possible to use different C++ - // runtimes than the default C++ runtime for the platform (e.g. libc++ on - // Windows rather than msvcprt). When C++ interop is enabled, we will need to - // surface this via a driver flag. For now, opt for the simpler approach of - // just using `clang` and avoid a dependency on the C++ runtime. - var clangPath = try getToolPath(.clang) - if let toolsDirPath = parsedOptions.getLastArgument(.toolsDirectory) { - // FIXME: What if this isn't an absolute path? - let toolsDir = try AbsolutePath(validating: toolsDirPath.asSingle) - - // If there is a clang in the toolchain folder, use that instead. - if let tool = lookupExecutablePath(filename: "clang.exe", searchPaths: [toolsDir]) { - clangPath = tool - } - - // Look for binutils in the toolchain folder. - commandLine.appendFlag("-B") - commandLine.appendPath(toolsDir) + // Configure the toolchain. + // + // By default use the system `clang` to perform the link. We use `clang` for + // the driver here because we do not wish to select a particular C++ runtime. + // Furthermore, until C++ interop is enabled, we cannot have a dependency on + // C++ code from pure Swift code. If linked libraries are C++ based, they + // should properly link C++. In the case of static linking, the user can + // explicitly specify the C++ runtime to link against. This is particularly + // important for platforms like android where as it is a Linux platform, the + // default C++ runtime is `libstdc++` which is unsupported on the target but + // as the builds are usually cross-compiled from Linux, libstdc++ is going to + // be present. This results in linking the wrong version of libstdc++ + // generating invalid binaries. It is also possible to use different C++ + // runtimes than the default C++ runtime for the platform (e.g. libc++ on + // Windows rather than msvcprt). When C++ interop is enabled, we will need to + // surface this via a driver flag. For now, opt for the simpler approach of + // just using `clang` and avoid a dependency on the C++ runtime. + var clangPath = try getToolPath(.clang) + if let toolsDirPath = parsedOptions.getLastArgument(.toolsDirectory) { + // FIXME: What if this isn't an absolute path? + let toolsDir = try AbsolutePath(validating: toolsDirPath.asSingle) + + // If there is a clang in the toolchain folder, use that instead. + if let tool = lookupExecutablePath(filename: "clang.exe", searchPaths: [toolsDir]) { + clangPath = tool } - let staticStdlib = parsedOptions.hasFlag(positive: .staticStdlib, - negative: .noStaticStdlib, - default: false) - let staticExecutable = parsedOptions.hasFlag(positive: .staticExecutable, - negative: .noStaticExecutable, - default: false) - let hasRuntimeArgs = !(staticStdlib || staticExecutable) - - let runtimePaths = try runtimeLibraryPaths( - for: targetTriple, - parsedOptions: &parsedOptions, - sdkPath: sdkPath, - isShared: hasRuntimeArgs - ) - - let sharedResourceDirPath = try computeResourceDirPath( - for: targetTriple, - parsedOptions: &parsedOptions, - isShared: true - ) - - let swiftrtPath = sharedResourceDirPath.appending( - components: "windows", targetTriple.archName, "swiftrt.obj" - ) - commandLine.appendPath(swiftrtPath) - - let inputFiles: [Job.ArgTemplate] = inputs.compactMap { input in - // Autolink inputs are handled specially - if input.type == .autolink { - return .responseFilePath(input.file) - } else if input.type == .object { - return .path(input.file) - } else { - return nil - } - } - commandLine.append(contentsOf: inputFiles) - - let fSystemArgs = parsedOptions.arguments(for: .F, .Fsystem) - for opt in fSystemArgs { - if opt.option == .Fsystem { - commandLine.appendFlag("-iframework") - } else { - commandLine.appendFlag(.F) - } - commandLine.appendPath(try VirtualPath(path: opt.argument.asSingle)) - } - - // Add the runtime library link paths. - for path in runtimePaths { - commandLine.appendFlag(.L) - commandLine.appendPath(path) - } + // Look for binutils in the toolchain folder. + commandLine.appendFlag("-B") + commandLine.appendPath(toolsDir) + } - // Link the standard library. In two paths, we do this using a .lnk file - // if we're going that route, we'll set `linkFilePath` to the path to that - // file. - var linkFilePath: AbsolutePath? = try computeResourceDirPath( - for: targetTriple, - parsedOptions: &parsedOptions, - isShared: false - ) - - if staticExecutable { - linkFilePath = linkFilePath?.appending(component: "static-executable-args.lnk") - } else if staticStdlib { - linkFilePath = linkFilePath?.appending(component: "static-stdlib-args.lnk") + let linker: String + if let arg = parsedOptions.getLastArgument(.useLd) { + linker = arg.asSingle + } else { + linker = "link" + } + commandLine.appendFlag("-fuse-ld=\(linker)") + + let staticStdlib = parsedOptions.hasFlag(positive: .staticStdlib, + negative: .noStaticStdlib, + default: false) + let staticExecutable = parsedOptions.hasFlag(positive: .staticExecutable, + negative: .noStaticExecutable, + default: false) + let hasRuntimeArgs = !(staticStdlib || staticExecutable) + + let runtimePaths = try runtimeLibraryPaths( + for: targetTriple, + parsedOptions: &parsedOptions, + sdkPath: sdkPath, + isShared: hasRuntimeArgs + ) + + let sharedResourceDirPath = try computeResourceDirPath( + for: targetTriple, + parsedOptions: &parsedOptions, + isShared: true + ) + + let swiftrtPath = sharedResourceDirPath.appending( + components: targetTriple.archName, "swiftrt.obj" + ) + commandLine.appendPath(swiftrtPath) + + let inputFiles: [Job.ArgTemplate] = inputs.compactMap { input in + // Autolink inputs are handled specially + if input.type == .autolink { + return .responseFilePath(input.file) + } else if input.type == .object { + return .path(input.file) } else { - linkFilePath = nil - commandLine.appendFlag("-lswiftCore") + return nil } + } + commandLine.append(contentsOf: inputFiles) - if let linkFile = linkFilePath { - guard fileSystem.isFile(linkFile) else { - fatalError("\(linkFile.pathString) not found") - } - commandLine.append(.responseFilePath(.absolute(linkFile))) + let fSystemArgs = parsedOptions.arguments(for: .F, .Fsystem) + for opt in fSystemArgs { + if opt.option == .Fsystem { + commandLine.appendFlag("-iframework") + } else { + commandLine.appendFlag(.F) } + commandLine.appendPath(try VirtualPath(path: opt.argument.asSingle)) + } - // Explicitly pass the target to the linker - commandLine.appendFlag("--target=\(targetTriple.triple)") - - // Delegate to Clang for sanitizers. It will figure out the correct linker - // options. - if linkerOutputType == .executable && !sanitizers.isEmpty { - let sanitizerNames = sanitizers - .map { $0.rawValue } - .sorted() // Sort so we get a stable, testable order - .joined(separator: ",") - commandLine.appendFlag("-fsanitize=\(sanitizerNames)") - - // The TSan runtime depends on the blocks runtime and libdispatch. - if sanitizers.contains(.thread) { - commandLine.appendFlag("-lBlocksRuntime") - commandLine.appendFlag("-ldispatch") - } - } + // Add the runtime library link paths. + for path in runtimePaths { + commandLine.appendFlag(.L) + commandLine.appendPath(path) + } - if parsedOptions.hasArgument(.profileGenerate) { - let libProfile = sharedResourceDirPath - .parentDirectory // remove platform name - .appending(components: "clang", "lib", targetTriple.osName, - "libclangrt_profile-\(targetTriple.archName).a") - commandLine.appendPath(libProfile) + if hasRuntimeArgs { + commandLine.appendFlag("-lswiftCore") + } - // HACK: Hard-coded from llvm::getInstrProfRuntimeHookVarName() - commandLine.appendFlag("-u__llvm_profile_runtime") + // Explicitly pass the target to the linker + commandLine.appendFlags("-target", targetTriple.triple) + + // Delegate to Clang for sanitizers. It will figure out the correct linker + // options. + if linkerOutputType == .executable && !sanitizers.isEmpty { + let sanitizerNames = sanitizers + .map { $0.rawValue } + .sorted() // Sort so we get a stable, testable order + .joined(separator: ",") + commandLine.appendFlag("-fsanitize=\(sanitizerNames)") + + // The TSan runtime depends on the blocks runtime and libdispatch. + if sanitizers.contains(.thread) { + commandLine.appendFlag("-lBlocksRuntime") + commandLine.appendFlag("-ldispatch") } + } + + if parsedOptions.hasArgument(.profileGenerate) { + let libProfile = try clangLibraryPath(for: targetTriple, parsedOptions: &parsedOptions) + .appending(components: "clang_rt.profile-\(archName(for: targetTriple)).lib") + commandLine.appendPath(libProfile) + } - // Run clang++ in verbose mode if "-v" is set - try commandLine.appendLast(.v, from: &parsedOptions) - - // These custom arguments should be right before the object file at the - // end. - try commandLine.append( - contentsOf: parsedOptions.arguments(in: .linkerOption) - ) - try commandLine.appendAllArguments(.Xlinker, from: &parsedOptions) - try commandLine.appendAllArguments(.XclangLinker, from: &parsedOptions) - - // This should be the last option, for convenience in checking output. - commandLine.appendFlag(.o) - commandLine.appendPath(outputFile) - return clangPath + // Run clang++ in verbose mode if "-v" is set + try commandLine.appendLast(.v, from: &parsedOptions) + + // These custom arguments should be right before the object file at the + // end. + try commandLine.append( + contentsOf: parsedOptions.arguments(in: .linkerOption) + ) + try commandLine.appendAllArguments(.Xlinker, from: &parsedOptions) + try commandLine.appendAllArguments(.XclangLinker, from: &parsedOptions) + + // This should be the last option, for convenience in checking output. + commandLine.appendFlag(.o) + commandLine.appendPath(outputFile) + return clangPath + case .staticLibrary: + commandLine.append(.joinedOptionAndPath("-out:", outputFile)) + commandLine.append(contentsOf: inputs.map { .path($0.file) }) + return try getToolPath(.staticLinker) } } } diff --git a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift index 7ae4db01b..88904b853 100644 --- a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift @@ -27,6 +27,16 @@ public final class WindowsToolchain: Toolchain { /// Doubles as path cache and point for overriding normal lookup private var toolPaths = [Tool: AbsolutePath]() + public func archName(for triple: Triple) -> String { + switch triple.arch { + case .aarch64: return "aarch64" + case .arm: return "armv7" + case .x86: return "i386" + case nil, .x86_64: return "x86_64" + default: fatalError("unknown arch \(triple.archName) on Windows") + } + } + public init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem = localFileSystem) { self.env = env self.executor = executor @@ -37,7 +47,7 @@ public final class WindowsToolchain: Toolchain { switch type { case .executable: return "\(moduleName).exe" case .dynamicLibrary: return "\(moduleName).dll" - case .staticLibrary: return "\(moduleName).lib" + case .staticLibrary: return "lib\(moduleName).lib" } } @@ -92,15 +102,6 @@ public final class WindowsToolchain: Toolchain { targetTriple: Triple, isShared: Bool ) throws -> String { - let archName: String = { - switch targetTriple.arch { - case .aarch64: return "aarch64" - case .arm: return "armv7" - case .x86: return "i386" - case nil, .x86_64: return "x86_64" - default: fatalError("unknown arch \(targetTriple.archName) on Windows") - } - }() - return "clang_rt.\(sanitizer.libraryName)-\(archName).lib" + return "clang_rt.\(sanitizer.libraryName)-\(archName(for: targetTriple)).lib" } } From 633801de73bead02059c1e4853c3c889284f26a4 Mon Sep 17 00:00:00 2001 From: stevapple Date: Tue, 15 Sep 2020 07:54:08 +0800 Subject: [PATCH 06/24] Fix clangLibraryPath for Windows --- .../Jobs/Toolchain+LinkerSupport.swift | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift index 44702d3ff..f495f09d5 100644 --- a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift @@ -54,12 +54,20 @@ extension Toolchain { for triple: Triple, parsedOptions: inout ParsedOptions ) throws -> AbsolutePath { - return try computeResourceDirPath(for: triple, - parsedOptions: &parsedOptions, - isShared: true) - .parentDirectory // Remove platform name. - .appending(components: "clang", "lib", - triple.platformName(conflatingDarwin: true)!) + #if os(Windows) + return try getToolPath(.swiftCompiler) + .parentDirectory // remove /swift + .parentDirectory // remove /bin + .appending(components: "lib", "swift", "clang", "lib", + triple.platformName(conflatingDarwin: true)!) + #else + return try computeResourceDirPath(for: triple, + parsedOptions: &parsedOptions, + isShared: true) + .parentDirectory // Remove platform name. + .appending(components: "clang", "lib", + triple.platformName(conflatingDarwin: true)!) + #endif } func runtimeLibraryPaths( From 59877f13445b6c1b81b6e296975dd5402035ab67 Mon Sep 17 00:00:00 2001 From: stevapple Date: Thu, 17 Sep 2020 00:38:16 +0800 Subject: [PATCH 07/24] Disable profiling support on Windows --- Sources/SwiftDriver/Driver/Driver.swift | 25 +++++++++++++++++-- .../Jobs/WindowsToolchain+LinkerSupport.swift | 5 ++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index b2ed3695f..66f7a5250 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -361,7 +361,8 @@ public struct Driver { Self.validateProfilingArgs(&parsedOptions, fileSystem: fileSystem, workingDirectory: workingDirectory, - diagnosticEngine: diagnosticEngine) + diagnosticEngine: diagnosticEngine, + targetTriple: self.frontendTargetInfo.target.triple) Self.validateCompilationConditionArgs(&parsedOptions, diagnosticEngine: diagnosticEngine) Self.validateFrameworkSearchPathArgs(&parsedOptions, diagnosticEngine: diagnosticEngine) Self.validateCoverageArgs(&parsedOptions, diagnosticsEngine: diagnosticEngine) @@ -1642,12 +1643,32 @@ extension Driver { static func validateProfilingArgs(_ parsedOptions: inout ParsedOptions, fileSystem: FileSystem, workingDirectory: AbsolutePath?, - diagnosticEngine: DiagnosticsEngine) { + diagnosticEngine: DiagnosticsEngine, + targetTriple: Triple) { if parsedOptions.hasArgument(.profileGenerate) && parsedOptions.hasArgument(.profileUse) { diagnosticEngine.emit(Error.conflictingOptions(.profileGenerate, .profileUse)) } + // Windows executables should be profiled with ETW, whose support needs to be + // implemented before we can enable the option. + if targetTriple.isWindows { + if parsedOptions.hasArgument(.profileGenerate) { + diagnosticEngine.emit( + .error_unsupported_opt_for_target( + arg: "-profile-generate", + target: targetTriple) + ) + } + if parsedOptions.hasArgument(.profileUse) { + diagnosticEngine.emit( + .error_unsupported_opt_for_target( + arg: "-profile-use=", + target: targetTriple) + ) + } + } + if let profileArgs = parsedOptions.getLastArgument(.profileUse)?.asMultiple, let workingDirectory = workingDirectory ?? fileSystem.currentWorkingDirectory { for profilingData in profileArgs { diff --git a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift index ad520fd10..72cb3d01a 100644 --- a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift @@ -153,9 +153,8 @@ extension WindowsToolchain { } if parsedOptions.hasArgument(.profileGenerate) { - let libProfile = try clangLibraryPath(for: targetTriple, parsedOptions: &parsedOptions) - .appending(components: "clang_rt.profile-\(archName(for: targetTriple)).lib") - commandLine.appendPath(libProfile) + // Profiling support for Windows isn't ready yet. It should have been disabled. + fatalError("Profiling support should have been disabled on Windows.") } // Run clang++ in verbose mode if "-v" is set From 4b69ec6d628231ff2c83c7546feb73089ff50a6a Mon Sep 17 00:00:00 2001 From: stevapple Date: Thu, 17 Sep 2020 00:38:32 +0800 Subject: [PATCH 08/24] Disable unsupported sanitizers on Windows --- Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift index f495f09d5..75d350bad 100644 --- a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift @@ -119,6 +119,13 @@ extension Toolchain { parsedOptions: inout ParsedOptions, isShared: Bool ) throws -> Bool { + // Currently only ASAN is supported on Windows, but clang builds may + // include runtime libraries for unsupported sanitizers. Manually + // disable unsupported sanitizers. + if targetTriple.isWindows && sanitizer != .address { + return false + } + let runtimeName = try runtimeLibraryName( for: sanitizer, targetTriple: targetTriple, From d8c4f60fe2d577bf8c37291fba16cafaeba6ba75 Mon Sep 17 00:00:00 2001 From: stevapple Date: Thu, 17 Sep 2020 14:33:09 +0800 Subject: [PATCH 09/24] Fix cross-compilation problems --- Sources/SwiftDriver/Driver/Driver.swift | 30 ++++----------- .../Jobs/Toolchain+LinkerSupport.swift | 7 ---- .../Toolchains/DarwinToolchain.swift | 3 -- .../Toolchains/GenericUnixToolchain.swift | 3 -- .../SwiftDriver/Toolchains/Toolchain.swift | 9 +++-- .../Toolchains/WindowsToolchain.swift | 37 ++++++++++++++++--- 6 files changed, 44 insertions(+), 45 deletions(-) diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index ebdeddcf4..712e6d302 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -373,8 +373,7 @@ public struct Driver { Self.validateProfilingArgs(&parsedOptions, fileSystem: fileSystem, workingDirectory: workingDirectory, - diagnosticEngine: diagnosticEngine, - targetTriple: self.frontendTargetInfo.target.triple) + diagnosticEngine: diagnosticEngine) Self.validateCompilationConditionArgs(&parsedOptions, diagnosticEngine: diagnosticEngine) Self.validateFrameworkSearchPathArgs(&parsedOptions, diagnosticEngine: diagnosticEngine) Self.validateCoverageArgs(&parsedOptions, diagnosticsEngine: diagnosticEngine) @@ -1300,6 +1299,11 @@ extension Driver { sanitizerSupported = false } + // Currently only ASAN is supported on Windows. + if sanitizer != .address && targetTriple.isWindows { + sanitizerSupported = false + } + if !sanitizerSupported { diagnosticEngine.emit( .error_unsupported_opt_for_target( @@ -1659,32 +1663,12 @@ extension Driver { static func validateProfilingArgs(_ parsedOptions: inout ParsedOptions, fileSystem: FileSystem, workingDirectory: AbsolutePath?, - diagnosticEngine: DiagnosticsEngine, - targetTriple: Triple) { + diagnosticEngine: DiagnosticsEngine) { if parsedOptions.hasArgument(.profileGenerate) && parsedOptions.hasArgument(.profileUse) { diagnosticEngine.emit(Error.conflictingOptions(.profileGenerate, .profileUse)) } - // Windows executables should be profiled with ETW, whose support needs to be - // implemented before we can enable the option. - if targetTriple.isWindows { - if parsedOptions.hasArgument(.profileGenerate) { - diagnosticEngine.emit( - .error_unsupported_opt_for_target( - arg: "-profile-generate", - target: targetTriple) - ) - } - if parsedOptions.hasArgument(.profileUse) { - diagnosticEngine.emit( - .error_unsupported_opt_for_target( - arg: "-profile-use=", - target: targetTriple) - ) - } - } - if let profileArgs = parsedOptions.getLastArgument(.profileUse)?.asMultiple, let workingDirectory = workingDirectory ?? fileSystem.currentWorkingDirectory { for profilingData in profileArgs { diff --git a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift index 75d350bad..f495f09d5 100644 --- a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift @@ -119,13 +119,6 @@ extension Toolchain { parsedOptions: inout ParsedOptions, isShared: Bool ) throws -> Bool { - // Currently only ASAN is supported on Windows, but clang builds may - // include runtime libraries for unsupported sanitizers. Manually - // disable unsupported sanitizers. - if targetTriple.isWindows && sanitizer != .address { - return false - } - let runtimeName = try runtimeLibraryName( for: sanitizer, targetTriple: targetTriple, diff --git a/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift b/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift index c6627b37e..9469bc3f1 100644 --- a/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift @@ -26,9 +26,6 @@ public final class DarwinToolchain: Toolchain { /// The file system to use for any file operations. public let fileSystem: FileSystem - /// The suffix of executable files. - public let executableSuffix = "" - /// Doubles as path cache and point for overriding normal lookup private var toolPaths = [Tool: AbsolutePath]() diff --git a/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift b/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift index 5a9bf124f..09ca4d00a 100644 --- a/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift @@ -21,9 +21,6 @@ public final class GenericUnixToolchain: Toolchain { /// The file system to use for queries. public let fileSystem: FileSystem - /// The suffix of executable files. - public let executableSuffix = "" - /// Doubles as path cache and point for overriding normal lookup private var toolPaths = [Tool: AbsolutePath]() diff --git a/Sources/SwiftDriver/Toolchains/Toolchain.swift b/Sources/SwiftDriver/Toolchains/Toolchain.swift index d1b434ae2..ebfe27107 100644 --- a/Sources/SwiftDriver/Toolchains/Toolchain.swift +++ b/Sources/SwiftDriver/Toolchains/Toolchain.swift @@ -38,8 +38,6 @@ public protocol Toolchain { var executor: DriverExecutor { get } - var executableSuffix: String { get } - /// Retrieve the absolute path to a particular tool. func getToolPath(_ tool: Tool) throws -> AbsolutePath @@ -131,7 +129,12 @@ extension Toolchain { /// looks in the `executableDir`, `xcrunFind` or in the `searchPaths`. /// - Parameter executable: executable to look for [i.e. `swift`]. func lookup(executable: String) throws -> AbsolutePath { - let filename = executable + executableSuffix + let filename: String + #if os(Windows) + filename = "\(executable).exe" + #else + filename = executable + #endif if let overrideString = envVar(forExecutable: executable) { return try AbsolutePath(validating: overrideString) } else if let path = lookupExecutablePath(filename: filename, searchPaths: [executableDir]) { diff --git a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift index 88904b853..432e6a460 100644 --- a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// import TSCBasic +import SwiftOptions /// Toolchain for Windows. public final class WindowsToolchain: Toolchain { @@ -21,9 +22,6 @@ public final class WindowsToolchain: Toolchain { /// The file system to use for queries. public let fileSystem: FileSystem - /// The suffix of executable files. - public let executableSuffix = ".exe" - /// Doubles as path cache and point for overriding normal lookup private var toolPaths = [Tool: AbsolutePath]() @@ -33,7 +31,7 @@ public final class WindowsToolchain: Toolchain { case .arm: return "armv7" case .x86: return "i386" case nil, .x86_64: return "x86_64" - default: fatalError("unknown arch \(triple.archName) on Windows") + default: fatalError("unknown arch \(triple.archName) for Windows") } } @@ -75,9 +73,9 @@ public final class WindowsToolchain: Toolchain { case .clang: return try lookup(executable: "clang") case .swiftAutolinkExtract: - fatalError("Trying to look up \"swift-autolink-extract\" on Windows") + return try lookup(executable: "swift-autolink-extract") case .dsymutil: - fatalError("Trying to look up \"dsymutil\" on Windows") + return try lookup(executable: "llvm-dsymutil") case .lldb: return try lookup(executable: "lldb") case .dwarfdump: @@ -105,3 +103,30 @@ public final class WindowsToolchain: Toolchain { return "clang_rt.\(sanitizer.libraryName)-\(archName(for: targetTriple)).lib" } } + +extension WindowsToolchain { + public func validateArgs(_ parsedOptions: inout ParsedOptions, + targetTriple: Triple, + targetVariantTriple: Triple?, + diagnosticsEngine: DiagnosticsEngine) throws { + // Windows executables should be profiled with ETW, whose support needs to be + // implemented before we can enable the option. + if parsedOptions.hasArgument(.profileGenerate) { + throw ToolchainValidationError.argumentNotSupported("-profile-generate") + } + if parsedOptions.hasArgument(.profileUse) { + throw ToolchainValidationError.argumentNotSupported("-profile-use=") + } + } +} + +public enum ToolchainValidationError: Error, DiagnosticData { + case argumentNotSupported(String) + + public var description: String { + switch self { + case .argumentNotSupported(let argument): + return "\(argument) is not supported for Windows" + } + } +} From ed1b8c2552d791befa681e03be2797eab52096a0 Mon Sep 17 00:00:00 2001 From: stevapple Date: Thu, 17 Sep 2020 15:33:32 +0800 Subject: [PATCH 10/24] Fix CRT support for Windows --- Sources/SwiftDriver/Driver/Driver.swift | 11 ++++++++ .../Jobs/WindowsToolchain+LinkerSupport.swift | 19 ++++++++++---- .../Toolchains/WindowsToolchain.swift | 25 +++++++++++++------ 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index 712e6d302..ba2aa52e0 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -370,6 +370,9 @@ public struct Driver { self.numParallelJobs = Self.determineNumParallelJobs(&parsedOptions, diagnosticsEngine: diagnosticEngine, env: env) Self.validateWarningControlArgs(&parsedOptions, diagnosticEngine: diagnosticEngine) + Self.validateCRuntimeArgs(&parsedOptions, + diagnosticEngine: diagnosticsEngine, + targetTriple: self.frontendTargetInfo.target.triple) Self.validateProfilingArgs(&parsedOptions, fileSystem: fileSystem, workingDirectory: workingDirectory, @@ -1660,6 +1663,14 @@ extension Driver { } } + static func validateCRuntimeArgs(_ parsedOptions: inout ParsedOptions, + diagnosticEngine: DiagnosticsEngine, + targetTriple: Triple) { + if parsedOptions.hasArgument(.libc) && !targetTriple.isWindows { + diagnosticEngine.emit(.error_unsupported_opt_for_target(arg: "-libc", target: targetTriple)) + } + } + static func validateProfilingArgs(_ parsedOptions: inout ParsedOptions, fileSystem: FileSystem, workingDirectory: AbsolutePath?, diff --git a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift index 72cb3d01a..02a75498d 100644 --- a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift @@ -75,6 +75,20 @@ extension WindowsToolchain { } commandLine.appendFlag("-fuse-ld=\(linker)") + // FIXME: Do we really need `oldnames`? + commandLine.appendFlags("-autolink-library", "oldnames") + if let crt = parsedOptions.getLastArgument(.libc) { + switch crt.asSingle { + case "MT": commandLine.appendFlags("-autolink-library", "libcmt") + case "MTd": commandLine.appendFlags("-autolink-library", "libcmtd") + case "MD": commandLine.appendFlags("-autolink-library", "msvcrt") + case "MDd": commandLine.appendFlags("-autolink-library", "msvcrtd") + default: fatalError("Invalid C runtime value should be filtered") + } + } else { + commandLine.appendFlags("-autolink-library", "msvcrt") + } + let staticStdlib = parsedOptions.hasFlag(positive: .staticStdlib, negative: .noStaticStdlib, default: false) @@ -152,11 +166,6 @@ extension WindowsToolchain { } } - if parsedOptions.hasArgument(.profileGenerate) { - // Profiling support for Windows isn't ready yet. It should have been disabled. - fatalError("Profiling support should have been disabled on Windows.") - } - // Run clang++ in verbose mode if "-v" is set try commandLine.appendLast(.v, from: &parsedOptions) diff --git a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift index 432e6a460..1659e9ca4 100644 --- a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift @@ -117,16 +117,25 @@ extension WindowsToolchain { if parsedOptions.hasArgument(.profileUse) { throw ToolchainValidationError.argumentNotSupported("-profile-use=") } + + if let crt = parsedOptions.getLastArgument(.libc) { + if !["MT", "MTd", "MD", "MDd"].contains(crt.asSingle) { + throw ToolchainValidationError.illegalCrtName(crt.asSingle) + } + } } -} - -public enum ToolchainValidationError: Error, DiagnosticData { - case argumentNotSupported(String) - public var description: String { - switch self { - case .argumentNotSupported(let argument): - return "\(argument) is not supported for Windows" + public enum ToolchainValidationError: Error, DiagnosticData { + case argumentNotSupported(String) + case illegalCrtName(String) + + public var description: String { + switch self { + case .argumentNotSupported(let argument): + return "\(argument) is not supported for Windows" + case .illegalCrtName(let argument): + return "\(argument) is not a valid C Runtime for Windows" + } } } } From 4bd54956c2b0acacef7254949878632bbcf37e27 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Sat, 19 Dec 2020 08:55:25 +0800 Subject: [PATCH 11/24] Keep update with the latest layout --- CMakeLists.txt | 5 +- Package.resolved | 14 +- Package.swift | 32 +- README.md | 29 +- Sources/CMakeLists.txt | 2 + Sources/SwiftDriver/CMakeLists.txt | 35 +- Sources/SwiftDriver/Driver/CompilerMode.swift | 26 +- Sources/SwiftDriver/Driver/DebugInfo.swift | 2 +- Sources/SwiftDriver/Driver/Driver.swift | 684 +++++++-- .../SwiftDriver/Driver/DriverVersion.swift | 18 + Sources/SwiftDriver/Driver/LinkKind.swift | 8 + .../SwiftDriver/Driver/ModuleOutputInfo.swift | 2 +- .../SwiftDriver/Driver/OutputFileMap.swift | 37 +- .../Driver/ToolExecutionDelegate.swift | 25 +- .../SwiftDriver/Execution/ArgsResolver.swift | 24 +- .../Execution/DriverExecutor.swift | 144 +- .../Execution/ParsableOutput.swift | 10 +- .../Execution/ProcessProtocol.swift | 2 +- Sources/SwiftDriver/Execution/llbuild.swift | 263 ---- .../ClangVersionedDependencyResolution.swift | 62 +- ...t => ExplicitDependencyBuildPlanner.swift} | 161 +- .../CommonDependencyOperations.swift | 183 +++ .../InterModuleDependencyGraph.swift | 120 +- .../InterModuleDependencyOracle.swift | 84 + .../ModuleDependencyScanning.swift | 49 +- .../PlaceholderDependencyResolution.swift | 200 ++- .../BidirectionalMap.swift | 41 + .../Incremental Compilation/BuildRecord.swift | 222 +++ .../BuildRecordInfo.swift | 208 +++ .../DependencyKey.swift | 130 ++ .../DictionaryOfDictionaries.swift | 137 ++ .../IncrementalCompilationState.swift | 595 +++++-- .../InputIInfoMap.swift | 176 --- .../Incremental Compilation/InputInfo.swift | 91 +- .../Integrator.swift | 219 +++ .../ModuleDependencyGraph Parts/Node.swift | 117 ++ .../NodeFinder.swift | 230 +++ .../SwiftDeps.swift | 54 + .../ModuleDependencyGraph Parts/Tracer.swift | 134 ++ .../ModuleDependencyGraph.swift | 264 ++++ .../Multidictionary.swift | 100 ++ .../SourceFileDependencyGraph.swift | 322 ++-- .../Incremental Compilation/TwoDMap.swift | 96 ++ .../SwiftDriver/Jobs/AutolinkExtractJob.swift | 25 +- Sources/SwiftDriver/Jobs/BackendJob.swift | 6 +- .../Jobs/CommandLineArguments.swift | 2 +- Sources/SwiftDriver/Jobs/CompileJob.swift | 78 +- .../Jobs/DarwinToolchain+LinkerSupport.swift | 346 +++-- Sources/SwiftDriver/Jobs/EmitModuleJob.swift | 24 +- .../SwiftDriver/Jobs/FrontendJobHelpers.swift | 70 +- .../SwiftDriver/Jobs/GenerateDSYMJob.swift | 1 + Sources/SwiftDriver/Jobs/GeneratePCHJob.swift | 1 + Sources/SwiftDriver/Jobs/GeneratePCMJob.swift | 1 + .../GenericUnixToolchain+LinkerSupport.swift | 55 +- Sources/SwiftDriver/Jobs/InterpretJob.swift | 5 +- Sources/SwiftDriver/Jobs/Job.swift | 16 +- Sources/SwiftDriver/Jobs/LinkJob.swift | 28 +- Sources/SwiftDriver/Jobs/MergeModuleJob.swift | 5 +- Sources/SwiftDriver/Jobs/ModuleWrapJob.swift | 1 + Sources/SwiftDriver/Jobs/Planning.swift | 683 ++++++--- .../SwiftDriver/Jobs/PrintTargetInfoJob.swift | 53 +- Sources/SwiftDriver/Jobs/ReplJob.swift | 1 + .../Jobs/Toolchain+InterpreterSupport.swift | 20 +- .../Jobs/Toolchain+LinkerSupport.swift | 81 +- .../SwiftDriver/Jobs/VerifyDebugInfoJob.swift | 1 + .../Jobs/VerifyModuleInterfaceJob.swift | 8 +- .../WebAssemblyToolchain+LinkerSupport.swift | 161 ++ .../Jobs/WindowsToolchain+LinkerSupport.swift | 24 +- .../Toolchains/DarwinToolchain.swift | 12 +- .../Toolchains/GenericUnixToolchain.swift | 14 +- .../SwiftDriver/Toolchains/Toolchain.swift | 24 +- .../Toolchains/WebAssemblyToolchain.swift | 134 ++ .../Toolchains/WindowsToolchain.swift | 39 +- Sources/SwiftDriver/Utilities/Bits.swift | 98 -- Sources/SwiftDriver/Utilities/Bitstream.swift | 371 ----- .../Utilities/DOTJobGraphSerializer.swift | 2 +- .../SwiftDriver/Utilities/DateAdditions.swift | 6 + .../SwiftDriver/Utilities/Diagnostics.swift | 27 +- .../PredictableRandomNumberGenerator.swift | 2 +- .../Utilities/RelativePathAdditions.swift | 11 - .../Utilities/Triple+Platforms.swift | 4 +- Sources/SwiftDriver/Utilities/Triple.swift | 14 + .../SwiftDriver/Utilities/VirtualPath.swift | 139 +- Sources/SwiftDriverExecution/CMakeLists.txt | 31 + .../MultiJobExecutor.swift | 301 +++- .../SwiftDriverExecutor.swift | 100 ++ Sources/SwiftDriverExecution/llbuild.swift | 269 ++++ Sources/SwiftOptions/ExtraOptions.swift | 6 +- Sources/SwiftOptions/Option.swift | 4 +- Sources/SwiftOptions/OptionParsing.swift | 30 +- Sources/SwiftOptions/Options.swift | 52 +- Sources/SwiftOptions/ParsedOptions.swift | 8 +- Sources/_CSwiftDriver/CMakeLists.txt | 10 + Sources/_CSwiftDriver/_CSwiftDriverImpl.c | 1 + Sources/_CSwiftDriver/include/_CSwiftDriver.h | 9 + .../_CSwiftDriver/include/module.modulemap | 4 + Sources/swift-driver/CMakeLists.txt | 3 +- Sources/swift-driver/main.swift | 1 + TestInputs/Incremental/hello.swiftmodule | Bin 0 -> 12976 bytes .../AssertDiagnosticsTests.swift | 2 +- .../ExplicitModuleBuildTests.swift | 201 ++- .../Helpers/AssertDiagnostics.swift | 4 +- .../Helpers/DriverExtensions.swift | 1 + .../IncrementalCompilationTests.swift | 661 +++++++- .../ExplicitModuleDependencyBuildInputs.swift | 161 +- .../Inputs/IncrementalCompilationInputs.swift | 12 +- Tests/SwiftDriverTests/IntegrationTests.swift | 11 +- Tests/SwiftDriverTests/JobExecutorTests.swift | 52 +- .../ModuleDependencyGraphTests.swift | 1362 +++++++++++++++++ .../ParsableMessageTests.swift | 2 +- ...redictableRandomNumberGeneratorTests.swift | 2 +- Tests/SwiftDriverTests/SwiftDriverTests.swift | 975 ++++++++++-- Tests/SwiftDriverTests/TwoDMapTests.swift | 108 ++ Tests/SwiftDriverTests/XCTestManifests.swift | 136 +- .../OptionParsingTests.swift | 9 + Utilities/build-script-helper.py | 456 +++++- 116 files changed, 10216 insertions(+), 2718 deletions(-) create mode 100644 Sources/SwiftDriver/Driver/DriverVersion.swift delete mode 100644 Sources/SwiftDriver/Execution/llbuild.swift rename Sources/SwiftDriver/Explicit Module Builds/{ExplicitModuleBuildHandler.swift => ExplicitDependencyBuildPlanner.swift} (79%) create mode 100644 Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/CommonDependencyOperations.swift rename Sources/SwiftDriver/Explicit Module Builds/{ => Inter Module Dependencies}/InterModuleDependencyGraph.swift (58%) create mode 100644 Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyOracle.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/BidirectionalMap.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/BuildRecord.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/BuildRecordInfo.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/DependencyKey.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/DictionaryOfDictionaries.swift delete mode 100644 Sources/SwiftDriver/Incremental Compilation/InputIInfoMap.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Integrator.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Node.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/NodeFinder.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/SwiftDeps.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Tracer.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/Multidictionary.swift create mode 100644 Sources/SwiftDriver/Incremental Compilation/TwoDMap.swift create mode 100644 Sources/SwiftDriver/Jobs/WebAssemblyToolchain+LinkerSupport.swift create mode 100644 Sources/SwiftDriver/Toolchains/WebAssemblyToolchain.swift delete mode 100644 Sources/SwiftDriver/Utilities/Bits.swift delete mode 100644 Sources/SwiftDriver/Utilities/Bitstream.swift create mode 100644 Sources/SwiftDriverExecution/CMakeLists.txt rename Sources/{SwiftDriver/Execution => SwiftDriverExecution}/MultiJobExecutor.swift (56%) create mode 100644 Sources/SwiftDriverExecution/SwiftDriverExecutor.swift create mode 100644 Sources/SwiftDriverExecution/llbuild.swift create mode 100644 Sources/_CSwiftDriver/CMakeLists.txt create mode 100644 Sources/_CSwiftDriver/_CSwiftDriverImpl.c create mode 100644 Sources/_CSwiftDriver/include/_CSwiftDriver.h create mode 100644 Sources/_CSwiftDriver/include/module.modulemap create mode 100644 TestInputs/Incremental/hello.swiftmodule create mode 100644 Tests/SwiftDriverTests/ModuleDependencyGraphTests.swift create mode 100644 Tests/SwiftDriverTests/TwoDMapTests.swift diff --git a/CMakeLists.txt b/CMakeLists.txt index a39da660f..222b4377f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,7 @@ cmake_minimum_required(VERSION 3.15.1) list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules) -project(SwiftDriver LANGUAGES Swift) +project(SwiftDriver LANGUAGES C Swift) set(SWIFT_VERSION 5) set(CMAKE_Swift_LANGUAGE_VERSION ${SWIFT_VERSION}) @@ -19,6 +19,9 @@ if(CMAKE_VERSION VERSION_LESS 3.16) set(CMAKE_LINK_LIBRARY_FLAG "-l") endif() +# ensure Swift compiler can find _CSwiftDriver +add_compile_options($<$:-I$${CMAKE_CURRENT_SOURCE_DIR}/Sources/_CSwiftDriver/include>) + set(CMAKE_Swift_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/swift) if(CMAKE_VERSION VERSION_LESS 3.16 AND CMAKE_SYSTEM_NAME STREQUAL Windows) diff --git a/Package.resolved b/Package.resolved index 8f61af71e..67e50aaf3 100644 --- a/Package.resolved +++ b/Package.resolved @@ -5,17 +5,17 @@ "package": "swift-argument-parser", "repositoryURL": "https://github.com/apple/swift-argument-parser.git", "state": { - "branch": null, - "revision": "92646c0cdbaca076c8d3d0207891785b3379cbff", - "version": "0.3.1" + "branch": "main", + "revision": "4273ad222e6c51969e8585541f9da5187ad94e47", + "version": null } }, { "package": "llbuild", "repositoryURL": "https://github.com/apple/swift-llbuild.git", "state": { - "branch": "master", - "revision": "0816cf594c2e4d890d1f188271da667f99763fa9", + "branch": "main", + "revision": "eb56a00ed9dfd62c2ce4ec86183ff0bc0afda997", "version": null } }, @@ -23,8 +23,8 @@ "package": "swift-tools-support-core", "repositoryURL": "https://github.com/apple/swift-tools-support-core.git", "state": { - "branch": "master", - "revision": "25fc6eaec5d9f79b79419c5bdaa04e434cbcd568", + "branch": "main", + "revision": "c98cfc216d22798dce7ce7b9cc171565371e967e", "version": null } }, diff --git a/Package.swift b/Package.swift index e373d830c..4556f46c7 100644 --- a/Package.swift +++ b/Package.swift @@ -24,18 +24,34 @@ let package = Package( .library( name: "SwiftDriver", targets: ["SwiftDriver"]), + .library( + name: "SwiftDriverDynamic", + type: .dynamic, + targets: ["SwiftDriver"]), .library( name: "SwiftOptions", targets: ["SwiftOptions"]), + .library( + name: "SwiftDriverExecution", + targets: ["SwiftDriverExecution"]), ], targets: [ + .target(name: "_CSwiftDriver"), + /// The driver library. .target( name: "SwiftDriver", - dependencies: ["SwiftOptions", "SwiftToolsSupport-auto", "Yams"]), + dependencies: ["SwiftOptions", "SwiftToolsSupport-auto", "Yams", "_CSwiftDriver"]), + + /// The execution library. + .target( + name: "SwiftDriverExecution", + dependencies: ["SwiftDriver", "SwiftToolsSupport-auto"]), + + /// Driver tests. .testTarget( name: "SwiftDriverTests", - dependencies: ["SwiftDriver", "swift-driver"]), + dependencies: ["SwiftDriver", "SwiftDriverExecution", "swift-driver"]), /// The options library. .target( @@ -48,7 +64,7 @@ let package = Package( /// The primary driver executable. .target( name: "swift-driver", - dependencies: ["SwiftDriver"]), + dependencies: ["SwiftDriverExecution", "SwiftDriver"]), /// The help executable. .target( @@ -66,7 +82,7 @@ let package = Package( if ProcessInfo.processInfo.environment["SWIFT_DRIVER_LLBUILD_FWK"] == nil { if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil { package.dependencies += [ - .package(url: "https://github.com/apple/swift-llbuild.git", .branch("master")), + .package(url: "https://github.com/apple/swift-llbuild.git", .branch("main")), ] } else { // In Swift CI, use a local path to llbuild to interoperate with tools @@ -75,21 +91,21 @@ if ProcessInfo.processInfo.environment["SWIFT_DRIVER_LLBUILD_FWK"] == nil { .package(path: "../llbuild"), ] } - package.targets.first(where: { $0.name == "SwiftDriver" })!.dependencies += ["llbuildSwift"] + package.targets.first(where: { $0.name == "SwiftDriverExecution" })!.dependencies += ["llbuildSwift"] } if ProcessInfo.processInfo.environment["SWIFTCI_USE_LOCAL_DEPS"] == nil { package.dependencies += [ - .package(url: "https://github.com/apple/swift-tools-support-core.git", .branch("master")), + .package(url: "https://github.com/apple/swift-tools-support-core.git", .branch("main")), .package(url: "https://github.com/jpsim/Yams.git", .upToNextMinor(from: "4.0.0")), // The 'swift-argument-parser' version declared here must match that // used by 'swift-package-manager' and 'sourcekit-lsp'. Please coordinate // dependency version changes here with those projects. - .package(url: "https://github.com/apple/swift-argument-parser.git", .upToNextMinor(from: "0.3.1")), + .package(url: "https://github.com/apple/swift-argument-parser.git", .branch("main")), ] } else { package.dependencies += [ - .package(path: "../swiftpm/swift-tools-support-core"), + .package(path: "../swift-tools-support-core"), .package(path: "../yams"), .package(path: "../swift-argument-parser"), ] diff --git a/README.md b/README.md index fa49d4e59..ec3fc8dce 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Swift's compiler driver is a program that coordinates the compilation of Swift source code into various compiled results: executables, libraries, object files, Swift modules and interfaces, etc. It is the program one invokes from the command line to build Swift code (i.e., `swift` or `swiftc`) and is often invoked on the developer's behalf by a build system such as the [Swift Package Manager (SwiftPM)](https://github.com/apple/swift-package-manager) or Xcode's build system. -The `swift-driver` project is a new implementation of the Swift compiler driver that is intended to replace the [existing driver](https://github.com/apple/swift/tree/master/lib/Driver) with a more extensible, maintainable, and robust code base. The specific goals of this project include: +The `swift-driver` project is a new implementation of the Swift compiler driver that is intended to replace the [existing driver](https://github.com/apple/swift/tree/main/lib/Driver) with a more extensible, maintainable, and robust code base. The specific goals of this project include: * A maintainable, robust, and flexible Swift code base * Library-based architecture that allows better integration with build tools @@ -42,9 +42,9 @@ available. Doing so requires several dependencies to be built first, all with CMake: * (Non-Apple platforms only) [swift-corelibs-foundation](https://github.com/apple/swift-corelibs-foundation) -* [llbuild](https://github.com/apple/swift-llbuild) configure CMake with `-DLLBUILD_SUPPORT_BINDINGS="Swift"` when building +* [llbuild](https://github.com/apple/swift-llbuild) configure CMake with `-DLLBUILD_SUPPORT_BINDINGS="Swift"` and `-DCMAKE_OSX_ARCHITECTURES=x86_64` (If building on Intel) when building ``` - cmake -B -G Ninja -DLLBUILD_SUPPORT_BINDINGS="Swift" + cmake -B -G Ninja -DLLBUILD_SUPPORT_BINDINGS="Swift" -DCMAKE_OSX_ARCHITECTURES=x86_64 ``` * [swift-argument-parser](https://github.com/apple/swift-argument-parser) * [Yams](https://github.com/jpsim/Yams) @@ -61,7 +61,7 @@ The new Swift driver is a work in progress, and there are numerous places for an ### Driver Documentation -For a conceptual overview of the driver, see [The Swift Driver, Compilation Model, and Command-Line Experience](https://github.com/apple/swift/blob/master/docs/Driver.md). To learn more about the internals, see [Driver Design & Internals](https://github.com/apple/swift/blob/master/docs/DriverInternals.md) and [Parseable Driver Output](https://github.com/apple/swift/blob/master/docs/DriverParseableOutput.md). +For a conceptual overview of the driver, see [The Swift Driver, Compilation Model, and Command-Line Experience](https://github.com/apple/swift/blob/main/docs/Driver.md). To learn more about the internals, see [Driver Design & Internals](https://github.com/apple/swift/blob/main/docs/DriverInternals.md) and [Parseable Driver Output](https://github.com/apple/swift/blob/main/docs/DriverParseableOutput.md). ### Testing @@ -100,7 +100,14 @@ Using: apple/swift-driver#208 @swift-ci smoke test ``` -@swift-ci cross-repository testing facilities are described [here](https://github.com/apple/swift/blob/master/docs/ContinuousIntegration.md#cross-repository-testing). +@swift-ci cross-repository testing facilities are described [here](https://github.com/apple/swift/blob/main/docs/ContinuousIntegration.md#cross-repository-testing). + +### Testing in Xcode with custom toolchain +After the toolchain is installed, Xcode needs to be told to use it. This can mean two things, building the driver with the toolchain and telling the driver to use the toolchain when running. + +Building with the toolchain is easy, set the toolchain in Xcode: Menu Bar > Xcode > Toolchains > select your toolchain + +Running the driver requires setting the TOOLCHAINS environment variable. This tells xcrun which toolchain to use (on darwin xcrun is used to find tools). This variable is the name of the toolchain and not the path (ex: `Swift Development Snapshot`). Important note: xcrun lookup is lower priority than the SWIFT_EXEC_*_EXEC family of environment variables, the tools directory, and any tools in the same directory as the driver (This includes a driver installed in a toolchain). Even though TOOLCHAINS is not highest priority it's a convenient way to run the xctest suite using a custom toolchain. #### Preparing a Linux docker for debug @@ -126,7 +133,7 @@ $ apt-get install libncurses-dev ### Rebuilding `Options.swift` -`Options.swift`, which contains the complete set of options that can be parsed by the driver, is automatically generated from the [option tables in the Swift compiler](https://github.com/apple/swift/tree/master/include/swift/Option). If you need to regenerate `Options.swift`, you will need to [build the Swift compiler](https://github.com/apple/swift#building-swift) and then build `makeOptions` program with a `-I` that allows the generated `Options.inc` to +`Options.swift`, which contains the complete set of options that can be parsed by the driver, is automatically generated from the [option tables in the Swift compiler](https://github.com/apple/swift/tree/main/include/swift/Option). If you need to regenerate `Options.swift`, you will need to [build the Swift compiler](https://github.com/apple/swift#building-swift) and then build `makeOptions` program with a `-I` that allows the generated `Options.inc` to be found, e.g.: ``` @@ -152,7 +159,7 @@ The goal of the new Swift driver is to provide a drop-in replacement for the exi * [ ] Find a better way to describe aliases for options. Can they be of some other type `OptionAlias` so we can't make the mistake of (e.g.) asking for an alias option when we're translating options? * [ ] Diagnose unused options on the command line * [ ] Typo correction for misspelled option names - * [ ] Find a better way than `makeOptions.cpp` to translate the command-line options from [Swift's repository](https://github.com/apple/swift/tree/master/include/swift/Option) into `Options.swift`. + * [ ] Find a better way than `makeOptions.cpp` to translate the command-line options from [Swift's repository](https://github.com/apple/swift/tree/main/include/swift/Option) into `Options.swift`. * Platform support * [x] Teach the `DarwinToolchain` to also handle iOS, tvOS, watchOS * [x] Fill out the `GenericUnixToolchain` toolchain to get it working @@ -173,10 +180,10 @@ The goal of the new Swift driver is to provide a drop-in replacement for the exi * [x] Complete `OutputFileMap` implementation to handle all file types uniformly * Testing * [ ] Build stuff with SwiftPM or Xcode or your favorite build system, using `swift-driver`. Were the results identical? What changed? - * [x] Shim in `swift-driver` so it can run the Swift repository's [driver test suite](https://github.com/apple/swift/tree/master/test/Driver). + * [x] Shim in `swift-driver` so it can run the Swift repository's [driver test suite](https://github.com/apple/swift/tree/main/test/Driver). * [ ] Investigate differences in the test results for the Swift repository's driver test suite (above) between the existing and new driver. - * [ ] Port interesting tests from the Swift repository's [driver test suite](https://github.com/apple/swift/tree/master/test/Driver) over to XCTest + * [ ] Port interesting tests from the Swift repository's [driver test suite](https://github.com/apple/swift/tree/main/test/Driver) over to XCTest * [ ] Fuzz the command-line options to try to crash the Swift driver itself * Integration - * [ ] Teach the Swift compiler's [`build-script`](https://github.com/apple/swift/blob/master/utils/build-script) to build `swift-driver`. - * [ ] Building on the above, teach the Swift compiler's [`build-toolchain`](https://github.com/apple/swift/blob/master/utils/build-toolchain) to install `swift-driver` as the primary driver so we can test full toolchains with the new driver + * [ ] Teach the Swift compiler's [`build-script`](https://github.com/apple/swift/blob/main/utils/build-script) to build `swift-driver`. + * [ ] Building on the above, teach the Swift compiler's [`build-toolchain`](https://github.com/apple/swift/blob/main/utils/build-toolchain) to install `swift-driver` as the primary driver so we can test full toolchains with the new driver diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt index f9cf91983..c93dd937f 100644 --- a/Sources/CMakeLists.txt +++ b/Sources/CMakeLists.txt @@ -6,7 +6,9 @@ # See http://swift.org/LICENSE.txt for license information # See http://swift.org/CONTRIBUTORS.txt for Swift project authors +add_subdirectory(_CSwiftDriver) add_subdirectory(SwiftOptions) add_subdirectory(SwiftDriver) +add_subdirectory(SwiftDriverExecution) add_subdirectory(swift-driver) add_subdirectory(swift-help) diff --git a/Sources/SwiftDriver/CMakeLists.txt b/Sources/SwiftDriver/CMakeLists.txt index b464e6d36..a20b03ebf 100644 --- a/Sources/SwiftDriver/CMakeLists.txt +++ b/Sources/SwiftDriver/CMakeLists.txt @@ -8,12 +8,14 @@ add_library(SwiftDriver "Explicit Module Builds/ClangModuleBuildJobCache.swift" - "Explicit Module Builds/ExplicitModuleBuildHandler.swift" - "Explicit Module Builds/PlaceholderDependencyResolution.swift" "Explicit Module Builds/ClangVersionedDependencyResolution.swift" - "Explicit Module Builds/InterModuleDependencyGraph.swift" + "Explicit Module Builds/ExplicitDependencyBuildPlanner.swift" "Explicit Module Builds/ModuleDependencyScanning.swift" + "Explicit Module Builds/PlaceholderDependencyResolution.swift" "Explicit Module Builds/SerializableModuleArtifacts.swift" + "Explicit Module Builds/Inter Module Dependencies/CommonDependencyOperations.swift" + "Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyGraph.swift" + "Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyOracle.swift" Driver/CompilerMode.swift Driver/DebugInfo.swift @@ -22,18 +24,29 @@ add_library(SwiftDriver Driver/ModuleOutputInfo.swift Driver/OutputFileMap.swift Driver/ToolExecutionDelegate.swift + Driver/DriverVersion.swift Execution/ArgsResolver.swift - Execution/MultiJobExecutor.swift Execution/DriverExecutor.swift Execution/ParsableOutput.swift Execution/ProcessProtocol.swift - Execution/llbuild.swift + "Incremental Compilation/ModuleDependencyGraph Parts/Integrator.swift" + "Incremental Compilation/ModuleDependencyGraph Parts/Node.swift" + "Incremental Compilation/ModuleDependencyGraph Parts/NodeFinder.swift" + "Incremental Compilation/ModuleDependencyGraph Parts/SwiftDeps.swift" + "Incremental Compilation/ModuleDependencyGraph Parts/Tracer.swift" + "Incremental Compilation/BidirectionalMap.swift" + "Incremental Compilation/BuildRecord.swift" + "Incremental Compilation/BuildRecordInfo.swift" + "Incremental Compilation/DependencyKey.swift" + "Incremental Compilation/DictionaryOfDictionaries.swift" "Incremental Compilation/IncrementalCompilationState.swift" - "Incremental Compilation/InputIInfoMap.swift" "Incremental Compilation/InputInfo.swift" + "Incremental Compilation/ModuleDependencyGraph.swift" + "Incremental Compilation/Multidictionary.swift" "Incremental Compilation/SourceFileDependencyGraph.swift" + "Incremental Compilation/TwoDMap.swift" Jobs/AutolinkExtractJob.swift Jobs/BackendJob.swift @@ -59,14 +72,14 @@ add_library(SwiftDriver Jobs/VerifyDebugInfoJob.swift Jobs/VerifyModuleInterfaceJob.swift Jobs/WindowsToolchain+LinkerSupport.swift + Jobs/WebAssemblyToolchain+LinkerSupport.swift Toolchains/DarwinToolchain.swift Toolchains/GenericUnixToolchain.swift Toolchains/Toolchain.swift Toolchains/WindowsToolchain.swift + Toolchains/WebAssemblyToolchain.swift - Utilities/Bits.swift - Utilities/Bitstream.swift Utilities/DOTJobGraphSerializer.swift Utilities/DateAdditions.swift Utilities/Diagnostics.swift @@ -86,11 +99,11 @@ add_library(SwiftDriver target_link_libraries(SwiftDriver PUBLIC TSCBasic TSCUtility - SwiftOptions - llbuildSwift) + SwiftOptions) target_link_libraries(SwiftDriver PRIVATE CYaml - Yams) + Yams + CSwiftDriver) set_property(GLOBAL APPEND PROPERTY SWIFTDRIVER_EXPORTS SwiftDriver) diff --git a/Sources/SwiftDriver/Driver/CompilerMode.swift b/Sources/SwiftDriver/Driver/CompilerMode.swift index bd2c95625..be6cd6e88 100644 --- a/Sources/SwiftDriver/Driver/CompilerMode.swift +++ b/Sources/SwiftDriver/Driver/CompilerMode.swift @@ -10,7 +10,7 @@ // //===----------------------------------------------------------------------===// /// The mode of the compiler. -public enum CompilerMode: Equatable { +@_spi(Testing) public enum CompilerMode: Equatable { /// A standard compilation, using multiple frontend invocations and -primary-file. case standardCompile @@ -33,7 +33,7 @@ public enum CompilerMode: Equatable { /// Information about batch mode, which is used to determine how to form /// the batches of jobs. -public struct BatchModeInfo: Equatable { +@_spi(Testing) public struct BatchModeInfo: Equatable { let seed: Int? let count: Int? let sizeLimit: Int? @@ -62,6 +62,28 @@ extension CompilerMode { } } + public var isStandardCompilationForPlanning: Bool { + switch self { + case .immediate, .repl, .compilePCM: + return false + case .batchCompile, .standardCompile, .singleCompile: + return true + } + } + + public var batchModeInfo: BatchModeInfo? { + switch self { + case let .batchCompile(info): + return info + default: + return nil + } + } + + public var isBatchCompile: Bool { + batchModeInfo != nil + } + // Whether this compilation mode supports the use of bridging pre-compiled // headers. public var supportsBridgingPCH: Bool { diff --git a/Sources/SwiftDriver/Driver/DebugInfo.swift b/Sources/SwiftDriver/Driver/DebugInfo.swift index a630dc95b..92c131501 100644 --- a/Sources/SwiftDriver/Driver/DebugInfo.swift +++ b/Sources/SwiftDriver/Driver/DebugInfo.swift @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// /// The debug information produced by the driver. -public struct DebugInfo { +@_spi(Testing) public struct DebugInfo { /// Describes the format used for debug information. public enum Format: String { diff --git a/Sources/SwiftDriver/Driver/Driver.swift b/Sources/SwiftDriver/Driver/Driver.swift index ba2aa52e0..42eb4bd1e 100644 --- a/Sources/SwiftDriver/Driver/Driver.swift +++ b/Sources/SwiftDriver/Driver/Driver.swift @@ -13,6 +13,7 @@ import TSCBasic import TSCUtility import Foundation import SwiftOptions +@_implementationOnly import _CSwiftDriver /// The Swift driver. public struct Driver { @@ -26,8 +27,10 @@ public struct Driver { case integratedReplRemoved case conflictingOptions(Option, Option) case unableToLoadOutputFileMap(String) - case unableToDecodeFrontendTargetInfo + case unableToDecodeFrontendTargetInfo(String?, [String], String) case failedToRetrieveFrontendTargetInfo + case failedToRunFrontendToRetrieveTargetInfo(Int, String?) + case unableToReadFrontendTargetInfo case missingProfilingData(String) case conditionalCompilationFlagHasRedundantPrefix(String) case conditionalCompilationFlagIsNotValidIdentifier(String) @@ -57,10 +60,22 @@ public struct Driver { return "Compiler-internal integrated REPL has been removed; use the LLDB-enhanced REPL instead." case .conflictingOptions(let one, let two): return "conflicting options '\(one.spelling)' and '\(two.spelling)'" - case .unableToDecodeFrontendTargetInfo: - return "could not decode frontend target info; compiler driver and frontend executables may be incompatible" + case let .unableToDecodeFrontendTargetInfo(outputString, arguments, errorDesc): + let output = outputString.map { ": \"\($0)\""} ?? "" + return """ + could not decode frontend target info; compiler driver and frontend executables may be incompatible + details: frontend: \(arguments.first ?? "") + arguments: \(arguments.dropFirst()) + error: \(errorDesc) + output\n\(output) + """ case .failedToRetrieveFrontendTargetInfo: return "failed to retrieve frontend target info" + case .unableToReadFrontendTargetInfo: + return "could not read frontend target info" + case let .failedToRunFrontendToRetrieveTargetInfo(returnCode, stderr): + return "frontend job retrieving target info failed with code \(returnCode)" + + (stderr.map {": \($0)"} ?? "") case .missingProfilingData(let arg): return "no profdata file exists at '\(arg)'" case .conditionalCompilationFlagHasRedundantPrefix(let name): @@ -90,30 +105,33 @@ public struct Driver { public let env: [String: String] /// The file system which we should interact with. - public let fileSystem: FileSystem + let fileSystem: FileSystem /// Diagnostic engine for emitting warnings, errors, etc. public let diagnosticEngine: DiagnosticsEngine /// The executor the driver uses to run jobs. - public let executor: DriverExecutor + let executor: DriverExecutor /// The toolchain to use for resolution. - public let toolchain: Toolchain + @_spi(Testing) public let toolchain: Toolchain /// Information about the target, as reported by the Swift frontend. - let frontendTargetInfo: FrontendTargetInfo + @_spi(Testing) public let frontendTargetInfo: FrontendTargetInfo /// The target triple. - public var targetTriple: Triple { frontendTargetInfo.target.triple } + @_spi(Testing) public var targetTriple: Triple { frontendTargetInfo.target.triple } /// The variant target triple. - public var targetVariantTriple: Triple? { + var targetVariantTriple: Triple? { frontendTargetInfo.targetVariant?.triple } + /// `true` if the driver should use the static resource directory. + let useStaticResourceDir: Bool + /// The kind of driver. - public let driverKind: DriverKind + let driverKind: DriverKind /// The option table we're using. let optionTable: OptionTable @@ -121,112 +139,142 @@ public struct Driver { /// The set of parsed options. var parsedOptions: ParsedOptions + /// Whether to print out extra info regarding jobs + let showJobLifecycle: Bool + /// Extra command-line arguments to pass to the Swift compiler. - public let swiftCompilerPrefixArgs: [String] + let swiftCompilerPrefixArgs: [String] /// The working directory for the driver, if there is one. - public let workingDirectory: AbsolutePath? + let workingDirectory: AbsolutePath? /// The set of input files - public let inputFiles: [TypedVirtualPath] + @_spi(Testing) public let inputFiles: [TypedVirtualPath] /// The last time each input file was modified, recorded at the start of the build. - public let recordedInputModificationDates: [TypedVirtualPath: Date] + @_spi(Testing) public let recordedInputModificationDates: [TypedVirtualPath: Date] /// The mapping from input files to output files for each kind. - internal let outputFileMap: OutputFileMap? + let outputFileMap: OutputFileMap? /// The number of files required before making a file list. - internal let fileListThreshold: Int + let fileListThreshold: Int /// Should use file lists for inputs (number of inputs exceeds `fileListThreshold`). - public let shouldUseInputFileList: Bool + let shouldUseInputFileList: Bool /// VirtualPath for shared all sources file list. `nil` if unused. - public let allSourcesFileList: VirtualPath? + let allSourcesFileList: VirtualPath? /// The mode in which the compiler will execute. - public let compilerMode: CompilerMode + @_spi(Testing) public let compilerMode: CompilerMode /// The type of the primary output generated by the compiler. - public let compilerOutputType: FileType? + @_spi(Testing) public let compilerOutputType: FileType? + + /// The type of the link-time-optimization we expect to perform. + @_spi(Testing) public let lto: LTOKind? /// The type of the primary output generated by the linker. - public let linkerOutputType: LinkOutputType? + @_spi(Testing) public let linkerOutputType: LinkOutputType? /// When > 0, the number of threads to use in a multithreaded build. - public let numThreads: Int + @_spi(Testing) public let numThreads: Int /// The specified maximum number of parallel jobs to execute. - public let numParallelJobs: Int? + @_spi(Testing) public let numParallelJobs: Int? /// The set of sanitizers that were requested - public let enabledSanitizers: Set + let enabledSanitizers: Set /// The debug information to produce. - public let debugInfo: DebugInfo + @_spi(Testing) public let debugInfo: DebugInfo // The information about the module to produce. - public let moduleOutputInfo: ModuleOutputInfo + @_spi(Testing) public let moduleOutputInfo: ModuleOutputInfo - /// Code & data for incremental compilation - public let incrementalCompilationState: IncrementalCompilationState + /// Info needed to write and maybe read the build record. + /// Only present when the driver will be writing the record. + /// Only used for reading when compiling incrementally. + let buildRecordInfo: BuildRecordInfo? + + /// Code & data for incremental compilation. Nil if not running in incremental mode + @_spi(Testing) public let incrementalCompilationState: IncrementalCompilationState? /// The path of the SDK. - public var sdkPath: String? { - frontendTargetInfo.paths.sdkPath + public var absoluteSDKPath: AbsolutePath? { + switch frontendTargetInfo.sdkPath?.path { + case .absolute(let path): + return path + case .relative(let path): + let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory + return cwd.map { AbsolutePath($0, path) } + case nil: + return nil + case .standardInput, .standardOutput, .temporary, .temporaryWithKnownContents, .fileList: + fatalError("Frontend target information will never include a path of this type.") + } } /// The path to the imported Objective-C header. - public let importedObjCHeader: VirtualPath? + let importedObjCHeader: VirtualPath? /// The path to the pch for the imported Objective-C header. - public let bridgingPrecompiledHeader: VirtualPath? + let bridgingPrecompiledHeader: VirtualPath? /// Path to the dependencies file. - public let dependenciesFilePath: VirtualPath? - - /// Path to the reference dependencies (.swiftdeps) file. - public let referenceDependenciesFilePath: VirtualPath? + let dependenciesFilePath: VirtualPath? /// Path to the serialized diagnostics file. - public let serializedDiagnosticsFilePath: VirtualPath? + let serializedDiagnosticsFilePath: VirtualPath? /// Path to the Objective-C generated header. - public let objcGeneratedHeaderPath: VirtualPath? + let objcGeneratedHeaderPath: VirtualPath? /// Path to the loaded module trace file. - public let loadedModuleTracePath: VirtualPath? + let loadedModuleTracePath: VirtualPath? /// Path to the TBD file (text-based dylib). - public let tbdPath: VirtualPath? + let tbdPath: VirtualPath? /// Path to the module documentation file. - public let moduleDocOutputPath: VirtualPath? + let moduleDocOutputPath: VirtualPath? /// Path to the Swift interface file. - public let swiftInterfacePath: VirtualPath? + let swiftInterfacePath: VirtualPath? + + /// Path to the Swift private interface file. + let swiftPrivateInterfacePath: VirtualPath? /// Path to the optimization record. - public let optimizationRecordPath: VirtualPath? + let optimizationRecordPath: VirtualPath? /// Path to the Swift module source information file. - public let moduleSourceInfoPath: VirtualPath? - - /// If the driver should force emit module in a single invocation. - /// - /// This will force the driver to first emit the module and then run compile jobs. - public var forceEmitModuleInSingleInvocation: Bool = false - - /// Handler for constructing module build jobs using Explicit Module Builds. - /// Constructed during the planning phase only when all modules will be prebuilt and treated - /// as explicit by the various compilation jobs. - @_spi(Testing) public var explicitModuleBuildHandler: ExplicitModuleBuildHandler? = nil - - /// A collection describing external dependencies for the current main module that may be invisible to - /// the driver itself, but visible to its clients (e.g. build systems like SwiftPM). Along with the external dependencies' - /// module dependency graphs. - @_spi(Testing) public var externalDependencyArtifactMap: ExternalDependencyArtifactMap? = nil + let moduleSourceInfoPath: VirtualPath? + + /// Force the driver to emit the module first and then run compile jobs. This could be used to unblock + /// dependencies in parallel builds. + var forceEmitModuleBeforeCompile: Bool = false + + // FIXME: We should soon be able to remove this from being in the Driver's state. + // Its only remaining use outside of actual dependency build planning is in + // command-line input option generation for the explicit main module compile job. + /// Planner for constructing module build jobs using Explicit Module Builds. + /// Constructed during the planning phase only when all module dependencies will be prebuilt and treated + /// as explicit inputs by the various compilation jobs. + @_spi(Testing) public var explicitDependencyBuildPlanner: ExplicitDependencyBuildPlanner? = nil + + /// An oracle for querying inter-module dependencies + /// Can either be an argument to the driver in many-module contexts where dependency information + /// is shared across many targets; otherwise, a new instance is created by the driver itself. + @_spi(Testing) public let interModuleDependencyOracle: InterModuleDependencyOracle + + // TODO: Once the clients have transitioned to using the InterModuleDependencyOracle API, + // this must convey information about the externally-prebuilt targets only + /// All external artifacts a build system (e.g. SwiftPM) may pass in as input to the explicit + /// build of the current module. Consists of a map of externally-built targets, and a map of all previously + /// discovered/scanned modules and their infos. + @_spi(Testing) public var externalBuildArtifacts: ExternalBuildArtifacts? = nil /// Handler for emitting diagnostics to stderr. public static let stderrDiagnosticsHandler: DiagnosticsEngine.DiagnosticsHandler = { diagnostic in @@ -264,15 +312,20 @@ public struct Driver { /// expand response files, etc. By default this is the local filesystem. /// - Parameter executor: Used by the driver to execute jobs. The default argument /// is present to streamline testing, it shouldn't be used in production. - /// - Parameter externalModuleDependencies: A collection of external modules that the main module - /// of the current compilation depends on. Explicit Module Build use only. + /// - Parameter externalBuildArtifacts: All external artifacts a build system may pass in as input to the explicit + /// build of the current module. Consists of a map of externally-built targets, and a map of all previously + /// discovered/scanned modules. public init( args: [String], env: [String: String] = ProcessEnv.vars, diagnosticsEngine: DiagnosticsEngine = DiagnosticsEngine(handlers: [Driver.stderrDiagnosticsHandler]), fileSystem: FileSystem = localFileSystem, executor: DriverExecutor, - externalModuleDependencies: ExternalDependencyArtifactMap? = nil + // FIXME: Duplication with externalBuildArtifacts and externalTargetModulePathMap + // is a temporary backwards-compatibility shim to help transition SwiftPM to the new API + externalBuildArtifacts: ExternalBuildArtifacts? = nil, + externalTargetModulePathMap: ExternalTargetModulePathMap? = nil, + interModuleDependencyOracle: InterModuleDependencyOracle? = nil ) throws { self.env = env self.fileSystem = fileSystem @@ -280,7 +333,11 @@ public struct Driver { self.diagnosticEngine = diagnosticsEngine self.executor = executor - self.externalDependencyArtifactMap = externalModuleDependencies + if let externalArtifacts = externalBuildArtifacts { + self.externalBuildArtifacts = externalArtifacts + } else if let externalTargetPaths = externalTargetModulePathMap { + self.externalBuildArtifacts = (externalTargetPaths, [:]) + } if case .subcommand = try Self.invocationRunMode(forArgs: args).mode { throw Error.subcommandPassedToDriver @@ -291,17 +348,11 @@ public struct Driver { self.driverKind = try Self.determineDriverKind(args: &args) self.optionTable = OptionTable() self.parsedOptions = try optionTable.parse(Array(args), for: self.driverKind) + self.showJobLifecycle = parsedOptions.contains(.driverShowJobLifecycle) // Determine the compilation mode. self.compilerMode = try Self.computeCompilerMode(&parsedOptions, driverKind: driverKind, diagnosticsEngine: diagnosticEngine) - // Build the toolchain and determine target information. - (self.toolchain, self.frontendTargetInfo, self.swiftCompilerPrefixArgs) = - try Self.computeToolchain( - &self.parsedOptions, diagnosticsEngine: diagnosticEngine, - compilerMode: self.compilerMode, env: env, - executor: self.executor, fileSystem: fileSystem) - // Compute the working directory. workingDirectory = try parsedOptions.getLastArgument(.workingDirectory).map { workingDirectoryArg in let cwd = fileSystem.currentWorkingDirectory @@ -313,6 +364,22 @@ public struct Driver { try Self.applyWorkingDirectory(workingDirectory, to: &self.parsedOptions) } + let staticExecutable = parsedOptions.hasFlag(positive: .staticExecutable, + negative: .noStaticExecutable, + default: false) + let staticStdlib = parsedOptions.hasFlag(positive: .staticStdlib, + negative: .noStaticStdlib, + default: false) + self.useStaticResourceDir = staticExecutable || staticStdlib + + // Build the toolchain and determine target information. + (self.toolchain, self.frontendTargetInfo, self.swiftCompilerPrefixArgs) = + try Self.computeToolchain( + &self.parsedOptions, diagnosticsEngine: diagnosticEngine, + compilerMode: self.compilerMode, env: env, + executor: self.executor, fileSystem: fileSystem, + useStaticResourceDir: self.useStaticResourceDir) + // Classify and collect all of the input files. let inputFiles = try Self.collectInputFiles(&self.parsedOptions) self.inputFiles = inputFiles @@ -323,32 +390,41 @@ public struct Driver { return ($0, modTime) }) - let outputFileMap: OutputFileMap? - // Initialize an empty output file map, which will be populated when we start creating jobs. - if let outputFileMapArg = parsedOptions.getLastArgument(.outputFileMap)?.asSingle { - do { - let path = try VirtualPath(path: outputFileMapArg) - outputFileMap = try .load(fileSystem: fileSystem, file: path, diagnosticEngine: diagnosticEngine) - } catch { - throw Error.unableToLoadOutputFileMap(outputFileMapArg) - } + // Create an instance of an inter-module dependency oracle, if the driver's + // client did not provide one. The clients are expected to provide an oracle + // when they wish to share module dependency information across targets. + if let dependencyOracle = interModuleDependencyOracle { + self.interModuleDependencyOracle = dependencyOracle } else { - outputFileMap = nil - } + self.interModuleDependencyOracle = InterModuleDependencyOracle() - if let workingDirectory = self.workingDirectory { - self.outputFileMap = outputFileMap?.resolveRelativePaths(relativeTo: workingDirectory) - } else { - self.outputFileMap = outputFileMap + // This is a shim for backwards-compatibility with ModuleInfoMap-based API + // used by SwiftPM + if let externalArtifacts = externalBuildArtifacts { + if !externalArtifacts.1.isEmpty { + try self.interModuleDependencyOracle.mergeModules(from: externalArtifacts.1) + } + } } - // If requested, print the output file map - if parsedOptions.contains(.driverPrintOutputFileMap) { - if let outputFileMap = self.outputFileMap { - stderrStream <<< outputFileMap.description - stderrStream.flush() + do { + let outputFileMap: OutputFileMap? + // Initialize an empty output file map, which will be populated when we start creating jobs. + if let outputFileMapArg = parsedOptions.getLastArgument(.outputFileMap)?.asSingle { + do { + let path = try VirtualPath(path: outputFileMapArg) + outputFileMap = try .load(fileSystem: fileSystem, file: path, diagnosticEngine: diagnosticEngine) + } catch { + throw Error.unableToLoadOutputFileMap(outputFileMapArg) + } } else { - diagnosticsEngine.emit(.error_no_output_file_map_specified) + outputFileMap = nil + } + + if let workingDirectory = self.workingDirectory { + self.outputFileMap = outputFileMap?.resolveRelativePaths(relativeTo: workingDirectory) + } else { + self.outputFileMap = outputFileMap } } @@ -362,6 +438,7 @@ public struct Driver { self.allSourcesFileList = nil } + self.lto = Self.ltoKind(&parsedOptions, diagnosticsEngine: diagnosticsEngine) // Figure out the primary outputs from the driver. (self.compilerOutputType, self.linkerOutputType) = Self.determinePrimaryOutputs(&parsedOptions, driverKind: driverKind, diagnosticsEngine: diagnosticEngine) @@ -391,24 +468,30 @@ public struct Driver { // Determine the module we're building and whether/how the module file itself will be emitted. self.moduleOutputInfo = try Self.computeModuleInfo( &parsedOptions, compilerOutputType: compilerOutputType, compilerMode: compilerMode, linkerOutputType: linkerOutputType, - debugInfoLevel: debugInfo.level, diagnosticsEngine: diagnosticEngine) + debugInfoLevel: debugInfo.level, diagnosticsEngine: diagnosticEngine, + workingDirectory: self.workingDirectory) + + self.buildRecordInfo = BuildRecordInfo( + actualSwiftVersion: self.frontendTargetInfo.compilerVersion, + compilerOutputType: compilerOutputType, + workingDirectory: self.workingDirectory ?? fileSystem.currentWorkingDirectory, + diagnosticEngine: diagnosticEngine, + fileSystem: fileSystem, + moduleOutputInfo: moduleOutputInfo, + outputFileMap: outputFileMap, + parsedOptions: parsedOptions, + recordedInputModificationDates: recordedInputModificationDates) // Determine the state for incremental compilation self.incrementalCompilationState = IncrementalCompilationState( - &parsedOptions, + buildRecordInfo: buildRecordInfo, compilerMode: compilerMode, - outputFileMap: self.outputFileMap, - compilerOutputType: self.compilerOutputType, - moduleOutput: self.moduleOutputInfo.output, + diagnosticEngine: diagnosticEngine, fileSystem: fileSystem, inputFiles: inputFiles, - diagnosticEngine: diagnosticEngine, - actualSwiftVersion: self.frontendTargetInfo.compilerVersion - ) - - // Local variable to alias the target triple, because self.targetTriple - // is not available until the end of this initializer. - let targetTriple = self.frontendTargetInfo.target.triple + outputFileMap: outputFileMap, + parsedOptions: &parsedOptions, + showJobLifecycle: showJobLifecycle) self.importedObjCHeader = try Self.computeImportedObjCHeader(&parsedOptions, compilerMode: compilerMode, diagnosticEngine: diagnosticEngine) self.bridgingPrecompiledHeader = try Self.computeBridgingPrecompiledHeader(&parsedOptions, @@ -420,7 +503,11 @@ public struct Driver { &parsedOptions, diagnosticEngine: diagnosticEngine, toolchain: toolchain, - targetTriple: targetTriple) + targetInfo: frontendTargetInfo) + + Self.validateSanitizerCoverageArgs(&parsedOptions, + anySanitizersEnabled: !enabledSanitizers.isEmpty, + diagnosticsEngine: diagnosticsEngine) // Supplemental outputs. self.dependenciesFilePath = try Self.computeSupplementaryOutputPath( @@ -430,13 +517,6 @@ public struct Driver { compilerMode: compilerMode, outputFileMap: self.outputFileMap, moduleName: moduleOutputInfo.name) - self.referenceDependenciesFilePath = try Self.computeSupplementaryOutputPath( - &parsedOptions, type: .swiftDeps, isOutputOptions: [.emitReferenceDependencies], - outputPath: .emitReferenceDependenciesPath, - compilerOutputType: compilerOutputType, - compilerMode: compilerMode, - outputFileMap: self.outputFileMap, - moduleName: moduleOutputInfo.name) self.serializedDiagnosticsFilePath = try Self.computeSupplementaryOutputPath( &parsedOptions, type: .diagnostics, isOutputOptions: [.serializeDiagnostics], outputPath: .serializeDiagnosticsPath, @@ -452,13 +532,19 @@ public struct Driver { compilerMode: compilerMode, outputFileMap: self.outputFileMap, moduleName: moduleOutputInfo.name) - self.loadedModuleTracePath = try Self.computeSupplementaryOutputPath( + + if let loadedModuleTraceEnvVar = env["SWIFT_LOADED_MODULE_TRACE_FILE"] { + self.loadedModuleTracePath = try VirtualPath(path: loadedModuleTraceEnvVar) + } else { + self.loadedModuleTracePath = try Self.computeSupplementaryOutputPath( &parsedOptions, type: .moduleTrace, isOutputOptions: [.emitLoadedModuleTrace], outputPath: .emitLoadedModuleTracePath, compilerOutputType: compilerOutputType, compilerMode: compilerMode, outputFileMap: self.outputFileMap, moduleName: moduleOutputInfo.name) + } + self.tbdPath = try Self.computeSupplementaryOutputPath( &parsedOptions, type: .tbd, isOutputOptions: [.emitTbd], outputPath: .emitTbdPath, @@ -491,6 +577,14 @@ public struct Driver { outputFileMap: self.outputFileMap, moduleName: moduleOutputInfo.name) + self.swiftPrivateInterfacePath = try Self.computeSupplementaryOutputPath( + &parsedOptions, type: .privateSwiftInterface, isOutputOptions: [], + outputPath: .emitPrivateModuleInterfacePath, + compilerOutputType: compilerOutputType, + compilerMode: compilerMode, + outputFileMap: self.outputFileMap, + moduleName: moduleOutputInfo.name) + var optimizationRecordFileType = FileType.yamlOptimizationRecord if let argument = parsedOptions.getLastArgument(.saveOptimizationRecordEQ)?.asSingle { switch argument { @@ -578,6 +672,18 @@ extension Driver { } } +extension Driver { + private static func ltoKind(_ parsedOptions: inout ParsedOptions, + diagnosticsEngine: DiagnosticsEngine) -> LTOKind? { + guard let arg = parsedOptions.getLastArgument(.lto)?.asSingle else { return nil } + guard let kind = LTOKind(rawValue: arg) else { + diagnosticsEngine.emit(.error_invalid_arg_value(arg: .lto, value: arg)) + return nil + } + return kind + } +} + // MARK: - Response files. extension Driver { /// Tokenize a single line in a response file. @@ -740,6 +846,16 @@ extension Driver { return } + if parsedOptions.contains(.driverPrintOutputFileMap) { + if let outputFileMap = self.outputFileMap { + stderrStream <<< outputFileMap.description + stderrStream.flush() + } else { + diagnosticEngine.emit(.error_no_output_file_map_specified) + } + return + } + if parsedOptions.contains(.driverPrintBindings) { for job in jobs { printBindings(job) @@ -747,6 +863,14 @@ extension Driver { return } + if parsedOptions.contains(.driverPrintActions) { + // Print actions using the same style as the old C++ driver + // This is mostly for testing purposes. We should print semantically + // equivalent actions as the old driver. + printActions(jobs) + return + } + if parsedOptions.contains(.driverPrintGraphviz) { var serializer = DOTJobGraphSerializer(jobs: jobs) serializer.writeDOT(to: &stdoutStream) @@ -755,10 +879,17 @@ extension Driver { } if jobs.contains(where: { $0.requiresInPlaceExecution }) - // Only one job and no cleanup required - || (jobs.count == 1 && !parsedOptions.hasArgument(.parseableOutput)) { + // Only one job and no cleanup required, e.g. not writing build record + || (jobs.count == 1 && !parsedOptions.hasArgument(.parseableOutput) && + buildRecordInfo == nil) { assert(jobs.count == 1, "Cannot execute in place for multi-job build plans") var job = jobs[0] + // Print the driver source version first before we print the compiler + // versions. + if job.kind == .versionRequest && !Driver.driverSourceVersion.isEmpty { + stderrStream <<< "swift-driver version: " <<< Driver.driverSourceVersion <<< " " + stderrStream.flush() + } // Require in-place execution for all single job plans. job.requiresInPlaceExecution = true try executor.execute(job: job, @@ -766,16 +897,14 @@ extension Driver { recordedInputModificationDates: recordedInputModificationDates) return } - - // Create and use the tool execution delegate if one is not provided explicitly. - let executorDelegate = createToolExecutionDelegate() - - // Perform the build - try executor.execute(jobs: jobs, - delegate: executorDelegate, - numParallelJobs: numParallelJobs ?? 1, - forceResponseFiles: forceResponseFiles, - recordedInputModificationDates: recordedInputModificationDates) + do { + defer { + buildRecordInfo?.writeBuildRecord( + jobs, + incrementalCompilationState?.skippedCompilationInputs) + } + try performTheBuild(allJobs: jobs, forceResponseFiles: forceResponseFiles) + } // If requested, warn for options that weren't used by the driver after the build is finished. if parsedOptions.hasArgument(.driverWarnUnusedOptions) { @@ -795,14 +924,34 @@ extension Driver { mode = .verbose } - return ToolExecutionDelegate(mode: mode) + return ToolExecutionDelegate( + mode: mode, + buildRecordInfo: buildRecordInfo, + incrementalCompilationState: incrementalCompilationState, + showJobLifecycle: showJobLifecycle, + diagnosticEngine: diagnosticEngine) + } + + private mutating func performTheBuild( + allJobs: [Job], + forceResponseFiles: Bool + ) throws { + let continueBuildingAfterErrors = computeContinueBuildingAfterErrors() + try executor.execute( + workload: .init(allJobs, + incrementalCompilationState, + continueBuildingAfterErrors: continueBuildingAfterErrors), + delegate: createToolExecutionDelegate(), + numParallelJobs: numParallelJobs ?? 1, + forceResponseFiles: forceResponseFiles, + recordedInputModificationDates: recordedInputModificationDates) } private func printBindings(_ job: Job) { stdoutStream <<< #"# ""# <<< targetTriple.triple stdoutStream <<< #"" - ""# <<< job.tool.basename stdoutStream <<< #"", inputs: ["# - stdoutStream <<< job.inputs.map { "\"" + $0.file.name + "\"" }.joined(separator: ", ") + stdoutStream <<< job.displayInputs.map { "\"" + $0.file.name + "\"" }.joined(separator: ", ") stdoutStream <<< "], output: {" @@ -813,8 +962,98 @@ extension Driver { stdoutStream.flush() } + /// This handles -driver-print-actions flag. The C++ driver has a concept of actions + /// which it builds up a list of actions before then creating them into jobs. + /// The swift-driver doesn't have actions, so the logic here takes the jobs and tries + /// to mimic the actions that would be created by the C++ driver and + /// prints them in *hopefully* the same order. + private func printActions(_ jobs: [Job]) { + defer { + stdoutStream.flush() + } + + // Put bridging header as first input if we have it + let allInputs: [TypedVirtualPath] + if let objcHeader = importedObjCHeader, bridgingPrecompiledHeader != nil { + allInputs = [TypedVirtualPath(file: objcHeader, type: .objcHeader)] + inputFiles + } else { + allInputs = inputFiles + } + + var jobIdMap = Dictionary() + // The C++ driver treats each input as an action, we should print them as + // an action too for testing purposes. + var inputIdMap = Dictionary() + var nextId: UInt = 0 + var allInputsIterator = allInputs.makeIterator() + for job in jobs { + // After "module input" jobs, print any left over inputs + switch job.kind { + case .generatePCH, .compile, .backend: + break + default: + while let input = allInputsIterator.next() { + Self.printInputIfNew(input, inputIdMap: &inputIdMap, nextId: &nextId) + } + } + // All input action IDs for this action. + var inputIds = [UInt]() + // Collect input job IDs. + for input in job.displayInputs.isEmpty ? job.inputs : job.displayInputs { + if let id = inputIdMap[input] { + inputIds.append(id) + continue + } + var foundInput = false + for (prevJob, id) in jobIdMap { + if prevJob.outputs.contains(input) { + foundInput = true + inputIds.append(id) + break + } + } + if !foundInput { + while let nextInputAction = allInputsIterator.next() { + Self.printInputIfNew(nextInputAction, inputIdMap: &inputIdMap, nextId: &nextId) + if let id = inputIdMap[input] { + inputIds.append(id) + break + } + } + } + } + + // Print current Job + stdoutStream <<< nextId <<< ": " <<< job.kind.rawValue <<< ", {" + switch job.kind { + // Don't sort for compile jobs. Puts pch last + case .compile: + stdoutStream <<< inputIds.map(\.description).joined(separator: ", ") + default: + stdoutStream <<< inputIds.sorted().map(\.description).joined(separator: ", ") + } + var typeName = job.outputs.first?.type.name + if typeName == nil { + typeName = "none" + } + stdoutStream <<< "}, " <<< typeName! <<< "\n" + jobIdMap[job] = nextId + nextId += 1 + } + } + + private static func printInputIfNew(_ input: TypedVirtualPath, inputIdMap: inout [TypedVirtualPath: UInt], nextId: inout UInt) { + if inputIdMap[input] == nil { + stdoutStream <<< nextId <<< ": " <<< "input, " + stdoutStream <<< "\"" <<< input.file <<< "\", " <<< input.type <<< "\n" + inputIdMap[input] = nextId + nextId += 1 + } + } + private func printVersion(outputStream: inout S) throws { - outputStream.write(try executor.checkNonZeroExit(args: toolchain.getToolPath(.swiftCompiler).pathString, "--version", environment: env)) + outputStream <<< frontendTargetInfo.compilerVersion <<< "\n" + outputStream <<< "Target: \(frontendTargetInfo.target.triple.triple)\n" outputStream.flush() } } @@ -904,10 +1143,15 @@ extension Driver { // Some output flags affect the compiler mode. if let outputOption = parsedOptions.getLast(in: .modes) { switch outputOption.option { - case .emitPch, .emitImportedModules: + case .emitImportedModules: return .singleCompile - case .repl, .lldbRepl: + case .repl: + if driverKind == .interactive, !parsedOptions.hasAnyInput { + diagnosticsEngine.emit(.warning_unnecessary_repl_mode(option: outputOption.option, kind: driverKind)) + } + fallthrough + case .lldbRepl: return .repl case .deprecatedIntegratedRepl: @@ -930,6 +1174,26 @@ extension Driver { let hasIndexFile = parsedOptions.hasArgument(.indexFile) let wantBatchMode = parsedOptions.hasFlag(positive: .enableBatchMode, negative: .disableBatchMode, default: false) + // AST dump doesn't work with `-wmo`/`-index-file`. Since it's not common to want to dump + // the AST, we assume that's the priority and ignore those flags, but we warn the + // user about this decision. + if useWMO && parsedOptions.hasArgument(.dumpAst) { + diagnosticsEngine.emit(.warning_option_overrides_another(overridingOption: .dumpAst, + overridenOption: .wmo)) + parsedOptions.eraseArgument(.wmo) + return .standardCompile + } + + if hasIndexFile && parsedOptions.hasArgument(.dumpAst) { + diagnosticsEngine.emit(.warning_option_overrides_another(overridingOption: .dumpAst, + overridenOption: .indexFile)) + parsedOptions.eraseArgument(.indexFile) + parsedOptions.eraseArgument(.indexFilePath) + parsedOptions.eraseArgument(.indexStorePath) + parsedOptions.eraseArgument(.indexIgnoreSystemModules) + return .standardCompile + } + if useWMO || hasIndexFile { if wantBatchMode { let disablingOption: Option = useWMO ? .wholeModuleOptimization : .indexFile @@ -952,6 +1216,10 @@ extension Driver { } extension Diagnostic.Message { + static func warning_unnecessary_repl_mode(option: Option, kind: DriverKind) -> Diagnostic.Message { + .warning("unnecessary option '\(option.spelling)'; this is the default for '\(kind.rawValue)' with no input files") + } + static func warn_ignoring_batch_mode(_ option: Option) -> Diagnostic.Message { .warning("ignoring '-enable-batch-mode' because '\(option.spelling)' was also specified") } @@ -1023,6 +1291,7 @@ extension Driver { // By default, the driver does not link its output. However, this will be updated below. var compilerOutputType: FileType? = (driverKind == .interactive ? nil : .object) var linkerOutputType: LinkOutputType? = nil + let objectLikeFileType: FileType = parsedOptions.getLastArgument(.lto) != nil ? .llvmBitcode : .object if let outputOption = parsedOptions.getLast(in: .modes) { switch outputOption.option { @@ -1031,11 +1300,11 @@ extension Driver { diagnosticsEngine.emit(.error_static_emit_executable_disallowed) } linkerOutputType = .executable - compilerOutputType = .object + compilerOutputType = objectLikeFileType case .emitLibrary: linkerOutputType = parsedOptions.hasArgument(.static) ? .staticLibrary : .dynamicLibrary - compilerOutputType = .object + compilerOutputType = objectLikeFileType case .emitObject, .c: compilerOutputType = .object @@ -1064,9 +1333,6 @@ extension Driver { case .dumpAst: compilerOutputType = .ast - case .emitPch: - compilerOutputType = .pch - case .emitPcm: compilerOutputType = .pcm @@ -1086,7 +1352,7 @@ extension Driver { compilerOutputType = nil case .i: - diagnosticsEngine.emit(.error_i_mode(driverKind)) + diagnosticsEngine.emit(.error_i_mode) case .repl, .deprecatedIntegratedRepl, .lldbRepl: compilerOutputType = nil @@ -1106,6 +1372,7 @@ extension Driver { } else if parsedOptions.hasArgument(.emitModule, .emitModulePath) { compilerOutputType = .swiftModule } else if driverKind != .interactive { + compilerOutputType = objectLikeFileType linkerOutputType = .executable } @@ -1124,11 +1391,11 @@ extension Driver { } extension Diagnostic.Message { - static func error_i_mode(_ driverKind: DriverKind) -> Diagnostic.Message { + static var error_i_mode: Diagnostic.Message { .error( """ the flag '-i' is no longer required and has been removed; \ - use '\(driverKind.usage) input-filename' + use '\(DriverKind.interactive.usage) input-filename' """ ) } @@ -1190,8 +1457,25 @@ extension Driver { return numJobs } + + private mutating func computeContinueBuildingAfterErrors() -> Bool { + // Note: Batch mode handling of serialized diagnostics requires that all + // batches get to run, in order to make sure that all diagnostics emitted + // during the compilation end up in at least one serialized diagnostic file. + // Therefore, treat batch mode as implying -continue-building-after-errors. + // (This behavior could be limited to only when serialized diagnostics are + // being emitted, but this seems more consistent and less surprising for + // users.) + // FIXME: We don't really need (or want) a full ContinueBuildingAfterErrors. + // If we fail to precompile a bridging header, for example, there's no need + // to go on to compilation of source files, and if compilation of source files + // fails, we shouldn't try to link. Instead, we'd want to let all jobs finish + // but not schedule any new ones. + return compilerMode.isBatchCompile || parsedOptions.contains(.continueBuildingAfterErrors) + } } + extension Diagnostic.Message { static func remark_max_determinism_overriding(_ option: Option) -> Diagnostic.Message { .remark("SWIFTC_MAXIMUM_DETERMINISM overriding \(option.spelling)") @@ -1248,7 +1532,7 @@ extension Driver { } if !parsedOptions.contains(in: .g) { - diagnosticsEngine.emit(.error_option_missing_required_argument(option: .debugInfoFormat, requiredArg: .g)) + diagnosticsEngine.emit(.error_option_missing_required_argument(option: .debugInfoFormat, requiredArg: "-g")) } } else { // Default to DWARF. @@ -1257,7 +1541,8 @@ extension Driver { if format == .codeView && (level == .lineTables || level == .dwarfTypes) { let levelOption = parsedOptions.getLast(in: .g)!.option - diagnosticsEngine.emit(.error_argument_not_allowed_with(arg: format.rawValue, other: levelOption.spelling)) + let fullNotAllowedOption = Option.debugInfoFormat.spelling + format.rawValue + diagnosticsEngine.emit(.error_argument_not_allowed_with(arg: fullNotAllowedOption, other: levelOption.spelling)) } return DebugInfo(format: format, level: level, shouldVerify: shouldVerify) @@ -1269,7 +1554,7 @@ extension Driver { _ parsedOptions: inout ParsedOptions, diagnosticEngine: DiagnosticsEngine, toolchain: Toolchain, - targetTriple: Triple + targetInfo: FrontendTargetInfo ) throws -> Set { var set = Set() @@ -1278,6 +1563,12 @@ extension Driver { .filter { $0.option == .sanitizeEQ } .flatMap { $0.argument.asMultiple } + // No sanitizer args found, we could return. + if args.isEmpty { + return set + } + + let targetTriple = targetInfo.target.triple // Find the sanitizer kind. for arg in args { guard let sanitizer = Sanitizer(rawValue: arg) else { @@ -1292,7 +1583,7 @@ extension Driver { // enabled. var sanitizerSupported = try toolchain.runtimeLibraryExists( for: sanitizer, - targetTriple: targetTriple, + targetInfo: targetInfo, parsedOptions: &parsedOptions, isShared: sanitizer != .fuzzer ) @@ -1420,7 +1711,8 @@ extension Driver { compilerMode: CompilerMode, linkerOutputType: LinkOutputType?, debugInfoLevel: DebugInfo.Level?, - diagnosticsEngine: DiagnosticsEngine + diagnosticsEngine: DiagnosticsEngine, + workingDirectory: AbsolutePath? ) throws -> ModuleOutputInfo { // Figure out what kind of module we will output. enum ModuleOutputKind { @@ -1439,7 +1731,8 @@ extension Driver { moduleOutputKind = .auxiliary } else if compilerMode != .singleCompile && parsedOptions.hasArgument(.emitObjcHeader, .emitObjcHeaderPath, - .emitModuleInterface, .emitModuleInterfacePath) { + .emitModuleInterface, .emitModuleInterfacePath, + .emitPrivateModuleInterfacePath) { // An option has been passed which requires whole-module knowledge, but we // don't have that. Generate a module, but treat it as an intermediate // output. @@ -1503,7 +1796,7 @@ extension Driver { } // Determine the module file to output. - let moduleOutputPath: VirtualPath + var moduleOutputPath: VirtualPath // FIXME: Look in the output file map. It looks like it is weirdly // anchored to the first input? @@ -1526,6 +1819,11 @@ extension Driver { moduleOutputPath = .temporary(RelativePath(moduleName.appendingFileTypeExtension(.swiftModule))) } + // Use working directory if specified + if let moduleRelative = moduleOutputPath.relativePath { + moduleOutputPath = Driver.useWorkingDirectory(moduleRelative, workingDirectory) + } + switch moduleOutputKind! { case .topLevel: return ModuleOutputInfo(output: .topLevel(moduleOutputPath), name: moduleName, nameIsFallback: moduleNameIsFallback) @@ -1722,6 +2020,28 @@ extension Driver { } } } + + private static func validateSanitizerCoverageArgs(_ parsedOptions: inout ParsedOptions, + anySanitizersEnabled: Bool, + diagnosticsEngine: DiagnosticsEngine) { + var foundRequiredArg = false + for arg in parsedOptions.arguments(for: .sanitizeCoverageEQ).flatMap(\.argument.asMultiple) { + if ["func", "bb", "edge"].contains(arg) { + foundRequiredArg = true + } else if !["indirect-calls", "trace-bb", "trace-cmp", "8bit-counters", "trace-pc", "trace-pc-guard"].contains(arg) { + diagnosticsEngine.emit(.error_unsupported_argument(argument: arg, option: .sanitizeCoverageEQ)) + } + + if !foundRequiredArg { + diagnosticsEngine.emit(.error_option_missing_required_argument(option: .sanitizeCoverageEQ, + requiredArg: #""func", "bb", "edge""#)) + } + } + + if parsedOptions.hasArgument(.sanitizeCoverageEQ) && !anySanitizersEnabled { + diagnosticsEngine.emit(.error_option_requires_sanitizer(option: .sanitizeCoverageEQ)) + } + } } extension Triple { @@ -1733,6 +2053,8 @@ extension Triple { return GenericUnixToolchain.self case .freeBSD, .haiku: return GenericUnixToolchain.self + case .wasi: + return WebAssemblyToolchain.self case .win32: return WindowsToolchain.self default: @@ -1758,7 +2080,8 @@ extension Driver { compilerMode: CompilerMode, env: [String: String], executor: DriverExecutor, - fileSystem: FileSystem + fileSystem: FileSystem, + useStaticResourceDir: Bool ) throws -> (Toolchain, FrontendTargetInfo, [String]) { let explicitTarget = (parsedOptions.getLastArgument(.target)?.asSingle) .map { @@ -1779,24 +2102,37 @@ extension Driver { let toolchainType = try explicitTarget?.toolchainType(diagnosticsEngine) ?? defaultToolchainType - let toolchain = toolchainType.init(env: env, executor: executor, fileSystem: fileSystem) + // Find tools directory and pass it down to the toolchain + var toolDir: AbsolutePath? + if let td = parsedOptions.getLastArgument(.toolsDirectory) { + toolDir = try AbsolutePath(validating: td.asSingle) + } + let toolchain = toolchainType.init(env: env, executor: executor, + fileSystem: fileSystem, + toolDirectory: toolDir) // Find the Swift compiler executable. let swiftCompilerPrefixArgs: [String] + let hasFrontendBeenRedirectedForTesting: Bool if let frontendPath = parsedOptions.getLastArgument(.driverUseFrontendPath){ var frontendCommandLine = frontendPath.asSingle.split(separator: ";").map { String($0) } if frontendCommandLine.isEmpty { diagnosticsEngine.emit(.error_no_swift_frontend) swiftCompilerPrefixArgs = [] + hasFrontendBeenRedirectedForTesting = false } else { - let frontendPath = frontendCommandLine.removeFirst() - toolchain.overrideToolPath( - .swiftCompiler, path: try AbsolutePath(validating: frontendPath)) + let frontendPathString = frontendCommandLine.removeFirst() + let frontendPath = try AbsolutePath(validating: frontendPathString) + toolchain.overrideToolPath(.swiftCompiler, path: frontendPath) swiftCompilerPrefixArgs = frontendCommandLine + // The tests in Driver/Dependencies redirect the frontend to a python + // script, so don't ask the frontend for target info in that case. + hasFrontendBeenRedirectedForTesting = frontendPath.basename == "Python" } } else { swiftCompilerPrefixArgs = [] + hasFrontendBeenRedirectedForTesting = false } // Find the SDK, if any. @@ -1805,14 +2141,19 @@ extension Driver { targetTriple: explicitTarget, fileSystem: fileSystem, diagnosticsEngine: diagnosticsEngine, env: env) - // Query the frontend to for target information. + // Query the frontend for target information. + // If there's a dummy frontend, don't query it. do { - var info = try executor.execute( + var info = hasFrontendBeenRedirectedForTesting + ? FrontendTargetInfo.dummyForTesting(toolchain) + : try executor.execute( job: toolchain.printTargetInfoJob( target: explicitTarget, targetVariant: explicitTargetVariant, sdkPath: sdkPath, resourceDirPath: resourceDirPath, runtimeCompatibilityVersion: - parsedOptions.getLastArgument(.runtimeCompatibilityVersion)?.asSingle + parsedOptions.getLastArgument(.runtimeCompatibilityVersion)?.asSingle, + useStaticResourceDir: useStaticResourceDir, + swiftCompilerPrefixArgs: swiftCompilerPrefixArgs ), capturingJSONOutputAs: FrontendTargetInfo.self, forceResponseFiles: false, @@ -1826,16 +2167,47 @@ extension Driver { if let version = SwiftVersion(string: versionString) { info.target.swiftRuntimeCompatibilityVersion = version info.targetVariant?.swiftRuntimeCompatibilityVersion = version - } else { + } else if (versionString != "none") { + // "none" was accepted by the old driver, diagnose other values. diagnosticsEngine.emit( .error_invalid_arg_value( arg: .runtimeCompatibilityVersion, value: versionString)) } } + // Check if the simulator environment was inferred for backwards compatibility. + if let explicitTarget = explicitTarget, + explicitTarget.environment != .simulator && info.target.triple.environment == .simulator { + diagnosticsEngine.emit(.warning_inferring_simulator_target(originalTriple: explicitTarget, + inferredTriple: info.target.triple)) + } + return (toolchain, info, swiftCompilerPrefixArgs) - } catch is DecodingError { - throw Error.unableToDecodeFrontendTargetInfo + } catch let JobExecutionError.decodingError(decodingError, + dataToDecode, + processResult) { + let stringToDecode = String(data: dataToDecode, encoding: .utf8) + let errorDesc: String + switch decodingError { + case let .typeMismatch(type, context): + errorDesc = "type mismatch: \(type), path: \(context.codingPath)" + case let .valueNotFound(type, context): + errorDesc = "value missing: \(type), path: \(context.codingPath)" + case let .keyNotFound(key, context): + errorDesc = "key missing: \(key), path: \(context.codingPath)" + case let .dataCorrupted(context): + errorDesc = "data corrupted at path: \(context.codingPath)" + @unknown default: + errorDesc = "unknown decoding error" + } + throw Error.unableToDecodeFrontendTargetInfo( + stringToDecode, + processResult.arguments, + errorDesc) + } catch let JobExecutionError.jobFailedWithNonzeroExitCode(returnCode, stdout) { + throw Error.failedToRunFrontendToRetrieveTargetInfo(returnCode, stdout) + } catch JobExecutionError.failedToReadJobOutput { + throw Error.unableToReadFrontendTargetInfo } catch { throw Error.failedToRetrieveFrontendTargetInfo } @@ -1994,7 +2366,7 @@ extension Driver { parentPath = moduleOutputPath.parentDirectory } - return parentPath.appending(component: moduleName).replacingExtension(with: type) + return parentPath.appending(component: moduleOutputPath.basename).replacingExtension(with: type) } // If the output option was not provided, don't produce this output at all. diff --git a/Sources/SwiftDriver/Driver/DriverVersion.swift b/Sources/SwiftDriver/Driver/DriverVersion.swift new file mode 100644 index 000000000..4346d2bfa --- /dev/null +++ b/Sources/SwiftDriver/Driver/DriverVersion.swift @@ -0,0 +1,18 @@ +//===------ DriverVersion.swift - Swift Driver Source Version--------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +extension Driver { +#if SWIFT_DRIVER_VERSION_DEFINED + static let driverSourceVersion: String = SWIFT_DRIVER_VERSION +#else + static let driverSourceVersion: String = "" +#endif +} diff --git a/Sources/SwiftDriver/Driver/LinkKind.swift b/Sources/SwiftDriver/Driver/LinkKind.swift index e996eb317..628a07310 100644 --- a/Sources/SwiftDriver/Driver/LinkKind.swift +++ b/Sources/SwiftDriver/Driver/LinkKind.swift @@ -21,3 +21,11 @@ public enum LinkOutputType { /// A static library (e.g., .a or .lib) case staticLibrary } + +/// Describes the kind of link-time-optimization we expect to perform. +public enum LTOKind: String, Hashable { + /// Perform LLVM ThinLTO. + case llvmThin = "llvm-thin" + /// Perform LLVM full LTO. + case llvmFull = "llvm-full" +} diff --git a/Sources/SwiftDriver/Driver/ModuleOutputInfo.swift b/Sources/SwiftDriver/Driver/ModuleOutputInfo.swift index efb7f6562..7c372d25f 100644 --- a/Sources/SwiftDriver/Driver/ModuleOutputInfo.swift +++ b/Sources/SwiftDriver/Driver/ModuleOutputInfo.swift @@ -11,7 +11,7 @@ //===----------------------------------------------------------------------===// /// The information about module output produced by the driver. -public struct ModuleOutputInfo { +@_spi(Testing) public struct ModuleOutputInfo { /// How should the Swift module output be handled? public enum ModuleOutput: Equatable { diff --git a/Sources/SwiftDriver/Driver/OutputFileMap.swift b/Sources/SwiftDriver/Driver/OutputFileMap.swift index 0e9123b58..83e279f6a 100644 --- a/Sources/SwiftDriver/Driver/OutputFileMap.swift +++ b/Sources/SwiftDriver/Driver/OutputFileMap.swift @@ -12,8 +12,6 @@ import TSCBasic import Foundation -public typealias FileSystem = TSCBasic.FileSystem - /// Mapping of input file paths to specific output files. public struct OutputFileMap: Hashable, Codable { static let singleInputKey = VirtualPath.relative(RelativePath("")) @@ -44,7 +42,16 @@ public struct OutputFileMap: Hashable, Codable { } public func existingOutput(inputFile: VirtualPath, outputType: FileType) -> VirtualPath? { - entries[inputFile]?[outputType] + if let path = entries[inputFile]?[outputType] { + return path + } + switch outputType { + case .swiftDocumentation, .swiftSourceInfoFile: + // Infer paths for these entities using .swiftmodule path. + return entries[inputFile]?[.swiftModule]?.replacingExtension(with: outputType) + default: + return nil + } } public func existingOutputForSingleInput(outputType: FileType) -> VirtualPath? { @@ -70,8 +77,19 @@ public struct OutputFileMap: Hashable, Codable { })) } + /// Slow, but only for debugging output + public func getInput(outputFile: VirtualPath) -> VirtualPath? { + entries + .compactMap { + $0.value.values.contains(outputFile) + ? $0.key + : nil + } + .first + } + /// Load the output file map at the given path. - public static func load( + @_spi(Testing) public static func load( fileSystem: FileSystem, file: VirtualPath, diagnosticEngine: DiagnosticsEngine @@ -140,7 +158,7 @@ fileprivate struct OutputFileMapJSON: Codable { } /// The data associated with an input file. - /// \c fileprivate so that the \c store method above can see it + /// `fileprivate` so that the `store` method above can see it fileprivate struct Entry: Codable { private struct CodingKeys: CodingKey { @@ -184,7 +202,7 @@ fileprivate struct OutputFileMapJSON: Codable { } /// The parsed entries - /// \c fileprivate so that the \c store method above can see it + /// `fileprivate` so that the `store` method above can see it fileprivate let entries: [String: Entry] init(from decoder: Decoder) throws { @@ -228,10 +246,3 @@ extension String { return self + "." + ext } } - -extension VirtualPath { - fileprivate func resolvedRelativePath(base: AbsolutePath) -> VirtualPath { - guard case let .relative(relPath) = self else { return self } - return .absolute(.init(base, relPath)) - } -} diff --git a/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift b/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift index 06fb3cb28..131b2e5eb 100644 --- a/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift +++ b/Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift @@ -14,7 +14,7 @@ import TSCBasic #if canImport(Darwin) import Darwin.C #elseif os(Windows) -import MSVCRT +import ucrt import WinSDK #elseif canImport(Glibc) import Glibc @@ -23,7 +23,7 @@ import Glibc #endif /// Delegate for printing execution information on the command-line. -public struct ToolExecutionDelegate: JobExecutionDelegate { +struct ToolExecutionDelegate: JobExecutionDelegate { public enum Mode { case verbose case parsableOutput @@ -31,8 +31,16 @@ public struct ToolExecutionDelegate: JobExecutionDelegate { } public let mode: Mode + public let buildRecordInfo: BuildRecordInfo? + public let incrementalCompilationState: IncrementalCompilationState? + public let showJobLifecycle: Bool + public let diagnosticEngine: DiagnosticsEngine + public func jobStarted(job: Job, arguments: [String], pid: Int) { + if showJobLifecycle { + diagnosticEngine.emit(.remark_job_lifecycle("Starting", job)) + } switch mode { case .regular: break @@ -60,6 +68,12 @@ public struct ToolExecutionDelegate: JobExecutionDelegate { } public func jobFinished(job: Job, result: ProcessResult, pid: Int) { + if showJobLifecycle { + diagnosticEngine.emit(.remark_job_lifecycle("Finished", job)) + } + + buildRecordInfo?.jobFinished(job: job, result: result) + switch mode { case .regular, .verbose: let output = (try? result.utf8Output() + result.utf8stderrOutput()) ?? "" @@ -97,3 +111,10 @@ public struct ToolExecutionDelegate: JobExecutionDelegate { stderrStream.flush() } } + +fileprivate extension Diagnostic.Message { + static func remark_job_lifecycle(_ what: String, _ job: Job + ) -> Diagnostic.Message { + .remark("\(what) \(job.descriptionForLifecycle)") + } +} diff --git a/Sources/SwiftDriver/Execution/ArgsResolver.swift b/Sources/SwiftDriver/Execution/ArgsResolver.swift index 50069dec3..29460396c 100644 --- a/Sources/SwiftDriver/Execution/ArgsResolver.swift +++ b/Sources/SwiftDriver/Execution/ArgsResolver.swift @@ -84,14 +84,22 @@ public final class ArgsResolver { // Return the path from the temporary directory if this is a temporary file. if path.isTemporary { let actualPath = temporaryDirectory.appending(component: path.name) - if case let .fileList(_, fileList) = path { - switch fileList { - case let .list(items): - try createFileList(path: actualPath, contents: items) - case let .outputFileMap(map): - try createFileList(path: actualPath, outputFileMap: map) + switch path { + case .temporary: + break // No special behavior required. + case let .temporaryWithKnownContents(_, contents): + // FIXME: Need a way to support this for distributed build systems... + if let absolutePath = actualPath.absolutePath { + try fileSystem.writeFileContents(absolutePath, bytes: .init(contents)) } + case let .fileList(_, .list(items)): + try createFileList(path: actualPath, contents: items) + case let .fileList(_, .outputFileMap(map)): + try createFileList(path: actualPath, outputFileMap: map) + case .relative, .absolute, .standardInput, .standardOutput: + fatalError("Not a temporary path.") } + let result = actualPath.name pathMapping[path] = result return result @@ -133,7 +141,9 @@ public final class ArgsResolver { private func quoteAndEscape(path: VirtualPath) -> String { let inputNode = Node.scalar(Node.Scalar(try! unsafeResolve(path: path), Tag(.str), .doubleQuoted)) - let string = try! Yams.serialize(node: inputNode) + // Width parameter of -1 sets preferred line-width to unlimited so that no extraneous + // line-breaks will be inserted during serialization. + let string = try! Yams.serialize(node: inputNode, width: -1) // Remove the newline from the end return string.trimmingCharacters(in: .whitespacesAndNewlines) } diff --git a/Sources/SwiftDriver/Execution/DriverExecutor.swift b/Sources/SwiftDriver/Execution/DriverExecutor.swift index ff2b146f4..b1c88a19e 100644 --- a/Sources/SwiftDriver/Execution/DriverExecutor.swift +++ b/Sources/SwiftDriver/Execution/DriverExecutor.swift @@ -22,6 +22,15 @@ public protocol DriverExecutor { forceResponseFiles: Bool, recordedInputModificationDates: [TypedVirtualPath: Date]) throws -> ProcessResult + /// Execute multiple jobs, tracking job status using the provided execution delegate. + /// Pass in the `IncrementalCompilationState` to allow for incremental compilation. + func execute(workload: DriverExecutorWorkload, + delegate: JobExecutionDelegate, + numParallelJobs: Int, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath: Date] + ) throws + /// Execute multiple jobs, tracking job status using the provided execution delegate. func execute(jobs: [Job], delegate: JobExecutionDelegate, @@ -29,7 +38,7 @@ public protocol DriverExecutor { forceResponseFiles: Bool, recordedInputModificationDates: [TypedVirtualPath: Date] ) throws - + /// Launch a process with the given command line and report the result. @discardableResult func checkNonZeroExit(args: String..., environment: [String: String]) throws -> String @@ -38,9 +47,34 @@ public protocol DriverExecutor { func description(of job: Job, forceResponseFiles: Bool) throws -> String } +public struct DriverExecutorWorkload { + public let continueBuildingAfterErrors: Bool + public enum Kind { + case all([Job]) + case incremental(IncrementalCompilationState) + } + public let kind: Kind + + public init(_ allJobs: [Job], + _ incrementalCompilationState: IncrementalCompilationState?, + continueBuildingAfterErrors: Bool + ) { + self.continueBuildingAfterErrors = continueBuildingAfterErrors + self.kind = incrementalCompilationState + .map {.incremental($0)} + ?? .all(allJobs) + } + + static public func all(_ jobs: [Job]) -> Self { + .init(jobs, nil, continueBuildingAfterErrors: false) + } +} + enum JobExecutionError: Error { case jobFailedWithNonzeroExitCode(Int, String) case failedToReadJobOutput + // A way to pass more information to the catch point + case decodingError(DecodingError, Data, ProcessResult) } extension DriverExecutor { @@ -59,8 +93,28 @@ extension DriverExecutor { guard let outputData = try? Data(result.utf8Output().utf8) else { throw JobExecutionError.failedToReadJobOutput } - - return try JSONDecoder().decode(outputType, from: outputData) + + do { + return try JSONDecoder().decode(outputType, from: outputData) + } + catch let err as DecodingError { + throw JobExecutionError.decodingError(err, outputData, result) + } + } + + public func execute( + jobs: [Job], + delegate: JobExecutionDelegate, + numParallelJobs: Int, + forceResponseFiles: Bool, + recordedInputModificationDates: [TypedVirtualPath: Date] + ) throws { + try execute( + workload: .all(jobs), + delegate: delegate, + numParallelJobs: numParallelJobs, + forceResponseFiles: forceResponseFiles, + recordedInputModificationDates: recordedInputModificationDates) } static func computeReturnCode(exitStatus: ProcessResult.ExitStatus) -> Int { @@ -84,87 +138,3 @@ public protocol JobExecutionDelegate { /// Called when a job finished. func jobFinished(job: Job, result: ProcessResult, pid: Int) } - -public final class SwiftDriverExecutor: DriverExecutor { - let diagnosticsEngine: DiagnosticsEngine - let processSet: ProcessSet - let fileSystem: FileSystem - public let resolver: ArgsResolver - let env: [String: String] - - public init(diagnosticsEngine: DiagnosticsEngine, - processSet: ProcessSet, - fileSystem: FileSystem, - env: [String: String]) throws { - self.diagnosticsEngine = diagnosticsEngine - self.processSet = processSet - self.fileSystem = fileSystem - self.env = env - self.resolver = try ArgsResolver(fileSystem: fileSystem) - } - - public func execute(job: Job, - forceResponseFiles: Bool = false, - recordedInputModificationDates: [TypedVirtualPath: Date] = [:]) throws -> ProcessResult { - let arguments: [String] = try resolver.resolveArgumentList(for: job, - forceResponseFiles: forceResponseFiles) - - try job.verifyInputsNotModified(since: recordedInputModificationDates, - fileSystem: fileSystem) - - if job.requiresInPlaceExecution { - for (envVar, value) in job.extraEnvironment { - try ProcessEnv.setVar(envVar, value: value) - } - - try exec(path: arguments[0], args: arguments) - fatalError("unreachable, exec always throws on failure") - } else { - var childEnv = env - childEnv.merge(job.extraEnvironment, uniquingKeysWith: { (_, new) in new }) - - let process = try Process.launchProcess(arguments: arguments, env: childEnv) - return try process.waitUntilExit() - } - } - - public func execute(jobs: [Job], - delegate: JobExecutionDelegate, - numParallelJobs: Int = 1, - forceResponseFiles: Bool = false, - recordedInputModificationDates: [TypedVirtualPath: Date] = [:] - ) throws { - let llbuildExecutor = MultiJobExecutor(jobs: jobs, - resolver: resolver, - executorDelegate: delegate, - diagnosticsEngine: diagnosticsEngine, - numParallelJobs: numParallelJobs, - processSet: processSet, - forceResponseFiles: forceResponseFiles, - recordedInputModificationDates: recordedInputModificationDates) - try llbuildExecutor.execute(env: env, fileSystem: fileSystem) - } - - @discardableResult - public func checkNonZeroExit(args: String..., environment: [String: String] = ProcessEnv.vars) throws -> String { - return try Process.checkNonZeroExit(arguments: args, environment: environment) - } - - public func description(of job: Job, forceResponseFiles: Bool) throws -> String { - let (args, usedResponseFile) = try resolver.resolveArgumentList(for: job, forceResponseFiles: forceResponseFiles) - var result = args.joined(separator: " ") - - if usedResponseFile { - // Print the response file arguments as a comment. - result += " # \(job.commandLine.joinedArguments)" - } - - if !job.extraEnvironment.isEmpty { - result += " #" - for (envVar, val) in job.extraEnvironment { - result += " \(envVar)=\(val)" - } - } - return result - } -} diff --git a/Sources/SwiftDriver/Execution/ParsableOutput.swift b/Sources/SwiftDriver/Execution/ParsableOutput.swift index 432f0a4a6..298f95a77 100644 --- a/Sources/SwiftDriver/Execution/ParsableOutput.swift +++ b/Sources/SwiftDriver/Execution/ParsableOutput.swift @@ -12,7 +12,7 @@ import Foundation -public struct ParsableMessage { +@_spi(Testing) public struct ParsableMessage { public enum Kind { case began(BeganMessage) case finished(FinishedMessage) @@ -39,7 +39,7 @@ public struct ParsableMessage { } } -public struct BeganMessage: Encodable { +@_spi(Testing) public struct BeganMessage: Encodable { public struct Output: Encodable { public let type: String public let path: String @@ -79,7 +79,7 @@ public struct BeganMessage: Encodable { } } -public struct FinishedMessage: Encodable { +@_spi(Testing) public struct FinishedMessage: Encodable { let exitStatus: Int let pid: Int let output: String? @@ -103,7 +103,7 @@ public struct FinishedMessage: Encodable { } } -public struct SignalledMessage: Encodable { +@_spi(Testing) public struct SignalledMessage: Encodable { let pid: Int let output: String? let errorMessage: String @@ -124,7 +124,7 @@ public struct SignalledMessage: Encodable { } } -extension ParsableMessage: Encodable { +@_spi(Testing) extension ParsableMessage: Encodable { enum CodingKeys: CodingKey { case name case kind diff --git a/Sources/SwiftDriver/Execution/ProcessProtocol.swift b/Sources/SwiftDriver/Execution/ProcessProtocol.swift index 65a1ffebc..3c9bef09b 100644 --- a/Sources/SwiftDriver/Execution/ProcessProtocol.swift +++ b/Sources/SwiftDriver/Execution/ProcessProtocol.swift @@ -18,7 +18,7 @@ public protocol ProcessProtocol { /// Clients that don't really launch a process can return /// a negative number to represent a "quasi-pid". /// - /// - SeeAlso: https://github.com/apple/swift/blob/master/docs/DriverParseableOutput.rst#quasi-pids + /// - SeeAlso: https://github.com/apple/swift/blob/main/docs/DriverParseableOutput.rst#quasi-pids var processID: Process.ProcessID { get } /// Wait for the process to finish execution. diff --git a/Sources/SwiftDriver/Execution/llbuild.swift b/Sources/SwiftDriver/Execution/llbuild.swift deleted file mode 100644 index 2aa6d01a5..000000000 --- a/Sources/SwiftDriver/Execution/llbuild.swift +++ /dev/null @@ -1,263 +0,0 @@ -//===--------------- llbuild.swift - Swift LLBuild Interaction ------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -// FIXME: This is directly copied from SwiftPM, consider moving this to llbuild. - -// We either export the llbuildSwift shared library or the llbuild framework. -#if canImport(llbuildSwift) -@_exported import llbuildSwift -@_exported import llbuild -#else -@_exported import llbuild -#endif - -import Foundation - -/// An llbuild value. -public protocol LLBuildValue: Codable { -} - -/// An llbuild key. -public protocol LLBuildKey: Codable { - /// The value that this key computes. - associatedtype BuildValue: LLBuildValue - - /// The rule that this key operates on. - associatedtype BuildRule: LLBuildRule -} - -public protocol LLBuildEngineDelegate { - func lookupRule(rule: String, key: Key) -> Rule -} - -public final class LLBuildEngine { - - enum Error: Swift.Error, CustomStringConvertible { - case failed(errors: [String]) - - var description: String { - switch self { - case .failed(let errors): - return errors.joined(separator: "\n") - } - } - } - - fileprivate final class Delegate: BuildEngineDelegate { - let delegate: LLBuildEngineDelegate - var errors: [String] = [] - - init(_ delegate: LLBuildEngineDelegate) { - self.delegate = delegate - } - - func lookupRule(_ key: Key) -> Rule { - let ruleKey = RuleKey(key) - return delegate.lookupRule( - rule: ruleKey.rule, key: Key(ruleKey.data)) - } - - func error(_ message: String) { - errors.append(message) - } - } - - private let engine: BuildEngine - private let delegate: Delegate - - public init(delegate: LLBuildEngineDelegate) { - self.delegate = Delegate(delegate) - engine = BuildEngine(delegate: self.delegate) - } - - deinit { - engine.close() - } - - public func build(key: T) throws -> T.BuildValue { - // Clear out any errors from the previous build. - delegate.errors.removeAll() - - let encodedKey = RuleKey( - rule: T.BuildRule.ruleName, data: key.toKey().data).toKey() - let value = engine.build(key: encodedKey) - - // Throw if the engine encountered any fatal error during the build. - if !delegate.errors.isEmpty || value.data.isEmpty { - throw Error.failed(errors: delegate.errors) - } - - return try T.BuildValue(value) - } - - public func attachDB(path: String, schemaVersion: Int = 2) throws { - try engine.attachDB(path: path, schemaVersion: schemaVersion) - } - - public func close() { - engine.close() - } -} - -// FIXME: Rename to something else. -public class LLTaskBuildEngine { - - let engine: TaskBuildEngine - let fileSystem: FileSystem - - init(_ engine: TaskBuildEngine, fileSystem: FileSystem) { - self.engine = engine - self.fileSystem = fileSystem - } - - public func taskNeedsInput(_ key: T, inputID: Int) { - let encodedKey = RuleKey( - rule: T.BuildRule.ruleName, data: key.toKey().data).toKey() - engine.taskNeedsInput(encodedKey, inputID: inputID) - } - - public func taskIsComplete(_ result: T) { - engine.taskIsComplete(result.toValue(), forceChange: false) - } -} - -/// An individual build rule. -open class LLBuildRule: Rule, Task { - - /// The name of the rule. - /// - /// This name will be available in the delegate's lookupRule(rule:key:). - open class var ruleName: String { - fatalError("subclass responsibility") - } - - let fileSystem: FileSystem - - public init(fileSystem: FileSystem) { - self.fileSystem = fileSystem - } - - public func createTask() -> Task { - return self - } - - public func start(_ engine: TaskBuildEngine) { - self.start(LLTaskBuildEngine(engine, fileSystem: fileSystem)) - } - - public func provideValue(_ engine: TaskBuildEngine, inputID: Int, value: Value) { - self.provideValue(LLTaskBuildEngine(engine, fileSystem: fileSystem), inputID: inputID, value: value) - } - - public func inputsAvailable(_ engine: TaskBuildEngine) { - self.inputsAvailable(LLTaskBuildEngine(engine, fileSystem: fileSystem)) - } - - // MARK:- - - open func isResultValid(_ priorValue: Value) -> Bool { - return true - } - - open func start(_ engine: LLTaskBuildEngine) { - } - - open func provideValue(_ engine: LLTaskBuildEngine, inputID: Int, value: Value) { - } - - open func inputsAvailable(_ engine: LLTaskBuildEngine) { - } -} - -// MARK:- Helpers - -private struct RuleKey: Codable { - - let rule: String - let data: [UInt8] - - init(rule: String, data: [UInt8]) { - self.rule = rule - self.data = data - } - - init(_ key: Key) { - self.init(key.data) - } - - init(_ data: [UInt8]) { - self = try! fromBytes(data) - } - - func toKey() -> Key { - return try! Key(toBytes(self)) - } -} - -public extension LLBuildKey { - init(_ key: Key) { - self.init(key.data) - } - - init(_ data: [UInt8]) { - do { - self = try fromBytes(data) - } catch { - let stringValue: String - if let str = String(bytes: data, encoding: .utf8) { - stringValue = str - } else { - stringValue = String(describing: data) - } - fatalError("Please file a bug at https://bugs.swift.org with this info -- LLBuildKey: ###\(error)### ----- ###\(stringValue)###") - } - } - - func toKey() -> Key { - return try! Key(toBytes(self)) - } -} - -public extension LLBuildValue { - init(_ value: Value) throws { - do { - self = try fromBytes(value.data) - } catch { - let stringValue: String - if let str = String(bytes: value.data, encoding: .utf8) { - stringValue = str - } else { - stringValue = String(describing: value.data) - } - fatalError("Please file a bug at https://bugs.swift.org with this info -- LLBuildValue: ###\(error)### ----- ###\(stringValue)###") - } - } - - func toValue() -> Value { - return try! Value(toBytes(self)) - } -} - -private func fromBytes(_ bytes: [UInt8]) throws -> T { - var bytes = bytes - let data = Data(bytes: &bytes, count: bytes.count) - return try JSONDecoder().decode(T.self, from: data) -} - -private func toBytes(_ value: T) throws -> [UInt8] { - let encoder = JSONEncoder() - if #available(macOS 10.13, iOS 11.0, watchOS 4.0, tvOS 11.0, *) { - encoder.outputFormatting = [.sortedKeys] - } - let encoded = try encoder.encode(value) - return [UInt8](encoded) -} diff --git a/Sources/SwiftDriver/Explicit Module Builds/ClangVersionedDependencyResolution.swift b/Sources/SwiftDriver/Explicit Module Builds/ClangVersionedDependencyResolution.swift index 76505e208..f190c9a7f 100644 --- a/Sources/SwiftDriver/Explicit Module Builds/ClangVersionedDependencyResolution.swift +++ b/Sources/SwiftDriver/Explicit Module Builds/ClangVersionedDependencyResolution.swift @@ -31,6 +31,8 @@ internal extension Driver { // to all Clang modules, and compute a set of distinct PCMArgs across all paths to a // given Clang module in the graph. let modulePCMArgsSetMap = try dependencyGraph.computePCMArgSetsForClangModules() + + // Set up the batch scan input let temporaryDirectory = try determineTempDirectory() let batchScanInputList = try modulePCMArgsSetMap.compactMap { (moduleId, pcmArgsSet) throws -> [BatchScanModuleInfo] in @@ -53,10 +55,13 @@ internal extension Driver { return moduleInfos }.reduce([], +) - // Batch scan all clang modules for each discovered unique set of PCMArgs, per module + // Batch scan all clang modules for each discovered new, unique set of PCMArgs, per module let moduleVersionedGraphMap: [ModuleDependencyId: [InterModuleDependencyGraph]] = try performBatchDependencyScan(moduleInfos: batchScanInputList) + + // Update the dependency graph to reflect the newly-discovered dependencies try dependencyGraph.resolveVersionedClangModules(using: moduleVersionedGraphMap) + try dependencyGraph.updateCapturedPCMArgClangDependencies(using: modulePCMArgsSetMap) } } @@ -103,26 +108,52 @@ private extension InterModuleDependencyGraph { pathPCMArtSet: Set<[String]>, pcmArgSetMap: inout [ModuleDependencyId : Set<[String]>]) throws { + guard let moduleInfo = modules[moduleId] else { + throw Driver.Error.missingModuleDependency(moduleId.moduleName) + } switch moduleId { case .swift: + guard case .swift(let swiftModuleDetails) = moduleInfo.details else { + throw Driver.Error.malformedModuleDependency(moduleId.moduleName, + "no Swift `details` object") + } // Add extraPCMArgs of the visited node to the current path set // and proceed to visit all direct dependencies - let modulePCMArgs = try swiftModulePCMArgs(of: moduleId) + let modulePCMArgs = swiftModuleDetails.extraPcmArgs var newPathPCMArgSet = pathPCMArtSet newPathPCMArgSet.insert(modulePCMArgs) - for dependencyId in try moduleInfo(of: moduleId).directDependencies! { + for dependencyId in moduleInfo.directDependencies! { try visit(dependencyId, pathPCMArtSet: newPathPCMArgSet, pcmArgSetMap: &pcmArgSetMap) } case .clang: + guard case .clang(let clangModuleDetails) = moduleInfo.details else { + throw Driver.Error.malformedModuleDependency(moduleId.moduleName, + "no Clang `details` object") + } + // The details of this module contain information on which sets of PCMArgs are already + // captured in the described dependencies of this module. Only re-scan at PCMArgs not + // already captured. + let alreadyCapturedPCMArgs = + clangModuleDetails.dependenciesCapturedPCMArgs ?? Set<[String]>() + let newPCMArgSet = pathPCMArtSet.filter { !alreadyCapturedPCMArgs.contains($0) } // Add current path's PCMArgs to the SetMap and stop traversal if pcmArgSetMap[moduleId] != nil { - pathPCMArtSet.forEach { pcmArgSetMap[moduleId]!.insert($0) } + newPCMArgSet.forEach { pcmArgSetMap[moduleId]!.insert($0) } } else { - pcmArgSetMap[moduleId] = pathPCMArtSet + pcmArgSetMap[moduleId] = newPCMArgSet } return + case .swiftPrebuiltExternal: + // We can rely on the fact that this pre-built module already has its + // versioned-PCM dependencies satisfied, so we do not need to add additional + // arguments. Proceed traversal to its dependencies. + for dependencyId in moduleInfo.directDependencies! { + try visit(dependencyId, + pathPCMArtSet: pathPCMArtSet, + pcmArgSetMap: &pcmArgSetMap) + } case .swiftPlaceholder: fatalError("Unresolved placeholder dependencies at planning stage: \(moduleId)") } @@ -133,4 +164,25 @@ private extension InterModuleDependencyGraph { pcmArgSetMap: &pcmArgSetMap) return pcmArgSetMap } + + /// Update the set of all PCMArgs against which a given clang module was re-scanned + mutating func updateCapturedPCMArgClangDependencies(using pcmArgSetMap: + [ModuleDependencyId : Set<[String]>] + ) throws { + for (moduleId, newPCMArgs) in pcmArgSetMap { + guard let moduleInfo = modules[moduleId] else { + throw Driver.Error.missingModuleDependency(moduleId.moduleName) + } + guard case .clang(var clangModuleDetails) = moduleInfo.details else { + throw Driver.Error.malformedModuleDependency(moduleId.moduleName, + "no Clang `details` object") + } + if clangModuleDetails.dependenciesCapturedPCMArgs == nil { + clangModuleDetails.dependenciesCapturedPCMArgs = Set<[String]>() + } + newPCMArgs.forEach { clangModuleDetails.dependenciesCapturedPCMArgs!.insert($0) } + modules[moduleId]!.details = .clang(clangModuleDetails) + } + } } + diff --git a/Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift b/Sources/SwiftDriver/Explicit Module Builds/ExplicitDependencyBuildPlanner.swift similarity index 79% rename from Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift rename to Sources/SwiftDriver/Explicit Module Builds/ExplicitDependencyBuildPlanner.swift index f46470c8a..c5f3266e0 100644 --- a/Sources/SwiftDriver/Explicit Module Builds/ExplicitModuleBuildHandler.swift +++ b/Sources/SwiftDriver/Explicit Module Builds/ExplicitDependencyBuildPlanner.swift @@ -1,4 +1,4 @@ -//===--------------- ExplicitModuleBuildHandler.swift ---------------------===// +//===--------------- ExplicitDependencyBuildPlanner.swift ---------------------===// // // This source file is part of the Swift.org open source project // @@ -13,14 +13,19 @@ import TSCBasic import TSCUtility import Foundation -/// A map from a module identifier to a pair consisting of a path to its .swiftmodule file and its module dependency graph. -public typealias ExternalDependencyArtifactMap = - [ModuleDependencyId: (AbsolutePath, InterModuleDependencyGraph)] +/// A map from a module identifier to a path to its .swiftmodule file. +public typealias ExternalTargetModulePathMap = [ModuleDependencyId: AbsolutePath] -/// In Explicit Module Build mode, this handler is responsible for generating and providing +// FIXME: ExternalBuildArtifacts is a temporary backwards-compatibility shim +// to help transition SwiftPM to the new API. +/// A tuple all external artifacts a build system may pass in as input to the explicit build of the current module +/// Consists of a map of externally-built targets, and a map of all previously discovered/scanned modules. +public typealias ExternalBuildArtifacts = (ExternalTargetModulePathMap, ModuleInfoMap) + +/// In Explicit Module Build mode, this planner is responsible for generating and providing /// build jobs for all module dependencies and providing compile command options /// that specify said explicit module dependencies. -@_spi(Testing) public struct ExplicitModuleBuildHandler { +@_spi(Testing) public struct ExplicitDependencyBuildPlanner { /// The module dependency graph. public var dependencyGraph: InterModuleDependencyGraph @@ -34,24 +39,10 @@ public typealias ExternalDependencyArtifactMap = /// The toolchain to be used for frontend job generation. private let toolchain: Toolchain - /// The file system which we should interact with. - /// FIXME: Our end goal is to not have any direct filesystem manipulation in here, but that's dependent on getting the - /// dependency scanner/dependency job generation moved into a Job. - private let fileSystem: FileSystem - - /// Path to the directory that will contain the temporary files. - /// e.g. Explicit Swift module artifact files - /// FIXME: Our end goal is to not have any direct filesystem manipulation in here, but that's dependent on getting the - /// dependency scanner/dependency job generation moved into a Job. - private let temporaryDirectory: AbsolutePath - public init(dependencyGraph: InterModuleDependencyGraph, - toolchain: Toolchain, - fileSystem: FileSystem) throws { + toolchain: Toolchain) throws { self.dependencyGraph = dependencyGraph self.toolchain = toolchain - self.fileSystem = fileSystem - self.temporaryDirectory = try determineTempDirectory() } /// Generate build jobs for all dependencies of the main module. @@ -185,6 +176,7 @@ public typealias ExternalDependencyArtifactMap = tool: .absolute(try toolchain.getToolPath(.swiftCompiler)), commandLine: commandLine, inputs: inputs, + primaryInputs: [], outputs: outputs ) } @@ -201,7 +193,7 @@ public typealias ExternalDependencyArtifactMap = // First, take the command line options provided in the dependency information let moduleDetails = try dependencyGraph.clangModuleDetails(of: moduleId) - moduleDetails.commandLine?.forEach { commandLine.appendFlags($0) } + moduleDetails.commandLine.forEach { commandLine.appendFlags($0) } // Add the `-target` option as inherited from the dependent Swift module's PCM args pcmArgs.forEach { commandLine.appendFlags($0) } @@ -212,7 +204,7 @@ public typealias ExternalDependencyArtifactMap = // Encode the target triple pcm args into the output `.pcm` filename let targetEncodedModulePath = - try ExplicitModuleBuildHandler.targetEncodedClangModuleFilePath(for: moduleInfo, + try ExplicitDependencyBuildPlanner.targetEncodedClangModuleFilePath(for: moduleInfo, pcmArgs: pcmArgs) outputs.append(TypedVirtualPath(file: targetEncodedModulePath, type: .pcm)) commandLine.appendFlags("-emit-pcm", "-module-name", moduleId.moduleName, @@ -230,6 +222,7 @@ public typealias ExternalDependencyArtifactMap = tool: .absolute(try toolchain.getToolPath(.swiftCompiler)), commandLine: commandLine, inputs: inputs, + primaryInputs: [], outputs: outputs ) } @@ -237,14 +230,11 @@ public typealias ExternalDependencyArtifactMap = /// Store the output file artifacts for a given module in a JSON file, return the file's path. private func serializeModuleDependencies(for moduleId: ModuleDependencyId, dependencyArtifacts: [SwiftModuleArtifactInfo] - ) throws -> AbsolutePath { - let dependencyFilePath = - temporaryDirectory.appending(component: "\(moduleId.moduleName)-dependencies.json") + ) throws -> VirtualPath { let encoder = JSONEncoder() encoder.outputFormatting = [.prettyPrinted] let contents = try encoder.encode(dependencyArtifacts) - try fileSystem.writeFileContents(dependencyFilePath, bytes: ByteString(contents)) - return dependencyFilePath + return .temporaryWithKnownContents(.init("\(moduleId.moduleName)-dependencies.json"), contents) } /// For the specified module, update the given command line flags and inputs @@ -275,7 +265,7 @@ public typealias ExternalDependencyArtifactMap = try serializeModuleDependencies(for: moduleId, dependencyArtifacts: swiftDependencyArtifacts) commandLine.appendFlag("-explicit-swift-module-map-file") commandLine.appendPath(dependencyFile) - inputs.append(TypedVirtualPath(file: try VirtualPath(path: dependencyFile.pathString), + inputs.append(TypedVirtualPath(file: dependencyFile, type: .jsonSwiftArtifacts)) // Each individual module binary is still an "input" to ensure the build system gets the // order correctly. @@ -325,6 +315,12 @@ public typealias ExternalDependencyArtifactMap = addedDependenciesSet: &addedDependenciesSet, clangDependencyArtifacts: &clangDependencyArtifacts, swiftDependencyArtifacts: &swiftDependencyArtifacts) + case .swiftPrebuiltExternal: + try addSwiftPrebuiltDependency(moduleId: moduleId, dependencyId: dependencyId, + pcmArgs: pcmArgs, + addedDependenciesSet: &addedDependenciesSet, + clangDependencyArtifacts: &clangDependencyArtifacts, + swiftDependencyArtifacts: &swiftDependencyArtifacts) case .swiftPlaceholder: fatalError("Unresolved placeholder dependencies at planning stage: \(dependencyId) of \(moduleId)") } @@ -344,27 +340,17 @@ public typealias ExternalDependencyArtifactMap = ) throws { // Add it as an explicit dependency let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId) - let swiftModulePath: TypedVirtualPath let isFramework: Bool - if case .swift(let details) = dependencyInfo.details, - let compiledModulePath = details.explicitCompiledModulePath { - // If an already-compiled module is available, use it. - swiftModulePath = .init(file: try VirtualPath(path: compiledModulePath), - type: .swiftModule) - // Since this module has already been built, it is no longer relevant - // whether it is a framework or not. - isFramework = false - } else { - // Generate a build job for the dependency module, if not already generated - if swiftModuleBuildCache[dependencyId] == nil { - try genSwiftModuleBuildJob(moduleId: dependencyId) - assert(swiftModuleBuildCache[dependencyId] != nil) - } - swiftModulePath = .init(file: try VirtualPath(path: dependencyInfo.modulePath), - type: .swiftModule) - isFramework = try dependencyGraph.swiftModuleDetails(of: dependencyId).isFramework + + // Generate a build job for the dependency module, if not already generated + if swiftModuleBuildCache[dependencyId] == nil { + try genSwiftModuleBuildJob(moduleId: dependencyId) + assert(swiftModuleBuildCache[dependencyId] != nil) } + swiftModulePath = .init(file: try VirtualPath(path: dependencyInfo.modulePath), + type: .swiftModule) + isFramework = try dependencyGraph.swiftModuleDetails(of: dependencyId).isFramework // Collect the required information about this module // TODO: add .swiftdoc and .swiftsourceinfo for this module. @@ -401,7 +387,7 @@ public typealias ExternalDependencyArtifactMap = let dependencyInfo = try dependencyGraph.moduleInfo(of: dependencyId) let dependencyClangModuleDetails = try dependencyGraph.clangModuleDetails(of: dependencyId) let clangModulePath = - try ExplicitModuleBuildHandler.targetEncodedClangModuleFilePath(for: dependencyInfo, + try ExplicitDependencyBuildPlanner.targetEncodedClangModuleFilePath(for: dependencyInfo, pcmArgs: pcmArgs) // Collect the requried information about this module @@ -415,10 +401,39 @@ public typealias ExternalDependencyArtifactMap = clangDependencyArtifacts: &clangDependencyArtifacts, swiftDependencyArtifacts: &swiftDependencyArtifacts) } + + /// Add a specific external target's pre-built Swift module dependency as an input and a corresponding command + /// line flag. + mutating private func addSwiftPrebuiltDependency(moduleId: ModuleDependencyId, + dependencyId: ModuleDependencyId, + pcmArgs: [String], + addedDependenciesSet: inout Set, + clangDependencyArtifacts: inout [ClangModuleArtifactInfo], + swiftDependencyArtifacts: inout [SwiftModuleArtifactInfo] + ) throws { + // Add it as an explicit dependency + let compiledModulePath = try dependencyGraph + .swiftPrebuiltDetails(of: dependencyId) + .compiledModulePath + let swiftModulePath: TypedVirtualPath = .init(file: try VirtualPath(path: compiledModulePath), + type: .swiftModule) + + // Collect the required information about this module + // TODO: add .swiftdoc and .swiftsourceinfo for this module. + swiftDependencyArtifacts.append( + SwiftModuleArtifactInfo(name: dependencyId.moduleName, + modulePath: swiftModulePath.file.description)) + + // Process all transitive dependencies as direct + try addModuleDependencies(moduleId: dependencyId, pcmArgs: pcmArgs, + addedDependenciesSet: &addedDependenciesSet, + clangDependencyArtifacts: &clangDependencyArtifacts, + swiftDependencyArtifacts: &swiftDependencyArtifacts) + } } /// Utility methods for encoding PCM's target triple into its name. -extension ExplicitModuleBuildHandler { +extension ExplicitDependencyBuildPlanner { /// Compute a full path to the resulting .pcm file for a given Clang module, with the /// target triple encoded in the name. public static func targetEncodedClangModuleFilePath(for moduleInfo: ModuleInfo, @@ -437,13 +452,22 @@ extension ExplicitModuleBuildHandler { /// is to be constructed with. public static func targetEncodedClangModuleName(for moduleName: String, pcmArgs: [String]) throws -> String { - var hasher = Hasher() - pcmArgs.forEach { hasher.combine($0) } - return moduleName + String(hasher.finalize()) + let hashInput = pcmArgs.sorted().joined() + let hashedArguments: String + #if os(macOS) + if #available(macOS 10.15, iOS 13, *) { + hashedArguments = CryptoKitSHA256().hash(hashInput).hexadecimalRepresentation + } else { + hashedArguments = SHA256().hash(hashInput).hexadecimalRepresentation + } + #else + hashedArguments = SHA256().hash(hashInput).hexadecimalRepresentation + #endif + return moduleName + hashedArguments } } -/// Encapsulates some of the common queries of the ExplicitModuleBuildeHandler with error-checking +/// Encapsulates some of the common queries of the ExplicitDependencyBuildPlanner with error-checking /// on the dependency graph's structure. internal extension InterModuleDependencyGraph { func moduleInfo(of moduleId: ModuleDependencyId) throws -> ModuleInfo { @@ -455,31 +479,40 @@ internal extension InterModuleDependencyGraph { func swiftModuleDetails(of moduleId: ModuleDependencyId) throws -> SwiftModuleDetails { guard case .swift(let swiftModuleDetails) = try moduleInfo(of: moduleId).details else { - throw Driver.Error.malformedModuleDependency(mainModuleName, "no Swift `details` object") + throw Driver.Error.malformedModuleDependency(moduleId.moduleName, "no Swift `details` object") } return swiftModuleDetails } + func swiftPrebuiltDetails(of moduleId: ModuleDependencyId) + throws -> SwiftPrebuiltExternalModuleDetails { + guard case .swiftPrebuiltExternal(let prebuiltModuleDetails) = + try moduleInfo(of: moduleId).details else { + throw Driver.Error.malformedModuleDependency(moduleId.moduleName, + "no SwiftPrebuiltExternal `details` object") + } + return prebuiltModuleDetails + } + func clangModuleDetails(of moduleId: ModuleDependencyId) throws -> ClangModuleDetails { guard case .clang(let clangModuleDetails) = try moduleInfo(of: moduleId).details else { - throw Driver.Error.malformedModuleDependency(mainModuleName, "no Clang `details` object") + throw Driver.Error.malformedModuleDependency(moduleId.moduleName, "no Clang `details` object") } return clangModuleDetails } func swiftModulePCMArgs(of moduleId: ModuleDependencyId) throws -> [String] { let moduleDetails = try swiftModuleDetails(of: moduleId) - guard let pcmArgs = moduleDetails.extraPcmArgs else { - throw Driver.Error.missingPCMArguments(mainModuleName) - } - return pcmArgs + return moduleDetails.extraPcmArgs } } -// To keep the ExplicitModuleBuildHandler an implementation detail, provide an API -// to access the dependency graph -extension Driver { - public var interModuleDependencyGraph: InterModuleDependencyGraph? { - return explicitModuleBuildHandler?.dependencyGraph +// InterModuleDependencyGraph printing, useful for debugging +internal extension InterModuleDependencyGraph { + func prettyPrintString() throws -> String { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted] + let contents = try encoder.encode(self) + return String(data: contents, encoding: .utf8)! } } diff --git a/Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/CommonDependencyOperations.swift b/Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/CommonDependencyOperations.swift new file mode 100644 index 000000000..e656cd2d9 --- /dev/null +++ b/Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/CommonDependencyOperations.swift @@ -0,0 +1,183 @@ +//===----------------- CommonDependencyOperations.swift -------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_spi(Testing) public extension InterModuleDependencyOracle { + /// An API to allow clients to accumulate InterModuleDependencyGraphs across mutiple main modules/targets + /// into a single collection of discovered modules. + func mergeModules(from dependencyGraph: InterModuleDependencyGraph) throws { + try self.lock.withLock { + for (moduleId, moduleInfo) in dependencyGraph.modules { + try InterModuleDependencyGraph.mergeModule(moduleId, moduleInfo, into: &modules) + } + } + } + + // This is a backwards-compatibility shim to handle existing ModuleInfoMap-based API + // used by SwiftPM + func mergeModules(from moduleInfoMap: ModuleInfoMap) throws { + try self.lock.withLock { + for (moduleId, moduleInfo) in moduleInfoMap { + try InterModuleDependencyGraph.mergeModule(moduleId, moduleInfo, into: &modules) + } + } + } +} + +public extension InterModuleDependencyGraph { + // This is a shim for backwards-compatibility with existing API used by SwiftPM. + // TODO: After SwiftPM switches to using the oracle, this should be deleted. + static func mergeModules( + from dependencyGraph: InterModuleDependencyGraph, + into discoveredModules: inout ModuleInfoMap + ) throws { + for (moduleId, moduleInfo) in dependencyGraph.modules { + try mergeModule(moduleId, moduleInfo, into: &discoveredModules) + } + } +} + + +@_spi(Testing) public extension InterModuleDependencyGraph { + /// Merge a module with a given ID and Info into a ModuleInfoMap + static func mergeModule(_ moduleId: ModuleDependencyId, + _ moduleInfo: ModuleInfo, + into moduleInfoMap: inout ModuleInfoMap) throws { + switch moduleId { + case .swift: + let prebuiltExternalModuleEquivalentId = + ModuleDependencyId.swiftPrebuiltExternal(moduleId.moduleName) + let placeholderEquivalentId = + ModuleDependencyId.swiftPlaceholder(moduleId.moduleName) + if moduleInfoMap[prebuiltExternalModuleEquivalentId] != nil || + moduleInfoMap[moduleId] != nil { + // If the set of discovered modules contains a .swiftPrebuiltExternal or .swift module + // with the same name, do not replace it. + break + } else if moduleInfoMap[placeholderEquivalentId] != nil { + // Replace the placeholder module with a full .swift ModuleInfo + // and fixup other modules' dependencies + replaceModule(originalId: placeholderEquivalentId, replacementId: moduleId, + replacementInfo: moduleInfo, in: &moduleInfoMap) + } else { + // Insert the new module + moduleInfoMap[moduleId] = moduleInfo + } + + case .swiftPrebuiltExternal: + // If the set of discovered modules contains a .swift module with the same name, + // replace it with the prebuilt version and fixup other modules' dependencies + let swiftModuleEquivalentId = ModuleDependencyId.swift(moduleId.moduleName) + let swiftPlaceholderEquivalentId = ModuleDependencyId.swiftPlaceholder(moduleId.moduleName) + if moduleInfoMap[swiftModuleEquivalentId] != nil { + // If the ModuleInfoMap contains an equivalent .swift module, replace it with the prebuilt + // version and update all other modules' dependencies + replaceModule(originalId: swiftModuleEquivalentId, replacementId: moduleId, + replacementInfo: moduleInfo, in: &moduleInfoMap) + } else if moduleInfoMap[swiftPlaceholderEquivalentId] != nil { + // If the moduleInfoMap contains an equivalent .swiftPlaceholder module, replace it with + // the prebuilt version and update all other modules' dependencies + replaceModule(originalId: swiftPlaceholderEquivalentId, replacementId: moduleId, + replacementInfo: moduleInfo, in: &moduleInfoMap) + } else { + // Insert the new module + moduleInfoMap[moduleId] = moduleInfo + } + + case .clang: + guard let existingModuleInfo = moduleInfoMap[moduleId] else { + moduleInfoMap[moduleId] = moduleInfo + break + } + // If this module *has* been seen before, merge the module infos to capture + // the super-set of so-far discovered dependencies of this module at various + // PCMArg scanning actions. + let combinedDependenciesInfo = mergeClangModuleInfoDependencies(moduleInfo, + existingModuleInfo) + replaceModule(originalId: moduleId, replacementId: moduleId, + replacementInfo: combinedDependenciesInfo, in: &moduleInfoMap) + case .swiftPlaceholder: + fatalError("Unresolved placeholder dependency at graph merge operation: \(moduleId)") + } + } + + /// Replace an existing module in the moduleInfoMap + static func replaceModule(originalId: ModuleDependencyId, replacementId: ModuleDependencyId, + replacementInfo: ModuleInfo, + in moduleInfoMap: inout ModuleInfoMap) { + precondition(moduleInfoMap[originalId] != nil) + moduleInfoMap.removeValue(forKey: originalId) + moduleInfoMap[replacementId] = replacementInfo + if originalId != replacementId { + updateDependencies(from: originalId, to: replacementId, in: &moduleInfoMap) + } + } + + /// Replace all references to the original module in other modules' dependencies with the new module. + static func updateDependencies(from originalId: ModuleDependencyId, + to replacementId: ModuleDependencyId, + in moduleInfoMap: inout ModuleInfoMap) { + for moduleId in moduleInfoMap.keys { + var moduleInfo = moduleInfoMap[moduleId]! + // Skip over placeholders, they do not have dependencies + if case .swiftPlaceholder(_) = moduleId { + continue + } + if let originalModuleIndex = moduleInfo.directDependencies?.firstIndex(of: originalId) { + moduleInfo.directDependencies![originalModuleIndex] = replacementId; + moduleInfoMap[moduleId] = moduleInfo + } + } + } + + /// Given two moduleInfos of clang modules, merge them by combining their directDependencies and + /// dependenciesCapturedPCMArgs and sourceFiles fields. These fields may differ across the same module + /// scanned at different PCMArgs (e.g. -target option). + static func mergeClangModuleInfoDependencies(_ firstInfo: ModuleInfo, _ secondInfo:ModuleInfo + ) -> ModuleInfo { + guard case .clang(let firstDetails) = firstInfo.details, + case .clang(let secondDetails) = secondInfo.details + else { + fatalError("mergeClangModules expected two valid ClangModuleDetails objects.") + } + + // As far as their dependencies go, these module infos are identical + if firstInfo.directDependencies == secondInfo.directDependencies, + firstDetails.dependenciesCapturedPCMArgs == secondDetails.dependenciesCapturedPCMArgs, + firstInfo.sourceFiles == secondInfo.sourceFiles { + return firstInfo + } + + // Create a new moduleInfo that represents this module with combined dependency information + let firstModuleSources = firstInfo.sourceFiles ?? [] + let secondModuleSources = secondInfo.sourceFiles ?? [] + let combinedSourceFiles = Array(Set(firstModuleSources + secondModuleSources)) + + let firstModuleDependencies = firstInfo.directDependencies ?? [] + let secondModuleDependencies = secondInfo.directDependencies ?? [] + let combinedDependencies = Array(Set(firstModuleDependencies + secondModuleDependencies)) + + let firstModuleCapturedPCMArgs = firstDetails.dependenciesCapturedPCMArgs ?? Set<[String]>() + let secondModuleCapturedPCMArgs = secondDetails.dependenciesCapturedPCMArgs ?? Set<[String]>() + let combinedCapturedPCMArgs = firstModuleCapturedPCMArgs.union(secondModuleCapturedPCMArgs) + + let combinedModuleDetails = + ClangModuleDetails(moduleMapPath: firstDetails.moduleMapPath, + dependenciesCapturedPCMArgs: combinedCapturedPCMArgs, + contextHash: firstDetails.contextHash, + commandLine: firstDetails.commandLine) + + return ModuleInfo(modulePath: firstInfo.modulePath, + sourceFiles: combinedSourceFiles, + directDependencies: combinedDependencies, + details: .clang(combinedModuleDetails)) + } +} diff --git a/Sources/SwiftDriver/Explicit Module Builds/InterModuleDependencyGraph.swift b/Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyGraph.swift similarity index 58% rename from Sources/SwiftDriver/Explicit Module Builds/InterModuleDependencyGraph.swift rename to Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyGraph.swift index b0203e5c5..ca075b4b3 100644 --- a/Sources/SwiftDriver/Explicit Module Builds/InterModuleDependencyGraph.swift +++ b/Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyGraph.swift @@ -10,17 +10,20 @@ // //===----------------------------------------------------------------------===// import Foundation - +/// A map from a module identifier to its info +public typealias ModuleInfoMap = [ModuleDependencyId: ModuleInfo] public enum ModuleDependencyId: Hashable { case swift(String) case swiftPlaceholder(String) + case swiftPrebuiltExternal(String) case clang(String) public var moduleName: String { switch self { case .swift(let name): return name case .swiftPlaceholder(let name): return name + case .swiftPrebuiltExternal(let name): return name case .clang(let name): return name } } @@ -30,6 +33,7 @@ extension ModuleDependencyId: Codable { enum CodingKeys: CodingKey { case swift case swiftPlaceholder + case swiftPrebuiltExternal case clang } @@ -43,8 +47,13 @@ extension ModuleDependencyId: Codable { let moduleName = try container.decode(String.self, forKey: .swiftPlaceholder) self = .swiftPlaceholder(moduleName) } catch { - let moduleName = try container.decode(String.self, forKey: .clang) - self = .clang(moduleName) + do { + let moduleName = try container.decode(String.self, forKey: .swiftPrebuiltExternal) + self = .swiftPrebuiltExternal(moduleName) + } catch { + let moduleName = try container.decode(String.self, forKey: .clang) + self = .clang(moduleName) + } } } } @@ -55,7 +64,9 @@ extension ModuleDependencyId: Codable { case .swift(let moduleName): try container.encode(moduleName, forKey: .swift) case .swiftPlaceholder(let moduleName): - try container.encode(moduleName, forKey: .swift) + try container.encode(moduleName, forKey: .swiftPlaceholder) + case .swiftPrebuiltExternal(let moduleName): + try container.encode(moduleName, forKey: .swiftPrebuiltExternal) case .clang(let moduleName): try container.encode(moduleName, forKey: .clang) } @@ -72,37 +83,30 @@ public struct BridgingHeader: Codable { /// Details specific to Swift modules. public struct SwiftModuleDetails: Codable { /// The module interface from which this module was built, if any. - @_spi(Testing) public var moduleInterfacePath: String? + public var moduleInterfacePath: String? /// The paths of potentially ready-to-use compiled modules for the interface. - @_spi(Testing) public var compiledModuleCandidates: [String]? - - /// The path to the already-compiled module that must be used instead of - /// generating a job to build this module. In standard compilation, the dependency scanner - /// may discover compiled module candidates to be used instead of re-compiling from interface. - /// In contrast, this explicitCompiledModulePath is only to be used for precompiled modules - /// external dependencies in Explicit Module Build mode - @_spi(Testing) public var explicitCompiledModulePath: String? + public var compiledModuleCandidates: [String]? /// The bridging header, if any. - var bridgingHeaderPath: String? + public var bridgingHeaderPath: String? /// The source files referenced by the bridging header. - var bridgingSourceFiles: [String]? = [] + public var bridgingSourceFiles: [String]? = [] /// Options to the compile command - var commandLine: [String]? = [] + public var commandLine: [String]? = [] /// To build a PCM to be used by this Swift module, we need to append these /// arguments to the generic PCM build arguments reported from the dependency /// graph. - @_spi(Testing) public var extraPcmArgs: [String]? + public var extraPcmArgs: [String] /// A flag to indicate whether or not this module is a framework. - @_spi(Testing) public var isFramework: Bool + public var isFramework: Bool } -/// Details specific to Swift external modules. +/// Details specific to Swift placeholder dependencies. public struct SwiftPlaceholderModuleDetails: Codable { /// The path to the .swiftModuleDoc file. var moduleDocPath: String? @@ -111,16 +115,51 @@ public struct SwiftPlaceholderModuleDetails: Codable { var moduleSourceInfoPath: String? } +/// Details specific to Swift externally-pre-built modules. +public struct SwiftPrebuiltExternalModuleDetails: Codable { + /// The path to the already-compiled module that must be used instead of + /// generating a job to build this module. + public var compiledModulePath: String + + /// The path to the .swiftModuleDoc file. + public var moduleDocPath: String? + + /// The path to the .swiftSourceInfo file. + public var moduleSourceInfoPath: String? + + public init(compiledModulePath: String, + moduleDocPath: String? = nil, + moduleSourceInfoPath: String? = nil) { + self.compiledModulePath = compiledModulePath + self.moduleDocPath = moduleDocPath + self.moduleSourceInfoPath = moduleSourceInfoPath + } +} + /// Details specific to Clang modules. public struct ClangModuleDetails: Codable { /// The path to the module map used to build this module. - @_spi(Testing) public var moduleMapPath: String + public var moduleMapPath: String + + /// Set of PCM Arguments of depending modules which + /// are covered by the directDependencies info of this module + public var dependenciesCapturedPCMArgs: Set<[String]>? /// clang-generated context hash - var contextHash: String? + public var contextHash: String /// Options to the compile command - var commandLine: [String]? = [] + public var commandLine: [String] = [] + + public init(moduleMapPath: String, + dependenciesCapturedPCMArgs: Set<[String]>?, + contextHash: String, + commandLine: [String]) { + self.moduleMapPath = moduleMapPath + self.dependenciesCapturedPCMArgs = dependenciesCapturedPCMArgs + self.contextHash = contextHash + self.commandLine = commandLine + } } public struct ModuleInfo: Codable { @@ -128,10 +167,10 @@ public struct ModuleInfo: Codable { public var modulePath: String /// The source files used to build this module. - public var sourceFiles: [String]? = [] + public var sourceFiles: [String]? /// The set of direct module dependencies of this module. - public var directDependencies: [ModuleDependencyId]? = [] + public var directDependencies: [ModuleDependencyId]? /// Specific details of a particular kind of module. public var details: Details @@ -142,19 +181,33 @@ public struct ModuleInfo: Codable { /// a bridging header. case swift(SwiftModuleDetails) - /// Swift external modules carry additional details that specify their + /// Swift placeholder modules carry additional details that specify their /// module doc path and source info paths. case swiftPlaceholder(SwiftPlaceholderModuleDetails) + /// Swift externally-prebuilt modules must communicate the path to pre-built binary artifacts + case swiftPrebuiltExternal(SwiftPrebuiltExternalModuleDetails) + /// Clang modules are built from a module map file. case clang(ClangModuleDetails) } + + public init(modulePath: String, + sourceFiles: [String]?, + directDependencies: [ModuleDependencyId]?, + details: Details) { + self.modulePath = modulePath + self.sourceFiles = sourceFiles + self.directDependencies = directDependencies + self.details = details + } } extension ModuleInfo.Details: Codable { enum CodingKeys: CodingKey { case swift case swiftPlaceholder + case swiftPrebuiltExternal case clang } @@ -165,11 +218,18 @@ extension ModuleInfo.Details: Codable { self = .swift(details) } catch { do { - let details = try container.decode(SwiftPlaceholderModuleDetails.self, forKey: .swiftPlaceholder) + let details = try container.decode(SwiftPlaceholderModuleDetails.self, + forKey: .swiftPlaceholder) self = .swiftPlaceholder(details) } catch { - let details = try container.decode(ClangModuleDetails.self, forKey: .clang) - self = .clang(details) + do { + let details = try container.decode(SwiftPrebuiltExternalModuleDetails.self, + forKey: .swiftPrebuiltExternal) + self = .swiftPrebuiltExternal(details) + } catch { + let details = try container.decode(ClangModuleDetails.self, forKey: .clang) + self = .clang(details) + } } } } @@ -181,6 +241,8 @@ extension ModuleInfo.Details: Codable { try container.encode(details, forKey: .swift) case .swiftPlaceholder(let details): try container.encode(details, forKey: .swiftPlaceholder) + case .swiftPrebuiltExternal(let details): + try container.encode(details, forKey: .swiftPrebuiltExternal) case .clang(let details): try container.encode(details, forKey: .clang) } @@ -194,7 +256,7 @@ public struct InterModuleDependencyGraph: Codable { public var mainModuleName: String /// The complete set of modules discovered - public var modules: [ModuleDependencyId: ModuleInfo] = [:] + public var modules: ModuleInfoMap = [:] /// Information about the main module. public var mainModule: ModuleInfo { modules[.swift(mainModuleName)]! } diff --git a/Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyOracle.swift new file mode 100644 index 000000000..52275ac01 --- /dev/null +++ b/Sources/SwiftDriver/Explicit Module Builds/Inter Module Dependencies/InterModuleDependencyOracle.swift @@ -0,0 +1,84 @@ +//===--------------- InterModuleDependencyOracle.swift --------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import TSCBasic + +// An inter-module dependency oracle, responsible for responding to queries about +// dependencies of a given module, caching already-discovered dependencies along the way. +// +// The oracle is currently implemented as a simple store of ModuleInfo nodes. +// It is the responsibility of the Driver to populate and update +// the store. It does so by invoking individual -scan-dependencies jobs and +// accumulating resulting dependency graphs into the oracle's store. +// +// The design of the oracle's public API is meant to abstract that away, +// allowing us to replace the underlying implementation in the future, with +// a persistent-across-targets dependency scanning library. +// +/// An abstraction of a cache and query-engine of inter-module dependencies +public class InterModuleDependencyOracle { + /// Query the ModuleInfo of a module with a given ID + @_spi(Testing) public func getModuleInfo(of moduleId: ModuleDependencyId) -> ModuleInfo? { + self.lock.withLock { + return modules[moduleId] + } + } + + /// Query the direct dependencies of a module with a given ID + @_spi(Testing) public func getDependencies(of moduleId: ModuleDependencyId) + -> [ModuleDependencyId]? { + return getModuleInfo(of: moduleId)?.directDependencies + } + + // TODO: This will require a SwiftDriver entry-point for scanning a module given + // a command invocation and a set of source-files. As-is, the driver itself is responsible + // for executing individual module dependency-scanning actions and updating oracle state. + // (Implemented with InterModuleDependencyOracle::mergeModules extension) + // + // func getFullDependencies(inputs: [TypedVirtualPath], + // commandLine: [Job.ArgTemplate]) -> InterModuleDependencyGraph {} + // + + internal let lock = Lock() + + /// The complete set of modules discovered so far, spanning potentially multiple targets + internal var modules: ModuleInfoMap = [:] + + /// Allow external clients to instantiate the oracle + public init() {} +} + +// This is a shim for backwards-compatibility with existing API used by SwiftPM. +// TODO: After SwiftPM switches to using the oracle, this should be deleted. +extension Driver { + public var interModuleDependencyGraph: InterModuleDependencyGraph? { + let mainModuleId : ModuleDependencyId = .swift(moduleOutputInfo.name) + var mainModuleDependencyGraph = + InterModuleDependencyGraph(mainModuleName: moduleOutputInfo.name) + + addModule(moduleId: mainModuleId, + moduleInfo: interModuleDependencyOracle.getModuleInfo(of: mainModuleId)!, + into: &mainModuleDependencyGraph) + return mainModuleDependencyGraph + } + + private func addModule(moduleId: ModuleDependencyId, + moduleInfo: ModuleInfo, + into dependencyGraph: inout InterModuleDependencyGraph) { + dependencyGraph.modules[moduleId] = moduleInfo + moduleInfo.directDependencies?.forEach { dependencyId in + addModule(moduleId: dependencyId, + moduleInfo: interModuleDependencyOracle.getModuleInfo(of: dependencyId)!, + into: &dependencyGraph) + } + } +} diff --git a/Sources/SwiftDriver/Explicit Module Builds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/Explicit Module Builds/ModuleDependencyScanning.swift index 8429b1437..3d566d0e4 100644 --- a/Sources/SwiftDriver/Explicit Module Builds/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/Explicit Module Builds/ModuleDependencyScanning.swift @@ -29,11 +29,10 @@ extension Driver { moduleDependencyGraphUse: .dependencyScan) // FIXME: MSVC runtime flags - // Pass in external dependencies to be treated as placeholder dependencies by the scanner - if let externalDependencyArtifactMap = externalDependencyArtifactMap { + // Pass in external target dependencies to be treated as placeholder dependencies by the scanner + if let externalBuildArtifacts = externalBuildArtifacts { let dependencyPlaceholderMapFile = - try serializeExternalDependencyArtifacts(externalDependencyArtifactMap: - externalDependencyArtifactMap) + try serializeExternalDependencyArtifacts(externalBuildArtifacts: externalBuildArtifacts) commandLine.appendFlag("-placeholder-dependency-module-map-file") commandLine.appendPath(dependencyPlaceholderMapFile) } @@ -48,28 +47,37 @@ extension Driver { commandLine: commandLine, displayInputs: inputs, inputs: inputs, + primaryInputs: [], outputs: [TypedVirtualPath(file: .standardOutput, type: .jsonDependencies)], supportsResponseFiles: true) } /// Serialize a map of placeholder (external) dependencies for the dependency scanner. - func serializeExternalDependencyArtifacts(externalDependencyArtifactMap: ExternalDependencyArtifactMap) - throws -> AbsolutePath { - let temporaryDirectory = try determineTempDirectory() - let placeholderMapFilePath = - temporaryDirectory.appending(component: "\(moduleOutputInfo.name)-placeholder-modules.json") - + func serializeExternalDependencyArtifacts(externalBuildArtifacts: ExternalBuildArtifacts) + throws -> VirtualPath { + let (externalTargetModulePathMap, externalModuleInfoMap) = externalBuildArtifacts var placeholderArtifacts: [SwiftModuleArtifactInfo] = [] - for (moduleId, dependencyInfo) in externalDependencyArtifactMap { + + // Explicit external targets + for (moduleId, binaryModulePath) in externalTargetModulePathMap { placeholderArtifacts.append( SwiftModuleArtifactInfo(name: moduleId.moduleName, - modulePath: dependencyInfo.0.description)) + modulePath: binaryModulePath.description)) + } + + // All other already-scanned Swift modules + for (moduleId, moduleInfo) in externalModuleInfoMap + where !externalTargetModulePathMap.keys.contains(moduleId) { + guard case .swift(_) = moduleId else { continue } + placeholderArtifacts.append( + SwiftModuleArtifactInfo(name: moduleId.moduleName, + modulePath: moduleInfo.modulePath)) } let encoder = JSONEncoder() encoder.outputFormatting = [.prettyPrinted] let contents = try encoder.encode(placeholderArtifacts) - try fileSystem.writeFileContents(placeholderMapFilePath, bytes: ByteString(contents)) - return placeholderMapFilePath + return .temporaryWithKnownContents(.init("\(moduleOutputInfo.name)-placeholder-modules.json"), + contents) } mutating func performBatchDependencyScan(moduleInfos: [BatchScanModuleInfo]) @@ -83,7 +91,7 @@ extension Driver { let success = batchScanResult.exitStatus == .terminated(code: EXIT_SUCCESS) guard success else { throw JobExecutionError.jobFailedWithNonzeroExitCode( - SwiftDriverExecutor.computeReturnCode(exitStatus: batchScanResult.exitStatus), + type(of: executor).computeReturnCode(exitStatus: batchScanResult.exitStatus), try batchScanResult.utf8stderrOutput()) } @@ -156,21 +164,18 @@ extension Driver { commandLine: commandLine, displayInputs: inputs, inputs: inputs, + primaryInputs: [], outputs: outputs, supportsResponseFiles: true) } /// Serialize a collection of modules into an input format expected by the batch module dependency scanner. func serializeBatchScanningModuleArtifacts(moduleInfos: [BatchScanModuleInfo]) - throws -> AbsolutePath { - let temporaryDirectory = try determineTempDirectory() - let batchScanInputFilePath = - temporaryDirectory.appending(component: "\(moduleOutputInfo.name)-batch-module-scan.json") - + throws -> VirtualPath { let encoder = JSONEncoder() encoder.outputFormatting = [.prettyPrinted] let contents = try encoder.encode(moduleInfos) - try fileSystem.writeFileContents(batchScanInputFilePath, bytes: ByteString(contents)) - return batchScanInputFilePath + return .temporaryWithKnownContents(.init("\(moduleOutputInfo.name)-batch-module-scan.json"), + contents) } } diff --git a/Sources/SwiftDriver/Explicit Module Builds/PlaceholderDependencyResolution.swift b/Sources/SwiftDriver/Explicit Module Builds/PlaceholderDependencyResolution.swift index bc75b1910..b1aab03da 100644 --- a/Sources/SwiftDriver/Explicit Module Builds/PlaceholderDependencyResolution.swift +++ b/Sources/SwiftDriver/Explicit Module Builds/PlaceholderDependencyResolution.swift @@ -41,132 +41,128 @@ import Foundation // which the driver will then resolve using B's full dependency graph provided by the client. /// Resolve all placeholder dependencies using external dependency information provided by the client - mutating func resolvePlaceholderDependencies(using externalDependencyMap: ExternalDependencyArtifactMap) + mutating func resolvePlaceholderDependencies(for externalBuildArtifacts: ExternalBuildArtifacts, + using dependencyOracle: InterModuleDependencyOracle) throws { - let placeholderModules = modules.keys.filter { + let externalTargetModulePathMap = externalBuildArtifacts.0 + let placeholderFilter : (ModuleDependencyId) -> Bool = { if case .swiftPlaceholder(_) = $0 { return true } return false } + var placeholderModules = modules.keys.filter(placeholderFilter) - // Resolve all placeholder modules - for moduleId in placeholderModules { - guard let (placeholderModulePath, placeholderDependencyGraph) = - externalDependencyMap[moduleId] else { + // Resolve all target placeholder modules + let placeholderTargetModules = placeholderModules.filter { externalTargetModulePathMap[$0] != nil } + for moduleId in placeholderTargetModules { + guard let placeholderModulePath = externalTargetModulePathMap[moduleId] else { throw Driver.Error.missingExternalDependency(moduleId.moduleName) } - try resolvePlaceholderDependency(placeholderModulePath: placeholderModulePath, - placeholderDependencyGraph: placeholderDependencyGraph) + try resolveTargetPlaceholder(placeholderId: moduleId, + placeholderPath: placeholderModulePath, + dependencyOracle: dependencyOracle) + } + + // Process remaining placeholders until there are none left + placeholderModules = modules.keys.filter(placeholderFilter) + while !placeholderModules.isEmpty { + let moduleId = placeholderModules.first! + let swiftModuleId = ModuleDependencyId.swift(moduleId.moduleName) + guard let moduleInfo = dependencyOracle.getModuleInfo(of: swiftModuleId) else { + throw Driver.Error.missingExternalDependency(moduleId.moduleName) + } + + // Insert the resolved module, replacing the placeholder. + try Self.mergeModule(swiftModuleId, moduleInfo, into: &modules) + + // Traverse and add all of this external module's dependencies to the current graph. + try resolvePlaceholderModuleDependencies(moduleId: swiftModuleId, + dependencyOracle: dependencyOracle) + + // Update the set of remaining placeholders to resolve + placeholderModules = modules.keys.filter(placeholderFilter) } } } fileprivate extension InterModuleDependencyGraph { - /// Merge a given external module's dependency graph in place of a placeholder dependency - mutating func resolvePlaceholderDependency(placeholderModulePath: AbsolutePath, - placeholderDependencyGraph: InterModuleDependencyGraph) + /// Resolve a placeholder dependency that is an external target. + mutating func resolveTargetPlaceholder(placeholderId: ModuleDependencyId, + placeholderPath: AbsolutePath, + dependencyOracle: InterModuleDependencyOracle) throws { - // For every Swift module in the placeholder dependency graph, generate a new module info - // containing only the pre-compiled module path, and insert it into the current module's - // dependency graph, replacing equivalent (non pre-built) modules, if necessary. + // For this placeholder dependency, generate a new module info containing only the pre-compiled + // module path, and insert it into the current module's dependency graph, + // replacing equivalent placeholder module. + // + // For all dependencies of this placeholder (direct and transitive), insert them + // into this module's graph. + // - Swift dependencies are inserted as-is + // - Clang dependencies are inserted as-is, if a matching Clang module is already found + // in this module's graph, we merge moduleInfos of the two modules, in order to obtain + // a super-set of their dependencies at all possible PCMArgs variants. // - // For every Clang module in the placeholder dependency graph, because PCM modules file names - // encode the specific pcmArguments of their dependees, we cannot use pre-built files here - // because we do not always know which target they corrspond to, nor do we have a way to map - // from a certain target to a specific pcm file. Because of this, all PCM dependencies, direct - // and transitive, have to be built for all modules. - for (moduleId, moduleInfo) in placeholderDependencyGraph.modules { - switch moduleId { - case .swift(_): - // Compute the compiled module path for this module. - // If this module is the placeholder itself, this information was passed from SwiftPM - // If this module is any other swift module, then the compiled module path is - // a part of the details field. - // Otherwise (for most other dependencies), it is the modulePath of the moduleInfo node. - let compiledModulePath : String - if moduleId.moduleName == placeholderDependencyGraph.mainModuleName { - compiledModulePath = placeholderModulePath.description - } else if case .swift(let details) = moduleInfo.details, - let explicitModulePath = details.explicitCompiledModulePath { - compiledModulePath = explicitModulePath - } else { - compiledModulePath = moduleInfo.modulePath.description - } + // The placeholder is resolved into a .swiftPrebuiltExternal module in the dependency graph. + // The placeholder's corresponding module may appear in the externalModuleInfoMap as either + // a .swift module or a .swiftPrebuiltExternal module if it had been resolved earlier + // in the multi-module build planning context. + let swiftModuleId = ModuleDependencyId.swift(placeholderId.moduleName) + let swiftPrebuiltModuleId = ModuleDependencyId.swiftPrebuiltExternal(placeholderId.moduleName) + let externalModuleId: ModuleDependencyId + let externalModuleInfo: ModuleInfo + if let moduleInfo = dependencyOracle.getModuleInfo(of: swiftModuleId) { + externalModuleId = swiftModuleId + externalModuleInfo = moduleInfo + } else if let prebuiltModuleInfo = dependencyOracle.getModuleInfo(of: swiftPrebuiltModuleId) { + externalModuleId = swiftPrebuiltModuleId + externalModuleInfo = prebuiltModuleInfo + } else { + throw Driver.Error.missingExternalDependency(placeholderId.moduleName) + } - // We require the extraPCMArgs of all swift modules in order to - // re-scan their clang module dependencies. - let extraPCMArgs : [String] = - try placeholderDependencyGraph.swiftModulePCMArgs(of: moduleId) + let newExternalModuleDetails = + SwiftPrebuiltExternalModuleDetails(compiledModulePath: placeholderPath.description) + let newInfo = ModuleInfo(modulePath: placeholderPath.description, + sourceFiles: [], + directDependencies: externalModuleInfo.directDependencies, + details: .swiftPrebuiltExternal(newExternalModuleDetails)) - let swiftDetails = - SwiftModuleDetails(compiledModulePath: compiledModulePath, - extraPcmArgs: extraPCMArgs) - let newInfo = ModuleInfo(modulePath: moduleInfo.modulePath.description, - sourceFiles: nil, - directDependencies: moduleInfo.directDependencies, - details: ModuleInfo.Details.swift(swiftDetails)) - try insertOrReplaceModule(moduleId: moduleId, moduleInfo: newInfo) - case .clang(_): - if modules[moduleId] == nil { - modules[moduleId] = moduleInfo - } - case .swiftPlaceholder(_): - try insertOrReplaceModule(moduleId: moduleId, moduleInfo: moduleInfo) - } - } + // Insert the resolved module, replacing the placeholder. + try Self.mergeModule(swiftPrebuiltModuleId, newInfo, into: &modules) + + // Traverse and add all of this external target's dependencies to the current graph. + try resolvePlaceholderModuleDependencies(moduleId: externalModuleId, + dependencyOracle: dependencyOracle) } - /// Insert a module into the handler's dependency graph. If a module with this identifier already exists, - /// replace it's module with a moduleInfo that contains a path to an existing prebuilt .swiftmodule - mutating func insertOrReplaceModule(moduleId: ModuleDependencyId, - moduleInfo: ModuleInfo) throws { - // Check for placeholders to be replaced - if modules[ModuleDependencyId.swiftPlaceholder(moduleId.moduleName)] != nil { - try replaceModule(originalId: .swiftPlaceholder(moduleId.moduleName), replacementId: moduleId, - replacementInfo: moduleInfo) - } - // Check for modules with the same Identifier, and replace if found - else if modules[moduleId] != nil { - try replaceModule(originalId: moduleId, replacementId: moduleId, replacementInfo: moduleInfo) - // This module is new to the current dependency graph - } else { - modules[moduleId] = moduleInfo + /// Resolve all dependencies of a placeholder module (direct and transitive), but merging them into the current graph. + mutating func resolvePlaceholderModuleDependencies(moduleId: ModuleDependencyId, + dependencyOracle: InterModuleDependencyOracle) + throws { + guard let resolvingModuleInfo = dependencyOracle.getModuleInfo(of: moduleId) else { + throw Driver.Error.missingExternalDependency(moduleId.moduleName) } - } - /// Replace a module with a new one. Replace all references to the original module in other modules' dependencies - /// with the new module. - mutating func replaceModule(originalId: ModuleDependencyId, - replacementId: ModuleDependencyId, - replacementInfo: ModuleInfo) throws { - modules.removeValue(forKey: originalId) - modules[replacementId] = replacementInfo - for moduleId in modules.keys { - var moduleInfo = modules[moduleId]! - // Skip over other placeholders, they do not have dependencies - if case .swiftPlaceholder(_) = moduleId { - continue + // Breadth-first traversal of all the dependencies of this module + var visited: Set = [] + var toVisit: [ModuleDependencyId] = resolvingModuleInfo.directDependencies ?? [] + var currentIndex = 0 + while let currentId = toVisit[currentIndex...].first { + currentIndex += 1 + visited.insert(currentId) + guard let currentInfo = dependencyOracle.getModuleInfo(of: currentId) else { + throw Driver.Error.missingExternalDependency(currentId.moduleName) } - if let originalModuleIndex = moduleInfo.directDependencies?.firstIndex(of: originalId) { - moduleInfo.directDependencies![originalModuleIndex] = replacementId; + + try Self.mergeModule(currentId, currentInfo, into: &modules) + + let currentDependencies = currentInfo.directDependencies ?? [] + for childId in currentDependencies where !visited.contains(childId) { + if !toVisit.contains(childId) { + toVisit.append(childId) + } } - modules[moduleId] = moduleInfo } } } - -/// Used for creating new module infos during placeholder dependency resolution -/// Modules created this way only contain a path to a pre-built module file. -private extension SwiftModuleDetails { - init(compiledModulePath: String, extraPcmArgs: [String]) { - self.moduleInterfacePath = nil - self.compiledModuleCandidates = nil - self.explicitCompiledModulePath = compiledModulePath - self.bridgingHeaderPath = nil - self.bridgingSourceFiles = nil - self.commandLine = nil - self.extraPcmArgs = extraPcmArgs - self.isFramework = false - } -} diff --git a/Sources/SwiftDriver/Incremental Compilation/BidirectionalMap.swift b/Sources/SwiftDriver/Incremental Compilation/BidirectionalMap.swift new file mode 100644 index 000000000..fdecdc498 --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/BidirectionalMap.swift @@ -0,0 +1,41 @@ +//===------------------ BidirectionalMap.swift ----------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// Like a two-way dictionary, only works for accessing present members +public struct BidirectionalMap { + private var map1: [T1: T2] = [:] + private var map2: [T2: T1] = [:] + + public subscript(_ key: T1) -> T2 { + get { + guard let value = map1[key] + else { + fatalError("\(key) was not present") + } + return value + } + set { + map1[key] = newValue + map2[newValue] = key + } + } + public subscript(_ key: T2) -> T1 { + get { + guard let value = map2[key] + else { + fatalError("\(key) was not present") + } + return value + } + set { self[newValue] = key } + } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/BuildRecord.swift b/Sources/SwiftDriver/Incremental Compilation/BuildRecord.swift new file mode 100644 index 000000000..0877ba393 --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/BuildRecord.swift @@ -0,0 +1,222 @@ +//===--------------- BuildRecord.swift - Swift Input File Info Map -------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import TSCBasic +import Foundation +@_implementationOnly import Yams + +/// Holds the info about inputs needed to plan incremenal compilation +/// A.k.a. BuildRecord was the legacy name +public struct BuildRecord { + public let swiftVersion: String + public let argsHash: String + public let buildTime: Date + /// The date is the modification time of the main input file the last time the driver ran + public let inputInfos: [VirtualPath: InputInfo] + + public init( argsHash: String, + swiftVersion: String, + buildTime: Date, + inputInfos: [VirtualPath: InputInfo]) { + self.argsHash = argsHash + self.swiftVersion = swiftVersion + self.buildTime = buildTime + self.inputInfos = inputInfos + } + + enum SimpleErrors: String, LocalizedError { + case + couldNotDecodeBuildRecord, + sectionNameNotString, + dateValuesNotSequence, + dateValuesNotDuo, + dateValuesNotInts, + inputInfosNotAMap, + inputNotString, + noVersion, + noArgsHash, + noBuildTime, + noInputInfos + + var localizedDescription: String { return rawValue } + } + enum Errors: LocalizedError { + case + unexpectedSection(String), + notAbsolutePath(VirtualPath) + + public var localizedDescription: String { + switch self { + case .unexpectedSection(let s): return "unexpected section \(s)" + case .notAbsolutePath(let p): return "not absolute path \(p)" + } + } + } + private enum SectionName: String, CaseIterable { + case + swiftVersion = "version", + argsHash = "options", + buildTime = "build_time", + inputInfos = "inputs" + + var serializedName: String { rawValue } + } + + var allInputs: Set { + Set( inputInfos.map {$0.key} ) + } +} + +// MARK: - Reading the old map and deciding whether to use it +public extension BuildRecord { + init(contents: String, defaultArgsHash: String? = nil) throws { + guard let sections = try Parser(yaml: contents, resolver: .basic, encoding: .utf8) + .singleRoot()?.mapping + else { throw SimpleErrors.couldNotDecodeBuildRecord } + var argsHash: String? = defaultArgsHash + var swiftVersion: String? + // Legacy driver does not disable incremental if no buildTime field. + var buildTime: Date = .distantPast + var inputInfos: [VirtualPath: InputInfo]? + for (key, value) in sections { + guard let k = key.string else { throw SimpleErrors.sectionNameNotString } + switch k { + case SectionName.swiftVersion.serializedName: + swiftVersion = value.string + case SectionName.argsHash.serializedName: + argsHash = value.string + case SectionName.buildTime.serializedName: + buildTime = try Self.decodeDate(value) + case SectionName.inputInfos.serializedName: + inputInfos = try Self.decodeInputInfos(value) + default: throw Errors.unexpectedSection(k) + } + } + try self.init(argsHash: argsHash, swiftVersion: swiftVersion, buildTime: buildTime, + inputInfos: inputInfos) + } + + private init(argsHash: String?, swiftVersion: String?, buildTime: Date?, + inputInfos: [VirtualPath: InputInfo]?) + throws { + guard let a = argsHash else { throw SimpleErrors.noArgsHash } + guard let s = swiftVersion else { throw SimpleErrors.noVersion } + guard let b = buildTime else { throw SimpleErrors.noBuildTime } + guard let i = inputInfos else { throw SimpleErrors.noInputInfos } + self.init(argsHash: a, swiftVersion: s, buildTime: b, inputInfos: i) + } + + private static func decodeDate(_ node: Yams.Node) throws -> Date { + guard let vals = node.sequence else { throw SimpleErrors.dateValuesNotSequence } + guard vals.count == 2 else {throw SimpleErrors.dateValuesNotDuo} + guard let secs = vals[0].int, let ns = vals[1].int + else {throw SimpleErrors.dateValuesNotInts} + return Date(legacyDriverSecs: secs, nanos: ns) + } + + private static func decodeInputInfos(_ node: Yams.Node) throws -> [VirtualPath: InputInfo] { + guard let map = node.mapping else { throw SimpleErrors.inputInfosNotAMap } + return try Dictionary(uniqueKeysWithValues: + map.map { + keyNode, valueNode in + guard let path = keyNode.string else { throw SimpleErrors.inputNotString } + return try ( + VirtualPath(path: path), + InputInfo(tag: valueNode.tag.description, previousModTime: decodeDate(valueNode)) + ) + } + ) + } +} + +// MARK: - Creating and writing a new map +extension BuildRecord { + /// Create a new buildRecord for writing + init(jobs: [Job], + finishedJobResults: [JobResult], + skippedInputs: Set?, + compilationInputModificationDates: [TypedVirtualPath: Date], + actualSwiftVersion: String, + argsHash: String, + timeBeforeFirstJob: Date + ) { + let jobResultsByInput = Dictionary(uniqueKeysWithValues: + finishedJobResults.flatMap { entry in + entry.j.primaryInputs.map { ($0, entry.result) } + }) + let inputInfosArray = compilationInputModificationDates + .map { input, modDate -> (VirtualPath, InputInfo) in + let status = InputInfo.Status( wasSkipped: skippedInputs?.contains(input), + jobResult: jobResultsByInput[input]) + return (input.file, InputInfo(status: status, previousModTime: modDate)) + } + + self.init( + argsHash: argsHash, + swiftVersion: actualSwiftVersion, + buildTime: timeBeforeFirstJob, + inputInfos: Dictionary(uniqueKeysWithValues: inputInfosArray) + ) + } + + /*@_spi(Testing)*/ public func encode() throws -> String { + let pathsAndInfos = inputInfos.map { + input, inputInfo -> (String, InputInfo) in + return (input.name, inputInfo) + } + let inputInfosNode = Yams.Node( + pathsAndInfos + .sorted {$0.0 < $1.0} + .map {(Yams.Node($0.0, .implicit, .doubleQuoted), Self.encode($0.1))} + ) + let fieldNodes = [ + (SectionName.swiftVersion, Yams.Node(swiftVersion, .implicit, .doubleQuoted)), + (SectionName.argsHash, Yams.Node(argsHash, .implicit, .doubleQuoted)), + (SectionName.buildTime, Self.encode(buildTime)), + (SectionName.inputInfos, inputInfosNode ) + ] .map { (Yams.Node($0.0.serializedName), $0.1) } + + let buildRecordNode = Yams.Node(fieldNodes, .implicit, .block) + // let options = Yams.Emitter.Options(canonical: true) + return try Yams.serialize(node: buildRecordNode, + width: -1, + sortKeys: false) + } + + private static func encode(_ date: Date, tag tagString: String? = nil) -> Yams.Node { + let secsAndNanos = date.legacyDriverSecsAndNanos + return Yams.Node( + secsAndNanos.map {Yams.Node(String($0))}, + tagString.map {Yams.Tag(Yams.Tag.Name(rawValue: $0))} ?? .implicit, + .flow) + } + + private static func encode(_ inputInfo: InputInfo) -> Yams.Node { + encode(inputInfo.previousModTime, tag: inputInfo.tag) + } + +} + +extension Diagnostic.Message { + static func warning_could_not_serialize_build_record(_ err: Error + ) -> Diagnostic.Message { + .warning("next compile won't be incremental; Could not serialize build record: \(err.localizedDescription)") + } + static func warning_could_not_write_build_record_not_absolutePath( + _ path: VirtualPath + ) -> Diagnostic.Message { + .warning("next compile won't be incremental; build record path was not absolute: \(path)") + } + static func warning_could_not_write_build_record(_ path: AbsolutePath + ) -> Diagnostic.Message { + .warning("next compile won't be incremental; could not write build record to \(path)") + } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/BuildRecordInfo.swift b/Sources/SwiftDriver/Incremental Compilation/BuildRecordInfo.swift new file mode 100644 index 000000000..03f802057 --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/BuildRecordInfo.swift @@ -0,0 +1,208 @@ +//===--------------- BuildRecordInfo.swift --------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import Foundation +import TSCBasic +import SwiftOptions + +struct JobResult { + let j: Job + let result: ProcessResult + init(_ j: Job, _ result: ProcessResult) { + self.j = j + self.result = result + } +} + +/// Holds information required to read and write the build record (aka compilation record) +/// This info is always written, but only read for incremental compilation. + class BuildRecordInfo { + let buildRecordPath: VirtualPath + let fileSystem: FileSystem + let argsHash: String + let actualSwiftVersion: String + let timeBeforeFirstJob: Date + let diagnosticEngine: DiagnosticsEngine + let compilationInputModificationDates: [TypedVirtualPath: Date] + + var finishedJobResults = [JobResult]() + + init?( + actualSwiftVersion: String, + compilerOutputType: FileType?, + workingDirectory: AbsolutePath?, + diagnosticEngine: DiagnosticsEngine, + fileSystem: FileSystem, + moduleOutputInfo: ModuleOutputInfo, + outputFileMap: OutputFileMap?, + parsedOptions: ParsedOptions, + recordedInputModificationDates: [TypedVirtualPath: Date] + ) { + // Cannot write a buildRecord without a path. + guard let buildRecordPath = Self.computeBuildRecordPath( + outputFileMap: outputFileMap, + compilerOutputType: compilerOutputType, + workingDirectory: workingDirectory, + diagnosticEngine: diagnosticEngine) + else { + return nil + } + self.actualSwiftVersion = actualSwiftVersion + self.argsHash = Self.computeArgsHash(parsedOptions) + self.buildRecordPath = buildRecordPath + self.compilationInputModificationDates = + recordedInputModificationDates.filter { input, _ in + input.type.isPartOfSwiftCompilation + } + self.diagnosticEngine = diagnosticEngine + self.fileSystem = fileSystem + self.timeBeforeFirstJob = Date() + } + + private static func computeArgsHash(_ parsedOptionsArg: ParsedOptions + ) -> String { + var parsedOptions = parsedOptionsArg + let hashInput = parsedOptions + .filter { $0.option.affectsIncrementalBuild && $0.option.kind != .input} + .map { $0.option.spelling } + .sorted() + .joined() + #if os(macOS) + if #available(macOS 10.15, iOS 13, *) { + return CryptoKitSHA256().hash(hashInput).hexadecimalRepresentation + } else { + return SHA256().hash(hashInput).hexadecimalRepresentation + } + #else + return SHA256().hash(hashInput).hexadecimalRepresentation + #endif + } + + /// Determine the input and output path for the build record + private static func computeBuildRecordPath( + outputFileMap: OutputFileMap?, + compilerOutputType: FileType?, + workingDirectory: AbsolutePath?, + diagnosticEngine: DiagnosticsEngine + ) -> VirtualPath? { + // FIXME: This should work without an output file map. We should have + // another way to specify a build record and where to put intermediates. + guard let ofm = outputFileMap else { + return nil + } + guard let partialBuildRecordPath = + ofm.existingOutputForSingleInput(outputType: .swiftDeps) + else { + diagnosticEngine.emit(.warning_incremental_requires_build_record_entry) + return nil + } + return workingDirectory + .map(partialBuildRecordPath.resolvedRelativePath(base:)) + ?? partialBuildRecordPath + } + + /// Write out the build record. + /// `Jobs` must include all of the compilation jobs. + /// `Inputs` will hold all the primary inputs that were not compiled because of incremental compilation + func writeBuildRecord(_ jobs: [Job], _ skippedInputs: Set? ) { + guard let absPath = buildRecordPath.absolutePath else { + diagnosticEngine.emit( + .warning_could_not_write_build_record_not_absolutePath(buildRecordPath)) + return + } + preservePreviousBuildRecord(absPath) + + let buildRecord = BuildRecord( + jobs: jobs, + finishedJobResults: finishedJobResults, + skippedInputs: skippedInputs, + compilationInputModificationDates: compilationInputModificationDates, + actualSwiftVersion: actualSwiftVersion, + argsHash: argsHash, + timeBeforeFirstJob: timeBeforeFirstJob) + + let contents: String + do { contents = try buildRecord.encode() } + catch let BuildRecord.Errors.notAbsolutePath(p) { + diagnosticEngine.emit( + .warning_could_not_write_build_record_not_absolutePath(p)) + return + } + catch { + diagnosticEngine.emit(.warning_could_not_serialize_build_record(error)) + return + } + do { + try fileSystem.writeFileContents(absPath, + bytes: ByteString(encodingAsUTF8: contents)) + } + catch { + diagnosticEngine.emit(.warning_could_not_write_build_record(absPath)) + return + } + } + + /// Before writing to the dependencies file path, preserve any previous file + /// that may have been there. No error handling -- this is just a nicety, it + /// doesn't matter if it fails. + /// Added for the sake of compatibility with the legacy driver. + private func preservePreviousBuildRecord(_ oldPath: AbsolutePath) { + let newPath = oldPath.withTilde() + try? fileSystem.move(from: oldPath, to: newPath) + } + + +// TODO: Incremental too many names, buildRecord BuildRecord outofdatemap + func populateOutOfDateBuildRecord( + inputFiles: [TypedVirtualPath], + defaultArgsHash: String?, + failed: (String) -> Void + ) -> BuildRecord? { + let outOfDateBuildRecord: BuildRecord + do { + let contents = try fileSystem.readFileContents(buildRecordPath).cString + outOfDateBuildRecord = try BuildRecord(contents: contents, + defaultArgsHash: defaultArgsHash) + } + catch { + failed("could not read build record at \(buildRecordPath): \(error.localizedDescription).") + return nil + } + guard actualSwiftVersion == outOfDateBuildRecord.swiftVersion else { + failed( + "the compiler version has changed from \(outOfDateBuildRecord.swiftVersion) to \(actualSwiftVersion)" + ) + return nil + } + guard argsHash == outOfDateBuildRecord.argsHash else { + failed( "different arguments were passed to the compiler" ) + return nil + } + let missingInputs = Set(outOfDateBuildRecord.inputInfos.keys).subtracting(inputFiles.map {$0.file}) + guard missingInputs.isEmpty else { + failed( "the following inputs were used in the previous compilation but not in this one: " + + missingInputs.map {$0.basename} .joined(separator: ", ")) + return nil + } + return outOfDateBuildRecord + } + + func jobFinished(job: Job, result: ProcessResult) { + finishedJobResults.append(JobResult(job, result)) + } +} + +fileprivate extension AbsolutePath { + func withTilde() -> Self { + parentDirectory.appending(component: basename + "~") + } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/DependencyKey.swift b/Sources/SwiftDriver/Incremental Compilation/DependencyKey.swift new file mode 100644 index 000000000..0b177d26b --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/DependencyKey.swift @@ -0,0 +1,130 @@ + +import Foundation + +/// A filename from another module +/*@_spi(Testing)*/ public struct ExternalDependency: Hashable, Comparable, CustomStringConvertible { + let fileName: String + + var file: VirtualPath? { + try? VirtualPath(path: fileName) + } + /*@_spi(Testing)*/ public init(_ path: String) { + self.fileName = path + } + public var description: String { + fileName.description + } + + public static func < (lhs: Self, rhs: Self) -> Bool { + lhs.fileName < rhs.fileName + } +} + + + +public struct DependencyKey: Hashable, CustomStringConvertible { + /// Instead of the status quo scheme of two kinds of "Depends", cascading and + /// non-cascading this code represents each entity ("Provides" in the status + /// quo), by a pair of nodes. One node represents the "implementation." If the + /// implementation changes, users of the entity need not be recompiled. The + /// other node represents the "interface." If the interface changes, any uses of + /// that definition will need to be recompiled. The implementation always + /// depends on the interface, since any change that alters the interface will + /// require the implementation to be rebuilt. The interface does not depend on + /// the implementation. In the dot files, interfaces are yellow and + /// implementations white. Each node holds an instance variable describing which + /// aspect of the entity it represents. + + /*@_spi(Testing)*/ public enum DeclAspect: Comparable { + case interface, implementation + } + + /// Encode the current sorts of dependencies as kinds of nodes in the dependency + /// graph, splitting the current *member* into \ref member and \ref + /// potentialMember and adding \ref sourceFileProvide. + /// + /*@_spi(Testing)*/ public enum Designator: Hashable, CustomStringConvertible { + case + topLevel(name: String), + dynamicLookup(name: String), + externalDepend(ExternalDependency), + sourceFileProvide(name: String) + + case + nominal(context: String), + potentialMember(context: String) + + case + member(context: String, name: String) + + var externalDependency: ExternalDependency? { + switch self { + case let .externalDepend(externalDependency): + return externalDependency + default: + return nil} + } + + public var description: String { + switch self { + case let .topLevel(name: name): + return "top-level name '\(name)'" + case let .nominal(context: context): + return "type '\(context)'" + case let .potentialMember(context: context): + return "potential members of '\(context)'" + case let .member(context: context, name: name): + return "member '\(name)' of '\(context)'" + case let .dynamicLookup(name: name): + return "AnyObject member '\(name)'" + case let .externalDepend(externalDependency): + return "module '\(externalDependency)'" + case let .sourceFileProvide(name: name): + return "source file '\((try? VirtualPath(path: name).basename) ?? name)'" + } + } + } + + /*@_spi(Testing)*/ public let aspect: DeclAspect + /*@_spi(Testing)*/ public let designator: Designator + + + /*@_spi(Testing)*/ public init( + aspect: DeclAspect, + designator: Designator) + { + self.aspect = aspect + self.designator = designator + } + + + /*@_spi(Testing)*/ public var correspondingImplementation: Self? { + guard aspect == .interface else { + return nil + } + return Self(aspect: .implementation, designator: designator) + } + + public var description: String { + "\(aspect) of \(designator)" + } + + @discardableResult + func verify() -> Bool { + // This space reserved for future use. + return true + } +} + +// MARK: - Comparing +/// Needed to sort nodes to make tracing deterministic to test against emitted diagnostics +extension DependencyKey: Comparable { + public static func < (lhs: Self, rhs: Self) -> Bool { + lhs.aspect != rhs.aspect ? lhs.aspect < rhs.aspect : + lhs.designator < rhs.designator + } +} + +extension DependencyKey.Designator: Comparable { +} + diff --git a/Sources/SwiftDriver/Incremental Compilation/DictionaryOfDictionaries.swift b/Sources/SwiftDriver/Incremental Compilation/DictionaryOfDictionaries.swift new file mode 100644 index 000000000..60149734c --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/DictionaryOfDictionaries.swift @@ -0,0 +1,137 @@ +//===---------------- DictionaryOfDictionaries.swift ----------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// A dictionary of dictionaries that also behaves like dictionary of tuples +/// It supports iterating over all 2nd-level pairs. See `subscript(key: OuterKey)` + +import Foundation +public struct DictionaryOfDictionaries: Collection { + public typealias InnerDict = [InnerKey: Value] + public typealias OuterDict = [OuterKey: InnerDict] + + public typealias Key = (OuterKey, InnerKey) + public typealias Element = (Key, Value) + + var outerDict = [OuterKey: [InnerKey: Value]]() +} + +// MARK: indices +extension DictionaryOfDictionaries { + public enum Index: Comparable { + case end + case notEnd(OuterDict.Index, InnerDict.Index) + + public static func < (lhs: Self, rhs: Self) + -> Bool + { + switch (lhs, rhs) { + case (.end, .end): return false + case (_, .end): return true + case (.end, _): return false + case let (.notEnd(lo, li), .notEnd(ro, ri)): + switch (lo, ro, li, ri) { + case let (lo, ro, _, _) where lo != ro: return lo < ro + case let (_, _, li, ri): return li < ri + } + } + } + } + + private func makeIndex(_ oi: OuterDict.Index, _ ii: InnerDict.Index) -> Index { + assert(outerDict[oi].value.indices.contains(ii)) + return .notEnd(oi, ii) + } + + public var startIndex: Index { + return outerDict.isEmpty + ? endIndex + : makeIndex(outerDict.startIndex, + outerDict.first!.value.startIndex) + } + public var endIndex: Index { + return .end + } + + public func index(after i: Index) -> Index { + switch i { + case .end: fatalError("index at end") + case let .notEnd(outerIndex, innerIndex): + let innerDict = outerDict[outerIndex].value + let nextInnerIndex = innerDict.index(after: innerIndex) + if nextInnerIndex < innerDict.endIndex { + return makeIndex(outerIndex, nextInnerIndex) + } + let nextOuterIndex = outerDict.index(after: outerIndex) + if nextOuterIndex < outerDict.endIndex { + return .notEnd(nextOuterIndex, outerDict[nextOuterIndex].value.startIndex) + } + return .end + } + } +} + +// MARK: - subscripting +extension DictionaryOfDictionaries { + public subscript(position: Index) -> Element { + switch position { + case .end: fatalError("index at end") + case let .notEnd(outerIndex, innerIndex): + let (outerKey, innerDict) = outerDict[outerIndex] + let (innerKey, value) = innerDict[innerIndex] + return (key: (outerKey, innerKey), value: value) + } + } + + public subscript(key: Key) -> Value? { + get {outerDict[key.0]?[key.1]} + set { + if let v = newValue { _ = updateValue(v, forKey: key) } + else { _ = removeValue(forKey: key) } + } + } + + public subscript(key: OuterKey) -> [InnerKey: Value]? { + get {outerDict[key]} + set { + if let v = newValue { _ = outerDict.updateValue(v, forKey: key) } + else { _ = outerDict.removeValue(forKey: key) } + } + } +} + +// MARK: - mutating +extension DictionaryOfDictionaries { + mutating func updateValue(_ v: Value, forKey keys : (OuterKey,InnerKey) + ) -> Value? { + if var innerDict = outerDict[keys.0] { + let old = innerDict.updateValue(v, forKey: keys.1) + outerDict.updateValue(innerDict, forKey: keys.0) + return old + } + outerDict.updateValue([keys.1: v], forKey: keys.0) + return nil + } + + mutating func removeValue(forKey keys : (OuterKey,InnerKey) + ) -> Value? { + guard var innerDict = outerDict[keys.0] + else { return nil } + let old = innerDict.removeValue(forKey: keys.1) + if innerDict.isEmpty { + outerDict.removeValue(forKey: keys.0) + } + else { + outerDict.updateValue(innerDict, forKey: keys.0) + } + return old + } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/IncrementalCompilationState.swift b/Sources/SwiftDriver/Incremental Compilation/IncrementalCompilationState.swift index 24ca36743..fa5fa7334 100644 --- a/Sources/SwiftDriver/Incremental Compilation/IncrementalCompilationState.swift +++ b/Sources/SwiftDriver/Incremental Compilation/IncrementalCompilationState.swift @@ -12,99 +12,138 @@ import TSCBasic import Foundation import SwiftOptions +public class IncrementalCompilationState { -public struct IncrementalCompilationState { - public let showIncrementalBuildDecisions: Bool - public let enableIncrementalBuild: Bool - public let buildRecordPath: VirtualPath? - public let outputBuildRecordForModuleOnlyBuild: Bool - public let argsHash: String - public let lastBuildTime: Date - public let outOfDateMap: InputInfoMap? - public var rebuildEverything: Bool { return outOfDateMap == nil } - - public init(_ parsedOptions: inout ParsedOptions, - compilerMode: CompilerMode, - outputFileMap: OutputFileMap?, - compilerOutputType: FileType?, - moduleOutput: ModuleOutputInfo.ModuleOutput?, - fileSystem: FileSystem, - inputFiles: [TypedVirtualPath], - diagnosticEngine: DiagnosticsEngine, - actualSwiftVersion: String? + /// The oracle for deciding what depends on what. Applies to this whole module. + private let moduleDependencyGraph: ModuleDependencyGraph + + /// If non-null outputs information for `-driver-show-incremental` for input path + public let reportIncrementalDecision: ((String, TypedVirtualPath?) -> Void)? + + /// All of the pre-compile or compilation job (groups) known to be required, preserving planning order + public private (set) var mandatoryPreOrCompileJobsInOrder = [Job]() + + /// All the pre- or compilation job (groups) known to be required, which have not finished yet. + /// (Changes as jobs complete.) + private var unfinishedMandatoryJobs = Set() + + /// Inputs that must be compiled, and swiftDeps processed. + /// When empty, the compile phase is done. + private var pendingInputs = Set() + + /// Input files that were skipped. + /// May shrink if one of these moves into pendingInputs. In that case, it will be an input to a + /// "newly-discovered" job. + private(set) var skippedCompilationInputs: Set + + /// Job groups that were skipped. + /// Need groups rather than jobs because a compile that emits bitcode and its backend job must be + /// treated as a unit. + private var skippedCompileGroups = [TypedVirtualPath: [Job]]() + + /// Jobs to run *after* the last compile, for instance, link-editing. + public private(set) var postCompileJobs = [Job]() + + /// A check for reentrancy. + private var amHandlingJobCompletion = false + +// MARK: - Creating IncrementalCompilationState if possible + /// Return nil if not compiling incrementally + init?( + buildRecordInfo: BuildRecordInfo?, + compilerMode: CompilerMode, + diagnosticEngine: DiagnosticsEngine, + fileSystem: FileSystem, + inputFiles: [TypedVirtualPath], + outputFileMap: OutputFileMap?, + parsedOptions: inout ParsedOptions, + showJobLifecycle: Bool ) { - let showIncrementalBuildDecisions = Self.getShowIncrementalBuildDecisions(&parsedOptions) - self.showIncrementalBuildDecisions = showIncrementalBuildDecisions - - let enableIncrementalBuild = Self.computeAndExplainShouldCompileIncrementally( - &parsedOptions, - showIncrementalBuildDecisions: showIncrementalBuildDecisions, - compilerMode: compilerMode, - diagnosticEngine: diagnosticEngine) - - self.enableIncrementalBuild = enableIncrementalBuild - - self.buildRecordPath = Self.computeBuildRecordPath( - outputFileMap: outputFileMap, - compilerOutputType: compilerOutputType, - diagnosticEngine: enableIncrementalBuild ? diagnosticEngine : nil) - - // If we emit module along with full compilation, emit build record - // file for '-emit-module' only mode as well. - self.outputBuildRecordForModuleOnlyBuild = self.buildRecordPath != nil && - moduleOutput?.isTopLevel ?? false - - let argsHash = Self.computeArgsHash(parsedOptions) - self.argsHash = argsHash - let lastBuildTime = Date() - self.lastBuildTime = lastBuildTime - - if let buRP = buildRecordPath, enableIncrementalBuild { - let outOfDateMap = InputInfoMap.populateOutOfDateMap( - argsHash: argsHash, - lastBuildTime: lastBuildTime, - fileSystem: fileSystem, - inputFiles: inputFiles, - buildRecordPath: buRP, - showIncrementalBuildDecisions: showIncrementalBuildDecisions, - diagnosticEngine: diagnosticEngine) - if let mismatchReason = outOfDateMap?.matches( - argsHash: argsHash, - inputFiles: inputFiles, - actualSwiftVersion: actualSwiftVersion - ) { - diagnosticEngine.emit(.remark_incremental_compilation_disabled(because: mismatchReason)) - self.outOfDateMap = nil - } - else { - self.outOfDateMap = outOfDateMap - } + guard Self.shouldAttemptIncrementalCompilation( + parsedOptions: &parsedOptions, + compilerMode: compilerMode, + diagnosticEngine: diagnosticEngine) + else { + return nil + } + + guard let outputFileMap = outputFileMap, + let buildRecordInfo = buildRecordInfo + else { + diagnosticEngine.emit(.warning_incremental_requires_output_file_map) + return nil } + + // Mimic the legacy driver for testing ease: If no `argsHash` section, + // record still matches. + let defaultArgsHash = buildRecordInfo.argsHash + + // FIXME: This should work without an output file map. We should have + // another way to specify a build record and where to put intermediates. + guard let outOfDateBuildRecord = buildRecordInfo.populateOutOfDateBuildRecord( + inputFiles: inputFiles, + defaultArgsHash: defaultArgsHash, + failed: { + diagnosticEngine.emit( + .remark_incremental_compilation_disabled(because: $0)) + }) else { - self.outOfDateMap = nil + return nil } - } - private static func getShowIncrementalBuildDecisions(_ parsedOptions: inout ParsedOptions) - -> Bool { - parsedOptions.hasArgument(.driverShowIncremental) + let reportIncrementalDecision = + parsedOptions.hasArgument(.driverShowIncremental) || showJobLifecycle + ? { Self.reportIncrementalDecisionFn($0, $1, outputFileMap, diagnosticEngine) } + : nil + + guard let (moduleDependencyGraph, inputsWithUnreadableSwiftDeps) = + ModuleDependencyGraph.buildInitialGraph( + diagnosticEngine: diagnosticEngine, + inputs: buildRecordInfo.compilationInputModificationDates.keys, + previousInputs: outOfDateBuildRecord.allInputs, + outputFileMap: outputFileMap, + parsedOptions: &parsedOptions, + remarkDisabled: Diagnostic.Message.remark_incremental_compilation_disabled, + reportIncrementalDecision: reportIncrementalDecision) + else { + return nil + } + // preserve legacy behavior + if let badSwiftDeps = inputsWithUnreadableSwiftDeps.first?.1 { + diagnosticEngine.emit( + .remark_incremental_compilation_disabled( + because: "malformed swift dependencies file '\(badSwiftDeps)'") + ) + return nil + } + + // But someday, just ensure inputsWithUnreadableSwiftDeps are compiled + self.skippedCompilationInputs = Self.computeSkippedCompilationInputs( + inputFiles: inputFiles, + inputsWithUnreadableSwiftDeps: inputsWithUnreadableSwiftDeps.map {$0.0}, + buildRecordInfo: buildRecordInfo, + moduleDependencyGraph: moduleDependencyGraph, + outOfDateBuildRecord: outOfDateBuildRecord, + alwaysRebuildDependents: parsedOptions.contains(.driverAlwaysRebuildDependents), + reportIncrementalDecision: reportIncrementalDecision) + + self.moduleDependencyGraph = moduleDependencyGraph + self.reportIncrementalDecision = reportIncrementalDecision } - private static func computeAndExplainShouldCompileIncrementally( - _ parsedOptions: inout ParsedOptions, - showIncrementalBuildDecisions: Bool, + /// Check various arguments to rule out incremental compilation if need be. + private static func shouldAttemptIncrementalCompilation( + parsedOptions: inout ParsedOptions, compilerMode: CompilerMode, diagnosticEngine: DiagnosticsEngine - ) - -> Bool - { + ) -> Bool { guard parsedOptions.hasArgument(.incremental) else { return false } guard compilerMode.supportsIncrementalCompilation else { - diagnosticEngine.emit( - .remark_incremental_compilation_disabled( - because: "it is not compatible with \(compilerMode)")) + diagnosticEngine.emit( + .remark_incremental_compilation_disabled( + because: "it is not compatible with \(compilerMode)")) return false } guard !parsedOptions.hasArgument(.embedBitcode) else { @@ -115,43 +154,8 @@ public struct IncrementalCompilationState { } return true } - - private static func computeBuildRecordPath( - outputFileMap: OutputFileMap?, - compilerOutputType: FileType?, - diagnosticEngine: DiagnosticsEngine? - ) -> VirtualPath? { - // FIXME: This should work without an output file map. We should have - // another way to specify a build record and where to put intermediates. - guard let ofm = outputFileMap else { - diagnosticEngine.map { $0.emit(.warning_incremental_requires_output_file_map) } - return nil - } - guard let partialBuildRecordPath = ofm.existingOutputForSingleInput(outputType: .swiftDeps) - else { - diagnosticEngine.map { $0.emit(.warning_incremental_requires_build_record_entry) } - return nil - } - // In 'emit-module' only mode, use build-record filename suffixed with - // '~moduleonly'. So that module-only mode doesn't mess up build-record - // file for full compilation. - return compilerOutputType == .swiftModule - ? partialBuildRecordPath.appendingToBaseName("~moduleonly") - : partialBuildRecordPath - } - - static private func computeArgsHash(_ parsedOptionsArg: ParsedOptions) -> String { - var parsedOptions = parsedOptionsArg - let hashInput = parsedOptions - .filter { $0.option.affectsIncrementalBuild && $0.option.kind != .input} - .map {$0.option.spelling} - .sorted() - .joined() - return SHA256().hash(hashInput).hexadecimalRepresentation - } } - fileprivate extension CompilerMode { var supportsIncrementalCompilation: Bool { switch self { @@ -161,17 +165,372 @@ fileprivate extension CompilerMode { } } +// MARK: - Outputting debugging info +fileprivate extension IncrementalCompilationState { + private static func reportIncrementalDecisionFn( + _ s: String, + _ path: TypedVirtualPath?, + _ outputFileMap: OutputFileMap, + _ diagnosticEngine: DiagnosticsEngine + ) { + let IO = path.flatMap { + $0.type == .swift ? $0.file : outputFileMap.getInput(outputFile: $0.file) + } + .map {($0.basename, + outputFileMap.getOutput(inputFile: $0, outputType: .object).basename + )} + let pathPart = IO.map { " {compile: \($0.1) <= \($0.0)}" } + let message = "\(s)\(pathPart ?? "")" + diagnosticEngine.emit(.remark_incremental_compilation(because: message)) + } +} + extension Diagnostic.Message { - static var warning_incremental_requires_output_file_map: Diagnostic.Message { + fileprivate static var warning_incremental_requires_output_file_map: Diagnostic.Message { .warning("ignoring -incremental (currently requires an output file map)") } static var warning_incremental_requires_build_record_entry: Diagnostic.Message { .warning( "ignoring -incremental; " + - "output file map has no master dependencies entry under \(FileType.swiftDeps)" + "output file map has no master dependencies entry (\"\(FileType.swiftDeps)\" under \"\")" + ) + } + fileprivate static func remark_incremental_compilation_disabled(because why: String) -> Diagnostic.Message { + .remark("Disabling incremental build: \(why)") + } + fileprivate static func remark_incremental_compilation(because why: String) -> Diagnostic.Message { + .remark("Incremental compilation: \(why)") + } +} + + +// MARK: - Scheduling the first wave, i.e. the mandatory pre- and compile jobs + +extension IncrementalCompilationState { + + /// Figure out which compilation inputs are *not* mandatory + private static func computeSkippedCompilationInputs( + inputFiles: [TypedVirtualPath], + inputsWithUnreadableSwiftDeps: [TypedVirtualPath], + buildRecordInfo: BuildRecordInfo, + moduleDependencyGraph: ModuleDependencyGraph, + outOfDateBuildRecord: BuildRecord, + alwaysRebuildDependents: Bool, + reportIncrementalDecision: ((String, TypedVirtualPath?) -> Void)? + ) -> Set { + + let changedInputs: [(TypedVirtualPath, InputInfo.Status)] = computeChangedInputs( + inputFiles: inputFiles, + buildRecordInfo: buildRecordInfo, + moduleDependencyGraph: moduleDependencyGraph, + outOfDateBuildRecord: outOfDateBuildRecord, + reportIncrementalDecision: reportIncrementalDecision) + let externalDependents = computeExternallyDependentInputs( + buildTime: outOfDateBuildRecord.buildTime, + fileSystem: buildRecordInfo.fileSystem, + moduleDependencyGraph: moduleDependencyGraph, + reportIncrementalDecision: reportIncrementalDecision) + + // Combine to obtain the inputs that definitely must be recompiled. + let definitelyRequiredInputs = Set(changedInputs.map {$0.0} + externalDependents + inputsWithUnreadableSwiftDeps) + if let report = reportIncrementalDecision { + for scheduledInput in definitelyRequiredInputs.sorted(by: {$0.file.name < $1.file.name}) { + report("Queuing (initial):", scheduledInput) + } + } + + // Sometimes, inputs run in the first wave that depend on the changed inputs for the + // first wave, even though they may not require compilation. + // Any such inputs missed, will be found by the rereading of swiftDeps + // as each first wave job finished. + let speculativeInputs = computeSpeculativeInputs( + changedInputs: changedInputs, + moduleDependencyGraph: moduleDependencyGraph, + alwaysRebuildDependents: alwaysRebuildDependents, + reportIncrementalDecision: reportIncrementalDecision) + .subtracting(definitelyRequiredInputs) + + if let report = reportIncrementalDecision { + for dependent in speculativeInputs.sorted(by: {$0.file.name < $1.file.name}) { + report("Queuing (dependent):", dependent) + } + } + let immediatelyCompiledInputs = definitelyRequiredInputs.union(speculativeInputs) + + let skippedInputs = Set(buildRecordInfo.compilationInputModificationDates.keys) + .subtracting(immediatelyCompiledInputs) + if let report = reportIncrementalDecision { + for skippedInput in skippedInputs.sorted(by: {$0.file.name < $1.file.name}) { + report("Skipping input:", skippedInput) + } + } + return skippedInputs + } + + /// Find the inputs that have changed since last compilation, or were marked as needed a build + private static func computeChangedInputs( + inputFiles: [TypedVirtualPath], + buildRecordInfo: BuildRecordInfo, + moduleDependencyGraph: ModuleDependencyGraph, + outOfDateBuildRecord: BuildRecord, + reportIncrementalDecision: ((String, TypedVirtualPath?) -> Void)? + ) -> [(TypedVirtualPath, InputInfo.Status)] { + inputFiles.compactMap { input in + guard input.type.isPartOfSwiftCompilation else { + return nil + } + let modDate = buildRecordInfo.compilationInputModificationDates[input] + ?? Date.distantFuture + let inputInfo = outOfDateBuildRecord.inputInfos[input.file] + let previousCompilationStatus = inputInfo?.status ?? .newlyAdded + let previousModTime = inputInfo?.previousModTime + + // Because legacy driver reads/writes dates wrt 1970, + // and because converting time intervals to/from Dates from 1970 + // exceeds Double precision, must not compare dates directly + var datesMatch: Bool { + modDate.timeIntervalSince1970 == previousModTime?.timeIntervalSince1970 + } + + switch previousCompilationStatus { + + case .upToDate where datesMatch: + reportIncrementalDecision?("May skip current input:", input) + return nil + + case .upToDate: + reportIncrementalDecision?("Scheduing changed input", input) + case .newlyAdded: + reportIncrementalDecision?("Scheduling new", input) + case .needsCascadingBuild: + reportIncrementalDecision?("Scheduling cascading build", input) + case .needsNonCascadingBuild: + reportIncrementalDecision?("Scheduling noncascading build", input) + } + return (input, previousCompilationStatus) + } + } + + + /// Any files dependent on modified files from other modules must be compiled, too. + private static func computeExternallyDependentInputs( + buildTime: Date, + fileSystem: FileSystem, + moduleDependencyGraph: ModuleDependencyGraph, + reportIncrementalDecision: ((String, TypedVirtualPath?) -> Void)? + ) -> [TypedVirtualPath] { + var externallyDependentSwiftDeps = Set() + for extDep in moduleDependencyGraph.externalDependencies { + let extModTime = extDep.file.flatMap { + try? fileSystem.getFileInfo($0).modTime} + ?? Date.distantFuture + if extModTime >= buildTime { + moduleDependencyGraph.forEachUntracedSwiftDepsDirectlyDependent(on: extDep) { + reportIncrementalDecision?( + "Scheduling externally-dependent on newer \(extDep.file?.basename ?? "extDep?")", + TypedVirtualPath(file: $0.file, type: .swiftDeps)) + externallyDependentSwiftDeps.insert($0) + } + } + } + return externallyDependentSwiftDeps.compactMap { + moduleDependencyGraph.sourceSwiftDepsMap[$0] + } + } + + /// Returns the cascaded files to compile in the first wave, even though it may not be need. + /// The needs[Non}CascadingBuild stuff was cargo-culted from the legacy driver. + /// TODO: something better, e.g. return nothing here, but process changed swiftDeps + /// before the whole frontend job finished. + private static func computeSpeculativeInputs( + changedInputs: [(TypedVirtualPath, InputInfo.Status)], + moduleDependencyGraph: ModuleDependencyGraph, + alwaysRebuildDependents: Bool, + reportIncrementalDecision: ((String, TypedVirtualPath?) -> Void)? + ) -> Set { + // Collect the files that will be compiled whose dependents should be schedule + let cascadingFiles: [TypedVirtualPath] = changedInputs.compactMap { input, status in + let basename = input.file.basename + switch (status, alwaysRebuildDependents) { + + case (_, true): + reportIncrementalDecision?( + "scheduling dependents of \(basename); -driver-always-rebuild-dependents", nil) + return input + case (.needsCascadingBuild, false): + reportIncrementalDecision?( + "scheduling dependents of \(basename); needed cascading build", nil) + return input + + case (.upToDate, false): // was up to date, but changed + reportIncrementalDecision?( + "not scheduling dependents of \(basename); unknown changes", nil) + return nil + case (.newlyAdded, false): + reportIncrementalDecision?( + "not scheduling dependents of \(basename): no entry in build record or dependency graph", nil) + return nil + case (.needsNonCascadingBuild, false): + reportIncrementalDecision?( + "not scheduling dependents of \(basename): does not need cascading build", nil) + return nil + } + } + // Collect the dependent files to speculatively schedule + var dependentFiles = Set() + let cascadingFileSet = Set(cascadingFiles) + for cascadingFile in cascadingFiles { + let dependentsOfOneFile = moduleDependencyGraph + .findDependentSourceFiles(of: cascadingFile, reportIncrementalDecision) + for dep in dependentsOfOneFile where !cascadingFileSet.contains(dep) { + if dependentFiles.insert(dep).0 { + reportIncrementalDecision?( + "Immediately scheduling dependent on \(cascadingFile.file.basename)", dep) + } + } + } + return dependentFiles + } +} + +// MARK: - Scheduling +extension IncrementalCompilationState { + /// Remember a job (group) that is before a compile or a compile itself. + /// (A group also includes the "backend" jobs for bitcode.) + /// Decide if a job can be skipped, and register accordingly + func addPreOrCompileJobGroups(_ groups: [[Job]], + formBatchedJobs: ([Job]) throws -> [Job] + ) throws { + let mandatoryPreOrCompileJobs = groups.flatMap { group -> [Job] in + if let firstJob = group.first, isSkipped(firstJob) { + recordSkippedGroup(group) + return [] + } + return group + } + let batchedMandatoryPreOrCompileJobs = try formBatchedJobs(mandatoryPreOrCompileJobs) + scheduleMandatoryPreOrCompile(jobs: batchedMandatoryPreOrCompileJobs) + } + + /// Remember that `group` (a compilation and possibly bitcode generation) + /// must definitely be executed. + private func scheduleMandatoryPreOrCompile(jobs: [Job]) { + if let report = reportIncrementalDecision { + for job in jobs { + report("Queuing \(job.descriptionForLifecycle)", nil) + } + } + mandatoryPreOrCompileJobsInOrder.append(contentsOf: jobs) + unfinishedMandatoryJobs.formUnion(jobs) + let mandatoryCompilationInputs = jobs + .flatMap {$0.kind == .compile ? $0.primaryInputs : []} + pendingInputs.formUnion(mandatoryCompilationInputs) + } + + /// Decide if this job does not need to run, unless some yet-to-be-discovered dependency changes. + private func isSkipped(_ job: Job) -> Bool { + guard job.kind == .compile else { + return false + } + assert(job.primaryInputs.count <= 1, "Jobs should not be batched here.") + return job.primaryInputs.first.map(skippedCompilationInputs.contains) ?? false + } + + /// Remember that this job-group will be skipped (but may be needed later) + private func recordSkippedGroup(_ group: [Job]) { + let job = group.first! + for input in job.primaryInputs { + if let _ = skippedCompileGroups.updateValue(group, forKey: input) { + fatalError("should not have two skipped jobs for same skipped input") + } + } + } + + /// Remember a job that runs after all compile jobs, e.g., ld + func addPostCompileJobs(_ jobs: [Job]) { + self.postCompileJobs = jobs + for job in jobs { + if let report = reportIncrementalDecision { + for input in job.primaryInputs { + report("Delaying pending discovering delayed dependencies", input) + } + } + } + } + + /// `job` just finished. Update state, and return the skipped compile job (groups) that are now known to be needed. + /// If no more compiles are needed, return nil. + /// Careful: job may not be primary. + public func getJobsDiscoveredToBeNeededAfterFinishing( + job finishedJob: Job, result: ProcessResult) + -> [Job]? { + defer { + amHandlingJobCompletion = false + } + assert(!amHandlingJobCompletion, "was reentered, need to synchronize") + amHandlingJobCompletion = true + + unfinishedMandatoryJobs.remove(finishedJob) + if finishedJob.kind == .compile { + finishedJob.primaryInputs.forEach { + if pendingInputs.remove($0) == nil { + fatalError("\($0) input to newly-finished \(finishedJob) should have been pending") + } + } + } + + // Find and deal with inputs that how need to be compiled + let discoveredInputs = collectInputsDiscovered(from: finishedJob) + assert(Set(discoveredInputs).isDisjoint(with: finishedJob.primaryInputs), + "Primaries should not overlap secondaries.") + skippedCompilationInputs.subtract(discoveredInputs) + pendingInputs.formUnion(discoveredInputs) + + if let report = reportIncrementalDecision { + for input in discoveredInputs { + report("Queuing because of dependencies discovered later:", input) + } + } + if pendingInputs.isEmpty && unfinishedMandatoryJobs.isEmpty { + // no more compilations are possible + return nil + } + return getJobsFor(discoveredCompilationInputs: discoveredInputs) + } + + /// After `job` finished find out which inputs must compiled that were not known to need compilation before + private func collectInputsDiscovered(from job: Job) -> [TypedVirtualPath] { + Array( + Set( + job.primaryInputs.flatMap { + input -> [TypedVirtualPath] in + if let found = moduleDependencyGraph.findSourcesToCompileAfterCompiling(input) { + return found + } + reportIncrementalDecision?("Failed to read some swiftdeps; compiling everything", input) + return Array(skippedCompilationInputs) + } + ) ) + .sorted {$0.file.name < $1.file.name} } - static func remark_incremental_compilation_disabled(because why: String) -> Diagnostic.Message { - .remark("Incremental compilation has been disabled, because \(why).\n") + + /// Find the jobs that now must be run that were not originally known to be needed. + private func getJobsFor( + discoveredCompilationInputs inputs: [TypedVirtualPath] + ) -> [Job] { + inputs.flatMap { input -> [Job] in + if let group = skippedCompileGroups.removeValue(forKey: input) { + let primaryInputs = group.first!.primaryInputs + assert(primaryInputs.count == 1) + assert(primaryInputs[0] == input) + reportIncrementalDecision?("Scheduling discovered", input) + return group + } + else { + reportIncrementalDecision?("Tried to schedule discovered input again", input) + return [] + } + } } } diff --git a/Sources/SwiftDriver/Incremental Compilation/InputIInfoMap.swift b/Sources/SwiftDriver/Incremental Compilation/InputIInfoMap.swift deleted file mode 100644 index fa2958106..000000000 --- a/Sources/SwiftDriver/Incremental Compilation/InputIInfoMap.swift +++ /dev/null @@ -1,176 +0,0 @@ -//===--------------- InputInfoMap.swift - Swift Input File Info Map -------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// -import TSCBasic -import Foundation -@_implementationOnly import Yams - -/// Holds the info about inputs needed to plan incremenal compilation -public struct InputInfoMap { - public let swiftVersion: String - public let argsHash: String - public let buildTime: Date - /// The date is the modification time of the main input file the last time the driver ran - public let inputInfos: [VirtualPath: InputInfo] - - public init( argsHash: String, - swiftVersion: String, - buildTime: Date, - inputInfos: [VirtualPath: InputInfo]) { - self.argsHash = argsHash - self.swiftVersion = swiftVersion - self.buildTime = buildTime - self.inputInfos = inputInfos - } -} - -// Reading -public extension InputInfoMap { - private enum SectionName: String, CaseIterable { - case - swiftVersion = "version", - argsHash = "options", - buildTime = "build_time", - inputInfos = "inputs" - } - - enum SimpleErrors: String, LocalizedError { - case - couldNotDecodeInputInfoMap, - sectionNameNotString, - dateValuesNotSequence, - dateValuesNotDuo, - dateValuesNotInts, - inputInfosNotAMap, - inputNotString, - noVersion, - noArgsHash, - noBuildTime, - noInputInfos - - var localizedDescription: String { return rawValue } - } - enum Errors: LocalizedError { - case - unexpectedSection(String) - public var localizedDescription: String { - switch self { - case .unexpectedSection(let s): return "unexpected section \(s)" - } - } - } - - init(contents: String) throws { - guard let sections = try Parser(yaml: contents, resolver: .basic, encoding: .utf8) - .singleRoot()?.mapping - else { throw SimpleErrors.couldNotDecodeInputInfoMap } - var argsHash, swiftVersion: String? - var buildTime: Date? - var inputInfos: [VirtualPath: InputInfo]? - for (key, value) in sections { - guard let k = key.string else { throw SimpleErrors.sectionNameNotString } - switch k { - case "version": swiftVersion = value.string - case "options": argsHash = value.string - case "build_time": buildTime = try Self.decodeDate(value) - case "inputs": inputInfos = try Self.decodeInputInfos(value) - default: throw Errors.unexpectedSection(k) - } - } - try self.init(argsHash: argsHash, swiftVersion: swiftVersion, buildTime: buildTime, - inputInfos: inputInfos) - } - - private init(argsHash: String?, swiftVersion: String?, buildTime: Date?, - inputInfos: [VirtualPath: InputInfo]?) - throws { - guard let a = argsHash else { throw SimpleErrors.noArgsHash } - guard let s = swiftVersion else { throw SimpleErrors.noVersion } - guard let b = buildTime else { throw SimpleErrors.noBuildTime } - guard let i = inputInfos else { throw SimpleErrors.noInputInfos } - self.init(argsHash: a, swiftVersion: s, buildTime: b, inputInfos: i) - } - - static private func decodeDate(_ node: Yams.Node) throws -> Date { - guard let vals = node.sequence else { throw SimpleErrors.dateValuesNotSequence } - guard vals.count == 2 else {throw SimpleErrors.dateValuesNotDuo} - guard let secs = vals[0].int, let ns = vals[1].int - else {throw SimpleErrors.dateValuesNotInts} - return Date(legacyDriverSecs: secs, nanos: ns) - } - - static private func decodeInputInfos(_ node: Yams.Node) throws -> [VirtualPath: InputInfo] { - guard let map = node.mapping else { throw SimpleErrors.inputInfosNotAMap } - return try Dictionary(uniqueKeysWithValues: - map.map { - keyNode, valueNode in - guard let path = keyNode.string else { throw SimpleErrors.inputNotString } - return try ( - VirtualPath(path: path), - InputInfo(tag: valueNode.tag.description, previousModTime: decodeDate(valueNode)) - ) - } - ) - } -} - - - -/// Reading the old map and deciding whether to use it -public extension InputInfoMap { - static func populateOutOfDateMap( - argsHash: String, - lastBuildTime: Date, - fileSystem: FileSystem, - inputFiles: [TypedVirtualPath], - buildRecordPath: VirtualPath, - showIncrementalBuildDecisions: Bool, - diagnosticEngine: DiagnosticsEngine - ) -> Self? { - let contents: String - do { - contents = try fileSystem.readFileContents(buildRecordPath).cString - return try Self(contents: contents) - } - catch { - if showIncrementalBuildDecisions { - diagnosticEngine.emit(.remark_could_not_read_build_record(error)) - } - return nil - } - } - - /// Returns why it did not match - func matches(argsHash: String, inputFiles: [TypedVirtualPath], actualSwiftVersion: String?) -> String? { - guard let actualSwiftVersion = actualSwiftVersion else { - return "the version of the compiler we will be using could not determined" - } - guard actualSwiftVersion == self.swiftVersion else { - return "the compiler version has changed from \(self.swiftVersion) to \(actualSwiftVersion)" - } - guard argsHash == self.argsHash else { - return "different arguments were passed to the compiler" - } - let missingInputs = Set(self.inputInfos.keys).subtracting(inputFiles.map {$0.file}) - guard missingInputs.isEmpty else { - return "the following inputs were used in the previous compilation but not in this one: " - + missingInputs.map {$0.name} .joined(separator: ", ") - } - return nil - } -} - - -extension Diagnostic.Message { - static func remark_could_not_read_build_record(_ error: Error) -> Diagnostic.Message { - .remark("Incremental compilation could not read build record: \(error.localizedDescription).") - } -} diff --git a/Sources/SwiftDriver/Incremental Compilation/InputInfo.swift b/Sources/SwiftDriver/Incremental Compilation/InputInfo.swift index 50232527a..c85135950 100644 --- a/Sources/SwiftDriver/Incremental Compilation/InputInfo.swift +++ b/Sources/SwiftDriver/Incremental Compilation/InputInfo.swift @@ -10,24 +10,50 @@ // //===----------------------------------------------------------------------===// import Foundation +import TSCBasic -public struct InputInfo: Equatable { +/*@_spi(Testing)*/ public struct InputInfo: Equatable { - let status: Status - let previousModTime: Date + /*@_spi(Testing)*/ public let status: Status + /*@_spi(Testing)*/ public let previousModTime: Date - public init(status: Status, previousModTime: Date) { + /*@_spi(Testing)*/ public init(status: Status, previousModTime: Date) { self.status = status self.previousModTime = previousModTime } } -public extension InputInfo { - enum Status { - case upToDate, needsCascadingBuild, needsNonCascadingBuild, newlyAdded +/*@_spi(Testing)*/ public extension InputInfo { + enum Status: Equatable { + case upToDate, + needsCascadingBuild, + needsNonCascadingBuild, + newlyAdded + } +} + +// MARK: - reading +extension InputInfo.Status { + public init?(identifier: String) { + switch identifier { + case "": self = .upToDate + case "!dirty": self = .needsCascadingBuild + case "!private": self = .needsNonCascadingBuild + default: return nil + } + assert(self.identifier == identifier, "Ensure reversibility") + } + + init(tag: String) { + // The Yams tag can be other values if there is no tag in the file + self = Self(identifier: tag) ?? Self(identifier: "")! + } +} - /// The identifier is used for the tag in the value of the input in the InputInfoMap - var identifier: String { +// MARK: - writing +extension InputInfo.Status { + /// The identifier is used for the tag in the value of the input in the BuildRecord + var identifier: String { switch self { case .upToDate: return "" @@ -35,30 +61,43 @@ public extension InputInfo { return "!dirty" case .needsNonCascadingBuild: return "!private" - } } - public init?(identifier: String) { - switch identifier { - case "": self = .upToDate - case "!dirty": self = .needsCascadingBuild - case "!private": self = .needsNonCascadingBuild - default: return nil - } + } + + /// Construct a status to write at the end of the compilation. + /// The status will be read for the next driver invocation and will control the scheduling of that job. + /// `upToDate` means only that the file was up to date when the build record was written. + init( wasSkipped: Bool?, jobResult: ProcessResult? ) { + if let _ = jobResult, wasSkipped == true { + fatalError("Skipped job cannot have finished") + } + let ok = wasSkipped == true || jobResult?.finishedWithoutError == true + let incrementally = wasSkipped != nil + switch (ok, incrementally) { + case (true, _): self = .upToDate + case (false, true): self = .needsNonCascadingBuild + case (false, false): self = .needsCascadingBuild } } } -/// decoding -public extension InputInfo { - init(tag: String, previousModTime: Date) throws { - self.init(status: Status(tag: tag), - previousModTime: previousModTime) +fileprivate extension ProcessResult { + var finishedWithoutError: Bool { + if case let .terminated(exitCode) = exitStatus, exitCode == 0 { + return true + } + return false } } -fileprivate extension InputInfo.Status { - init(tag: String) { - /// The Yams tag can be other values if there is no tag in the file - self = Self(identifier: tag) ?? Self(identifier: "")! +// MARK: - reading +public extension InputInfo { + init(tag: String, previousModTime: Date) { + self.init(status: Status(tag: tag), previousModTime: previousModTime) } } + +// MARK: - writing +extension InputInfo { + var tag: String { status.identifier } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Integrator.swift b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Integrator.swift new file mode 100644 index 000000000..39b60ed8a --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Integrator.swift @@ -0,0 +1,219 @@ +//===------------------ Integrator.swift ----------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import TSCBasic + +extension ModuleDependencyGraph { + + // MARK: Integrator - state & creation + + /// Integrates a \c SourceFileDependencyGraph into a \c ModuleDependencyGraph + /*@_spi(Testing)*/ public struct Integrator { + + // Shorthands + /*@_spi(Testing)*/ public typealias Graph = ModuleDependencyGraph + + /*@_spi(Testing)*/ public typealias Changes = Set + + let source: SourceFileDependencyGraph + let swiftDeps: SwiftDeps + let destination: ModuleDependencyGraph + + /// When done, changedNodes contains a set of nodes that changed as a result of this integration. + + var changedNodes = Changes() + + /// Starts with all nodes in swiftDeps. Nodes that persist will be removed. + /// After integration is complete, contains the nodes that have disappeared. + var disappearedNodes = [DependencyKey: Graph.Node]() + + init(source: SourceFileDependencyGraph, + swiftDeps: SwiftDeps, + destination: ModuleDependencyGraph) + { + self.source = source + self.swiftDeps = swiftDeps + self.destination = destination + self.disappearedNodes = destination.nodeFinder.findNodes(for: swiftDeps) + ?? [:] + } + } +} + +// MARK: - integrate a swiftDeps file +extension ModuleDependencyGraph.Integrator { + private enum IntegrateError: Error { + case notSwiftDeps + } + /// returns nil for error + static func integrate( + swiftDeps: Graph.SwiftDeps, + into destination: Graph, + input: TypedVirtualPath, + reportIncrementalDecision: ((String, TypedVirtualPath) -> Void)?, + diagnosticEngine: DiagnosticsEngine + ) -> Changes? { + guard let sfdg = try? SourceFileDependencyGraph.read( + from: swiftDeps) + else { + reportIncrementalDecision?("Could not read \(swiftDeps)", input) + return nil + } + return integrate(from: sfdg, + swiftDeps: swiftDeps, + into: destination) + } +} +// MARK: - integrate a graph + +extension ModuleDependencyGraph.Integrator { + /// Integrate a SourceFileDepGraph into the receiver. + /// Integration happens when the driver needs to read SourceFileDepGraph. + /// Returns changed nodes + /*@_spi(Testing)*/ public static func integrate( + from g: SourceFileDependencyGraph, + swiftDeps: Graph.SwiftDeps, + into destination: Graph + ) -> Changes { + var integrator = Self(source: g, + swiftDeps: swiftDeps, + destination: destination) + integrator.integrate() + + if destination.verifyDependencyGraphAfterEveryImport { + integrator.verifyAfterImporting() + } + if destination.emitDependencyDotFileAfterEveryImport { + destination.emitDotFile(g, swiftDeps) + } + return integrator.changedNodes + } + + private mutating func integrate() { + integrateEachSourceNode() + handleDisappearedNodes() + destination.ensureGraphWillRetraceDependents(of: changedNodes) + } + private mutating func integrateEachSourceNode() { + source.forEachNode { integrate(oneNode: $0) } + } + private mutating func handleDisappearedNodes() { + for (_, node) in disappearedNodes { + changedNodes.insert(node) + destination.nodeFinder.remove(node) + } + } +} +// MARK: - integrate one node +extension ModuleDependencyGraph.Integrator { + private mutating func integrate( + oneNode integrand: SourceFileDependencyGraph.Node) + { + guard integrand.isProvides else { + // depends are captured by recordWhatIsDependedUpon below + return + } + + let integratedNode = destination.nodeFinder.findNodes(for: integrand.key) + .flatMap { + integrateWithNodeHere(integrand, $0) ?? + integrateWithExpat( integrand, $0) + } + ?? integrateWithNewNode(integrand) + + recordDefsForThisUse(integrand, integratedNode) + } + + /// If there is already a node in the graph for this swiftDeps, merge the integrand into that, + /// and return the merged node. Remember that the merged node has changed if it has. + private mutating func integrateWithNodeHere( + _ integrand: SourceFileDependencyGraph.Node, + _ nodesMatchingKey: [Graph.SwiftDeps?: Graph.Node] + ) -> Graph.Node? { + guard let matchHere = nodesMatchingKey[swiftDeps] else { + return nil + } + assert(matchHere.swiftDeps == swiftDeps) + // Node was and still is. Do not remove it. + disappearedNodes.removeValue(forKey: matchHere.dependencyKey) + if matchHere.fingerprint != integrand.fingerprint { + changedNodes.insert(matchHere) + } + return matchHere + } + + /// If there is an expat node with this key, replace it with a ndoe for this swiftDeps + /// and return the replacement. Remember that the replace has changed. + private mutating func integrateWithExpat( + _ integrand: SourceFileDependencyGraph.Node, + _ nodesMatchingKey: [Graph.SwiftDeps?: Graph.Node] + ) -> Graph.Node? { + guard let expat = nodesMatchingKey[nil] else { + return nil + } + assert(nodesMatchingKey.count == 1, + "If an expat exists, then must not be any matches in other files") + let integratedNode = destination.nodeFinder + .replace(expat, + newSwiftDeps: swiftDeps, + newFingerprint: integrand.fingerprint) + changedNodes.insert(integratedNode) + return integratedNode + } + + /// Integrate by creating a whole new node. Remember that it has changed. + private mutating func integrateWithNewNode( + _ integrand: SourceFileDependencyGraph.Node + ) -> Graph.Node { + precondition(integrand.isProvides, "Dependencies are arcs in the module graph") + let newNode = Graph.Node( + key: integrand.key, + fingerprint: integrand.fingerprint, + swiftDeps: swiftDeps) + let oldNode = destination.nodeFinder.insert(newNode) + assert(oldNode == nil, "Should be new!") + changedNodes.insert(newNode) + return newNode + } + + /// Find the keys of nodes used by this node, and record the def-use links. + /// Also see if any of those keys are external dependencies, and if such is a new dependency, + /// record the external dependency, and record the node as changed. + private mutating func recordDefsForThisUse( + _ sourceFileUseNode: SourceFileDependencyGraph.Node, + _ moduleUseNode: Graph.Node + ) { + source.forEachDefDependedUpon(by: sourceFileUseNode) { + def in + let isNewUse = destination.nodeFinder.record(def: def.key, + use: moduleUseNode) + if let externalDependency = def.key.designator.externalDependency, + isNewUse { + destination.externalDependencies.insert(externalDependency) + changedNodes.insert(moduleUseNode) + } + } + } +} + +// MARK: - verification +extension ModuleDependencyGraph.Integrator { + @discardableResult + func verifyAfterImporting() -> Bool { + guard let nodesInFile = destination.nodeFinder.findNodes(for: swiftDeps), + !nodesInFile.isEmpty + else { + fatalError("Just imported \(swiftDeps), should have nodes") + } + return destination.verifyGraph() + } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Node.swift b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Node.swift new file mode 100644 index 000000000..70df7639c --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Node.swift @@ -0,0 +1,117 @@ +//===------------------------ Node.swift ----------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import Foundation +import TSCBasic + +// MARK: - ModuleDependencyGraph.Node +extension ModuleDependencyGraph { + + /// A node in the per-module (i.e. the driver) dependency graph + /// Each node represents a `Decl` from the frontend. + /// If a file references a `Decl` we haven't seen yet, the node's `swiftDeps` will be nil, otherwise + /// it will hold the name of the swiftdeps file from which the node was read. + /// A dependency is represented by an arc, in the `usesByDefs` map. + /// (Cargo-culted and modified from the legacy driver.) + /// + /// Use a class, not a struct because otherwise it would be duplicated for each thing it uses + + /*@_spi(Testing)*/ public final class Node { + + /*@_spi(Testing)*/ public typealias Graph = ModuleDependencyGraph + + /// Def->use arcs go by DependencyKey. There may be >1 node for a given key. + let dependencyKey: DependencyKey + + /// The frontend records in the fingerprint, all of the information about an + /// entity, such that any uses need be rebuilt only if the fingerprint + /// changes. + /// When the driver reloads a dependency graph (after a frontend job has run), + /// it can use the fingerprint to determine if the entity has changed and thus + /// if uses need to be recompiled. + /// + /// However, at present, the frontend does not record this information for + /// every Decl; it only records it for the source-file-as-a-whole in the + /// interface hash. The inteface hash is a product of all the tokens that are + /// not inside of function bodies. Thus, if there is no fingerprint, when the + /// frontend creates an interface node, + /// it adds a dependency to it from the implementation source file node (which + /// has the intefaceHash as its fingerprint). + let fingerprint: String? + + + /// The swiftDeps file that holds this entity iff the entities .swiftdeps is known. + /// If more than one source file has the same DependencyKey, then there + /// will be one node for each in the driver, distinguished by this field. + /// Nodes can move from file to file when the driver reads the result of a + /// compilation. + /// Nil represents a node with no known residance + let swiftDeps: SwiftDeps? + var isExpat: Bool { swiftDeps == nil } + + /// This swiftDeps is the file where the swiftDeps was read, not necessarily anything in the + /// SourceFileDependencyGraph or the DependencyKeys + init(key: DependencyKey, fingerprint: String?, swiftDeps: SwiftDeps?) { + self.dependencyKey = key + self.fingerprint = fingerprint + self.swiftDeps = swiftDeps + } + } +} +// MARK: - comparing, hashing +extension ModuleDependencyGraph.Node: Equatable, Hashable { + public static func == (lhs: Graph.Node, rhs: Graph.Node ) -> Bool { + lhs.dependencyKey == rhs.dependencyKey && lhs.fingerprint == rhs.fingerprint + && lhs.swiftDeps == rhs.swiftDeps + } + + public func hash(into hasher: inout Hasher) { + hasher.combine(dependencyKey) + hasher.combine(fingerprint) + hasher.combine(swiftDeps) + } +} + +extension ModuleDependencyGraph.Node: Comparable { + public static func < (lhs: ModuleDependencyGraph.Node, rhs: ModuleDependencyGraph.Node) -> Bool { + func lt (_ a: T?, _ b: T?) -> Bool { + switch (a, b) { + case let (x?, y?): return x < y + case (nil, nil): return false + case (nil, _?): return true + case (_?, nil): return false + } + } + return lhs.dependencyKey != rhs.dependencyKey ? lhs.dependencyKey < rhs.dependencyKey : + lhs.swiftDeps != rhs.swiftDeps ? lt(lhs.swiftDeps, rhs.swiftDeps) + : lt(lhs.fingerprint, rhs.fingerprint) + } +} + + +extension ModuleDependencyGraph.Node: CustomStringConvertible { + public var description: String { + "\(dependencyKey) \( swiftDeps.map { "in \($0.description)" } ?? "" )" + } +} + +extension ModuleDependencyGraph.Node { + public func verify() { + verifyExpatsHaveNoFingerprints() + dependencyKey.verify() + } + + public func verifyExpatsHaveNoFingerprints() { + if isExpat && fingerprint != nil { + fatalError(#function) + } + } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/NodeFinder.swift b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/NodeFinder.swift new file mode 100644 index 000000000..ca98fd205 --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/NodeFinder.swift @@ -0,0 +1,230 @@ +//===-------------------- NodeFinder.swift --------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +extension ModuleDependencyGraph { + + // Shorthands + + /// The core information for the ModuleDependencyGraph + /// Isolate in a sub-structure in order to faciliate invariant maintainance + struct NodeFinder { + typealias Graph = ModuleDependencyGraph + + /// Maps swiftDeps files and DependencyKeys to Nodes + fileprivate typealias NodeMap = TwoDMap + fileprivate var nodeMap = NodeMap() + + /// Since dependency keys use baseNames, they are coarser than individual + /// decls. So two decls might map to the same key. Given a use, which is + /// denoted by a node, the code needs to find the files to recompile. So, the + /// key indexes into the nodeMap, and that yields a submap of nodes keyed by + /// file. The set of keys in the submap are the files that must be recompiled + /// for the use. + /// (In a given file, only one node exists with a given key, but in the future + /// that would need to change if/when we can recompile a smaller unit than a + /// source file.) + + /// Tracks def-use relationships by DependencyKey. + private(set)var usesByDef = Multidictionary() + } +} +// MARK: - finding +extension ModuleDependencyGraph.NodeFinder { + func findFileInterfaceNode(forMock swiftDeps: ModuleDependencyGraph.SwiftDeps + ) -> Graph.Node? { + let fileKey = DependencyKey(fileKeyForMockSwiftDeps: swiftDeps) + return findNode((swiftDeps, fileKey)) + } + func findNode(_ mapKey: (Graph.SwiftDeps?, DependencyKey)) -> Graph.Node? { + nodeMap[mapKey] + } + func findCorrespondingImplementation(of n: Graph.Node) -> Graph.Node? { + n.dependencyKey.correspondingImplementation + .flatMap {findNode((n.swiftDeps, $0))} + } + + func findNodes(for swiftDeps: Graph.SwiftDeps?) -> [DependencyKey: Graph.Node]? { + nodeMap[swiftDeps] + } + func findNodes(for key: DependencyKey) -> [Graph.SwiftDeps?: Graph.Node]? { + nodeMap[key] + } + + func forEachUse(of def: Graph.Node, _ fn: (Graph.Node, Graph.SwiftDeps) -> Void) { + func fnVerifyingSwiftDeps(_ use: Graph.Node) { + fn(use, useMustHaveSwiftDeps(use)) + } + usesByDef[def.dependencyKey].map { + $0.values.forEach(fnVerifyingSwiftDeps) + } + // Add in implicit interface->implementation dependency + findCorrespondingImplementation(of: def) + .map(fnVerifyingSwiftDeps) + } + + func forEachUseInOrder(of def: Graph.Node, _ fn: (Graph.Node, Graph.SwiftDeps) -> Void) { + var uses = [(Graph.Node, Graph.SwiftDeps)]() + forEachUse(of: def) { + uses.append(($0, $1)) + } + uses.sorted {$0.0 < $1.0} .forEach { fn($0.0, $0.1) } + } + + func mappings(of n: Graph.Node) -> [(Graph.SwiftDeps?, DependencyKey)] + { + nodeMap.compactMap { + k, _ in + k.0 == n.swiftDeps && k.1 == n.dependencyKey + ? k + : nil + } + } + + func defsUsing(_ n: Graph.Node) -> [DependencyKey] { + usesByDef.keysContainingValue(n) + } +} + +fileprivate extension ModuleDependencyGraph.Node { + var mapKey: (Graph.SwiftDeps?, DependencyKey) { + return (swiftDeps, dependencyKey) + } +} + +// MARK: - inserting + +extension ModuleDependencyGraph.NodeFinder { + + /// Add `node` to the structure, return the old node if any at those coordinates. + @discardableResult + mutating func insert(_ n: Graph.Node) -> Graph.Node? { + nodeMap.updateValue(n, forKey: n.mapKey) + } + + /// record def-use, return if is new use + mutating func record(def: DependencyKey, use: Graph.Node) -> Bool { + verifyUseIsOK(use) + return usesByDef.addValue(use, forKey: def) + } +} + +// MARK: - removing +extension ModuleDependencyGraph.NodeFinder { + mutating func remove(_ nodeToErase: Graph.Node) { + // uses first preserves invariant that every used node is in nodeMap + removeUsings(of: nodeToErase) + removeMapping(of: nodeToErase) + } + + private mutating func removeUsings(of nodeToNotUse: Graph.Node) { + usesByDef.removeValue(nodeToNotUse) + assert(defsUsing(nodeToNotUse).isEmpty) + } + + private mutating func removeMapping(of nodeToNotMap: Graph.Node) { + let old = nodeMap.removeValue(forKey: nodeToNotMap.mapKey) + assert(old == nodeToNotMap, "Should have been there") + assert(mappings(of: nodeToNotMap).isEmpty) + } +} + +// MARK: - moving +extension ModuleDependencyGraph.NodeFinder { + /// When integrating a SourceFileDepGraph, there might be a node representing + /// a Decl that had previously been read as an expat, that is a node + /// representing a Decl in no known file (to that point). (Recall the the + /// Frontend processes name lookups as dependencies, but does not record in + /// which file the name was found.) In such a case, it is necessary to move + /// the node to the proper collection. + /// + /// Now that nodes are immutable, this function needs to replace the node + mutating func replace(_ original: Graph.Node, + newSwiftDeps: Graph.SwiftDeps, + newFingerprint: String? + ) -> Graph.Node { + let replacement = Graph.Node(key: original.dependencyKey, + fingerprint: newFingerprint, + swiftDeps: newSwiftDeps) + usesByDef.replace(original, with: replacement, forKey: original.dependencyKey) + nodeMap.removeValue(forKey: original.mapKey) + nodeMap.updateValue(replacement, forKey: replacement.mapKey) + return replacement + } +} + +// MARK: - asserting & verifying +extension ModuleDependencyGraph.NodeFinder { + func verify() -> Bool { + verifyNodeMap() + verifyUsesByDef() + return true + } + + private func verifyNodeMap() { + var nodes = [Set(), Set()] + nodeMap.verify { + _, v, submapIndex in + if let prev = nodes[submapIndex].update(with: v) { + fatalError("\(v) is also in nodeMap at \(prev), submap: \(submapIndex)") + } + v.verify() + } + } + + private func verifyUsesByDef() { + usesByDef.forEach { + def, use in + // def may have disappeared from graph, nothing to do + verifyUseIsOK(use) + } + } + + private func useMustHaveSwiftDeps(_ n: Graph.Node) -> Graph.SwiftDeps { + assert(verifyUseIsOK(n)) + return n.swiftDeps! + } + + @discardableResult + private func verifyUseIsOK(_ n: Graph.Node) -> Bool { + verifyUsedIsNotExpat(n) + verifyNodeIsMapped(n) + return true + } + + private func verifyNodeIsMapped(_ n: Graph.Node) { + if findNode(n.mapKey) == nil { + fatalError("\(n) should be mapped") + } + } + + @discardableResult + private func verifyUsedIsNotExpat(_ use: Graph.Node) -> Bool { + guard use.isExpat else { return true } + fatalError("An expat is not defined anywhere and thus cannot be used") + } +} +// MARK: - key helpers + +fileprivate extension DependencyKey { + init(fileKeyForMockSwiftDeps swiftDeps: ModuleDependencyGraph.SwiftDeps) { + self.init(aspect: .interface, + designator: + .sourceFileProvide(name: swiftDeps.sourceFileProvidesNameForMocking) + ) + } +} +fileprivate extension ModuleDependencyGraph.SwiftDeps { + var sourceFileProvidesNameForMocking: String { + // Only when mocking are these two guaranteed to be the same + file.name + } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/SwiftDeps.swift b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/SwiftDeps.swift new file mode 100644 index 000000000..c028fae09 --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/SwiftDeps.swift @@ -0,0 +1,54 @@ +//===------------------------ Node.swift ----------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import Foundation +import TSCBasic + +// MARK: - SwiftDeps +extension ModuleDependencyGraph { + /*@_spi(Testing)*/ public struct SwiftDeps: Hashable, CustomStringConvertible { + + let file: VirtualPath + + init?(_ typedFile: TypedVirtualPath) { + guard typedFile.type == .swiftDeps else { return nil } + self.init(typedFile.file) + } + init(_ file: VirtualPath) { + self.file = file + } + /*@_spi(Testing)*/ public init(mock i: Int) { + self.file = try! VirtualPath(path: String(i)) + } + /*@_spi(Testing)*/ public var mockID: Int { + Int(file.name)! + } + public var description: String { + file.description + } + } +} + +// MARK: - testing +extension ModuleDependencyGraph.SwiftDeps { + /*@_spi(Testing)*/ public var sourceFileProvideNameForMockSwiftDeps: String { + file.name + } + /*@_spi(Testing)*/ public var interfaceHashForMockSwiftDeps: String { + file.name + } +} +// MARK: - comparing +extension ModuleDependencyGraph.SwiftDeps: Comparable { + public static func < (lhs: Self, rhs: Self) -> Bool { + lhs.file.name < rhs.file.name + } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Tracer.swift b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Tracer.swift new file mode 100644 index 000000000..4cdf18d35 --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph Parts/Tracer.swift @@ -0,0 +1,134 @@ +//===----------------------------- Tracer.swift ---------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import Foundation +import TSCBasic + +extension ModuleDependencyGraph { + +/// Trace dependencies through the graph + struct Tracer { + typealias Graph = ModuleDependencyGraph + + let startingPoints: [Node] + let graph: ModuleDependencyGraph + + private(set) var tracedUses: [Node] = [] + + /// Record the paths taking so that -driver-show-incremental can explain why things are recompiled + /// If tracing dependencies, holds a vector used to hold the current path + /// def - use/def - use/def - ... + private var currentPathIfTracing: [Node]? + + private let diagnosticEngine: DiagnosticsEngine + } +} + +// MARK:- Tracing +extension ModuleDependencyGraph.Tracer { + + /// Find all uses of `defs` that have not already been traced. + /// (If already traced, jobs have already been scheduled.) + static func findPreviouslyUntracedUsesOf ( + defs: Nodes, + in graph: ModuleDependencyGraph, + diagnosticEngine: DiagnosticsEngine + ) -> Self + where Nodes.Element == ModuleDependencyGraph.Node + { + var tracer = Self(findingUsesOf: defs, + in: graph, + diagnosticEngine: diagnosticEngine) + tracer.findPreviouslyUntracedDependents() + return tracer + } + + private init(findingUsesOf defs: Nodes, + in graph: ModuleDependencyGraph, + diagnosticEngine: DiagnosticsEngine) + where Nodes.Element == ModuleDependencyGraph.Node + { + self.graph = graph + // Sort so "Tracing" diagnostics are deterministically ordered + self.startingPoints = defs.sorted() + self.currentPathIfTracing = graph.reportIncrementalDecision != nil ? [] : nil + self.diagnosticEngine = diagnosticEngine + } + + private mutating func findPreviouslyUntracedDependents() { + for n in startingPoints { + findNextPreviouslyUntracedDependent(of: n) + } + } + + private mutating func findNextPreviouslyUntracedDependent( + of definition: ModuleDependencyGraph.Node + ) { + guard graph.isUntraced(definition) else { return } + graph.amTracing(definition) + + tracedUses.append(definition) + + // If this node is merely used, but not defined anywhere, nothing else + // can possibly depend upon it. + if definition.isExpat { return } + + let pathLengthAfterArrival = traceArrival(at: definition); + + // If this use also provides something, follow it + graph.nodeFinder.forEachUseInOrder(of: definition) { use, _ in + findNextPreviouslyUntracedDependent(of: use) + } + traceDeparture(pathLengthAfterArrival); + } + + + + private mutating func traceArrival(at visitedNode: ModuleDependencyGraph.Node + ) -> Int { + guard var currentPath = currentPathIfTracing else { + return 0 + } + currentPath.append(visitedNode) + currentPathIfTracing = currentPath + + printPath(currentPath) + + return currentPath.count + } + + + private mutating func traceDeparture(_ pathLengthAfterArrival: Int) { + guard var currentPath = currentPathIfTracing else { return } + assert(pathLengthAfterArrival == currentPath.count, + "Path must be maintained throughout recursive visits.") + currentPath.removeLast() + currentPathIfTracing = currentPath + } + + + private func printPath(_ path: [Graph.Node]) { + guard path.first?.swiftDeps != path.last?.swiftDeps else {return} + graph.reportIncrementalDecision?( + [ + "Traced:", + path + .compactMap { node in + node.swiftDeps + .flatMap {graph.sourceSwiftDepsMap[$0] } + .map { "\(node.dependencyKey) from: \($0.file.basename)"} + } + .joined(separator: " -> ") + ].joined(separator: " "), + nil + ) + } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph.swift b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph.swift new file mode 100644 index 000000000..46c130b33 --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/ModuleDependencyGraph.swift @@ -0,0 +1,264 @@ +//===------- ModuleDependencyGraph.swift ----------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import Foundation +import TSCBasic +import SwiftOptions + + +// MARK: - ModuleDependencyGraph + +/*@_spi(Testing)*/ public final class ModuleDependencyGraph { + + var nodeFinder = NodeFinder() + + /// When integrating a change, want to find untraced nodes so we can kick off jobs that have not been + /// kicked off yet + private var tracedNodes = Set() + + private(set) var sourceSwiftDepsMap = BidirectionalMap() + + // Supports requests from the driver to getExternalDependencies. + public internal(set) var externalDependencies = Set() + + let verifyDependencyGraphAfterEveryImport: Bool + let emitDependencyDotFileAfterEveryImport: Bool + let reportIncrementalDecision: ((String, TypedVirtualPath?) -> Void)? + + private let diagnosticEngine: DiagnosticsEngine + + public init( + diagnosticEngine: DiagnosticsEngine, + reportIncrementalDecision: ((String, TypedVirtualPath?) -> Void)?, + emitDependencyDotFileAfterEveryImport: Bool, + verifyDependencyGraphAfterEveryImport: Bool) + { + self.verifyDependencyGraphAfterEveryImport = verifyDependencyGraphAfterEveryImport + self.emitDependencyDotFileAfterEveryImport = emitDependencyDotFileAfterEveryImport + self.reportIncrementalDecision = reportIncrementalDecision + self.diagnosticEngine = diagnosticEngine + } +} +// MARK: - initial build only +extension ModuleDependencyGraph { + /// Builds a graph + /// Returns nil if some input has no place to put a swiftdeps file + /// Returns a list of inputs whose swiftdeps files could not be read + static func buildInitialGraph( + diagnosticEngine: DiagnosticsEngine, + inputs: Inputs, + previousInputs: Set, + outputFileMap: OutputFileMap?, + parsedOptions: inout ParsedOptions, + remarkDisabled: (String) -> Diagnostic.Message, + reportIncrementalDecision: ((String, TypedVirtualPath?) -> Void)? + ) -> (ModuleDependencyGraph, [(TypedVirtualPath, VirtualPath)])? + where Inputs.Element == TypedVirtualPath + { + let emitOpt = Option.driverEmitFineGrainedDependencyDotFileAfterEveryImport + let veriOpt = Option.driverVerifyFineGrainedDependencyGraphAfterEveryImport + let graph = Self ( + diagnosticEngine: diagnosticEngine, + reportIncrementalDecision: reportIncrementalDecision, + emitDependencyDotFileAfterEveryImport: parsedOptions.contains(emitOpt), + verifyDependencyGraphAfterEveryImport: parsedOptions.contains(veriOpt)) + + let inputsAndSwiftdeps = inputs.map {input in + (input, outputFileMap?.existingOutput( inputFile: input.file, + outputType: .swiftDeps) + ) + } + for isd in inputsAndSwiftdeps where isd.1 == nil { + diagnosticEngine.emit( + remarkDisabled("\(isd.0.file.basename) has no swiftDeps file") + ) + return nil + } + let previousInputsWithMalformedSwiftDeps = inputsAndSwiftdeps.compactMap { + input, swiftDepsFile -> (TypedVirtualPath, VirtualPath)? in + guard let swiftDepsFile = swiftDepsFile + else { + return nil + } + let swiftDeps = SwiftDeps(swiftDepsFile) + graph.sourceSwiftDepsMap[input] = swiftDeps + guard previousInputs.contains(input.file) + else { + // do not try to read swiftdeps of a new input + return nil + } + let changes = Integrator.integrate(swiftDeps: swiftDeps, + into: graph, + input: input, + reportIncrementalDecision: reportIncrementalDecision, + diagnosticEngine: diagnosticEngine) + return changes == nil ? (input, swiftDepsFile) : nil + } + return (graph, previousInputsWithMalformedSwiftDeps) + } +} +// MARK: - Scheduling the first wave +extension ModuleDependencyGraph { + /// Find all the sources that depend on `sourceFile`. For some source files, these will be + /// speculatively scheduled in the first wave. + func findDependentSourceFiles( + of sourceFile: TypedVirtualPath, + _ reportIncrementalDecision: ((String, TypedVirtualPath?) -> Void)? + ) -> [TypedVirtualPath] { + var allSwiftDepsToRecompile = Set() + + let swiftDeps = sourceSwiftDepsMap[sourceFile] + + for swiftDepsToRecompile in + findSwiftDepsToRecompileWhenWholeSwiftDepsChanges( swiftDeps ) { + if swiftDepsToRecompile != swiftDeps { + allSwiftDepsToRecompile.insert(swiftDepsToRecompile) + } + } + return allSwiftDepsToRecompile.map { + let dependentSource = sourceSwiftDepsMap[$0] + reportIncrementalDecision?( + "Found dependent of \(sourceFile.file.basename):", dependentSource) + return dependentSource + } + } + + /// Find all the swiftDeps files that depend on `swiftDeps`. + /// Really private, except for testing. + /*@_spi(Testing)*/ public func findSwiftDepsToRecompileWhenWholeSwiftDepsChanges( + _ swiftDeps: SwiftDeps + ) -> Set { + let nodes = nodeFinder.findNodes(for: swiftDeps) ?? [:] + /// Tests expect this to be reflexive + return findSwiftDepsToRecompileWhenNodesChange(nodes.values) + } +} +// MARK: - Scheduling the 2nd wave +extension ModuleDependencyGraph { + /// After `source` has been compiled, figure out what other source files need compiling. + /// Used to schedule the 2nd wave. + /// Return nil in case of an error. + func findSourcesToCompileAfterCompiling( + _ source: TypedVirtualPath + ) -> [TypedVirtualPath]? { + findSourcesToCompileAfterIntegrating( + input: source, + swiftDeps: sourceSwiftDepsMap[source], + reportIncrementalDecision: reportIncrementalDecision + ) + } + + /// After a compile job has finished, read its swiftDeps file and return the source files needing + /// recompilation. + /// Return nil in case of an error. + private func findSourcesToCompileAfterIntegrating( + input: TypedVirtualPath, + swiftDeps: SwiftDeps, + reportIncrementalDecision: ((String, TypedVirtualPath) -> Void)? + ) -> [TypedVirtualPath]? { + Integrator.integrate(swiftDeps: swiftDeps, + into: self, + input: input, + reportIncrementalDecision: reportIncrementalDecision, + diagnosticEngine: diagnosticEngine) + .map { + findSwiftDepsToRecompileWhenNodesChange($0) + .subtracting([swiftDeps]) + .map {sourceSwiftDepsMap[$0]} + } + } +} + +// MARK: - Scheduling either wave +extension ModuleDependencyGraph { + /// Find all the swiftDeps affected when the nodes change. + /*@_spi(Testing)*/ public func findSwiftDepsToRecompileWhenNodesChange( + _ nodes: Nodes + ) -> Set + where Nodes.Element == Node + { + let affectedNodes = Tracer.findPreviouslyUntracedUsesOf( + defs: nodes, + in: self, + diagnosticEngine: diagnosticEngine) + .tracedUses + return Set(affectedNodes.compactMap {$0.swiftDeps}) + } + + /*@_spi(Testing)*/ public func forEachUntracedSwiftDepsDirectlyDependent( + on externalSwiftDeps: ExternalDependency, + _ fn: (SwiftDeps) -> Void + ) { + // These nodes will depend on the *interface* of the external Decl. + let key = DependencyKey(interfaceFor: externalSwiftDeps) + let node = Node(key: key, fingerprint: nil, swiftDeps: nil) + nodeFinder.forEachUseInOrder(of: node) { use, useSwiftDeps in + if isUntraced(use) { + fn(useSwiftDeps) + } + } + } +} +fileprivate extension DependencyKey { + init(interfaceFor dep: ExternalDependency ) { + self.init(aspect: .interface, designator: .externalDepend(dep)) + } +} +// MARK: - tracking traced nodes +extension ModuleDependencyGraph { + + func isUntraced(_ n: Node) -> Bool { + !isTraced(n) + } + func isTraced(_ n: Node) -> Bool { + tracedNodes.contains(n) + } + func amTracing(_ n: Node) { + tracedNodes.insert(n) + } + func ensureGraphWillRetraceDependents(of nodes: Nodes) + where Nodes.Element == Node + { + nodes.forEach { tracedNodes.remove($0) } + } +} + +// MARK: - utilities for unit testing +extension ModuleDependencyGraph { + /// Testing only + /*@_spi(Testing)*/ public func haveAnyNodesBeenTraversed(inMock i: Int) -> Bool { + let swiftDeps = SwiftDeps(mock: i) + // optimization + if let fileNode = nodeFinder.findFileInterfaceNode(forMock: swiftDeps), + isTraced(fileNode) { + return true + } + if let nodes = nodeFinder.findNodes(for: swiftDeps)?.values, + nodes.contains(where: isTraced) { + return true + } + return false + } +} +// MARK: - verification +extension ModuleDependencyGraph { + @discardableResult + func verifyGraph() -> Bool { + nodeFinder.verify() + } +} +// MARK: - debugging +extension ModuleDependencyGraph { + func emitDotFile(_ g: SourceFileDependencyGraph, _ swiftDeps: SwiftDeps) { + // TODO: Incremental emitDotFIle + fatalError("unimplmemented, writing dot file of dependency graph") + } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/Multidictionary.swift b/Sources/SwiftDriver/Incremental Compilation/Multidictionary.swift new file mode 100644 index 000000000..ff9e5ae01 --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/Multidictionary.swift @@ -0,0 +1,100 @@ +//===------------------------- Multidictionary.swift ----------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// Like a Dictionary, but can have >1 value per key (i.e., a multimap) +struct Multidictionary: Collection { + public typealias OuterDict = [Key: Set] + public typealias InnerSet = Set + private var outerDict = OuterDict() + + public typealias Element = (key: Key, value: Value) + + public enum Index: Comparable { + case end + case notEnd(OuterDict.Index, InnerSet.Index) + } + + public var startIndex: Index { + outerDict.first.map {Index.notEnd(outerDict.startIndex, $0.value.startIndex)} ?? .end + } + public var endIndex: Index {.end} + + public func index(after i: Index) -> Index { + switch i { + case .end: fatalError() + case let .notEnd(outerIndex, innerIndex): + let innerSet = outerDict[outerIndex].value + let nextInner = innerSet.index(after: innerIndex) + if nextInner < innerSet.endIndex { + return .notEnd(outerIndex, nextInner) + } + let nextOuter = outerDict.index(after: outerIndex) + if nextOuter < outerDict.endIndex { + return .notEnd(nextOuter, outerDict[nextOuter].value.startIndex) + } + return .end + } + } + + public subscript(position: Index) -> (key: Key, value: Value) { + switch position { + case .end: fatalError() + case let .notEnd(outerIndex, innerIndex): + let (key, vals) = outerDict[outerIndex] + return (key: key, value: vals[innerIndex]) + } + } + + public subscript(key: Key) -> (key: Key, values: Set)? { + outerDict[key].map { (key: key, values: $0) } + } + + public func keysContainingValue(_ v: Value) -> [Key] { + outerDict.compactMap { (k, vs) in vs.contains(v) ? k : nil } + } + + /// Returns true if inserted + public mutating func addValue(_ v: Value, forKey key: Key) -> Bool { + if var inner = outerDict[key] { + let old = inner.insert(v).inserted + outerDict[key] = inner + return old + } + outerDict[key] = Set([v]) + return true + } + + public mutating func removeValue(_ v: Value) { + let changedPairs = outerDict.compactMap { kv -> (key: Key, value: Set?)? in + var vals = kv.value + guard vals.contains(v) else { return nil} + vals.remove(v) + return (key: kv.key, value: (vals.isEmpty ? nil : vals)) + } + changedPairs.forEach { + outerDict[$0.key] = $0.value + } + } + + @discardableResult + public mutating func replace(_ original: Value, + with replacement: Value, + forKey key: Key) + -> Bool + { + guard var vals = outerDict[key], + let _ = vals.remove(original) else { return false } + vals.insert(replacement) + outerDict.updateValue(vals, forKey: key) + return true + } +} diff --git a/Sources/SwiftDriver/Incremental Compilation/SourceFileDependencyGraph.swift b/Sources/SwiftDriver/Incremental Compilation/SourceFileDependencyGraph.swift index b01d12796..4130571ba 100644 --- a/Sources/SwiftDriver/Incremental Compilation/SourceFileDependencyGraph.swift +++ b/Sources/SwiftDriver/Incremental Compilation/SourceFileDependencyGraph.swift @@ -10,31 +10,32 @@ // //===----------------------------------------------------------------------===// import Foundation +import TSCUtility -public struct SourceFileDependencyGraph { +/*@_spi(Testing)*/ public struct SourceFileDependencyGraph { public static let sourceFileProvidesInterfaceSequenceNumber: Int = 0 public static let sourceFileProvidesImplementationSequenceNumber: Int = 1 - + public var majorVersion: UInt64 public var minorVersion: UInt64 public var compilerVersionString: String private var allNodes: [Node] - + public var sourceFileNodePair: (interface: Node, implementation: Node) { (interface: allNodes[SourceFileDependencyGraph.sourceFileProvidesInterfaceSequenceNumber], implementation: allNodes[SourceFileDependencyGraph.sourceFileProvidesImplementationSequenceNumber]) } - + public func forEachNode(_ doIt: (Node) -> Void) { allNodes.forEach(doIt) } - + public func forEachDefDependedUpon(by node: Node, _ doIt: (Node) -> Void) { for sequenceNumber in node.defsIDependUpon { doIt(allNodes[sequenceNumber]) } } - + public func forEachArc(_ doIt: (Node, Node) -> Void) { forEachNode { useNode in forEachDefDependedUpon(by: useNode) { defNode in @@ -42,7 +43,7 @@ public struct SourceFileDependencyGraph { } } } - + @discardableResult public func verify() -> Bool { assert(Array(allNodes.indices) == allNodes.map { $0.sequenceNumber }) forEachNode { @@ -52,38 +53,6 @@ public struct SourceFileDependencyGraph { } } -public enum DeclAspect: UInt64 { - case interface, implementation -} - -public struct DependencyKey { - public enum Kind: UInt64 { - case topLevel - case nominal - case potentialMember - case member - case dynamicLookup - case externalDepend - case sourceFileProvide - } - - public var kind: Kind - public var aspect: DeclAspect - public var context: String - public var name: String - - public func verify() { - assert(kind != .externalDepend || aspect == .interface, "All external dependencies must be interfaces.") - switch kind { - case .topLevel, .dynamicLookup, .externalDepend, .sourceFileProvide: - assert(context.isEmpty && !name.isEmpty, "Must only have a name") - case .nominal, .potentialMember: - assert(!context.isEmpty && name.isEmpty, "Must only have a context") - case .member: - assert(!context.isEmpty && !name.isEmpty, "Must have both") - } - } -} extension SourceFileDependencyGraph { public struct Node { @@ -92,11 +61,25 @@ extension SourceFileDependencyGraph { public var sequenceNumber: Int public var defsIDependUpon: [Int] public var isProvides: Bool - + + /*@_spi(Testing)*/ public init( + key: DependencyKey, + fingerprint: String?, + sequenceNumber: Int, + defsIDependUpon: [Int], + isProvides: Bool + ) { + self.key = key + self.fingerprint = fingerprint + self.sequenceNumber = sequenceNumber + self.defsIDependUpon = defsIDependUpon + self.isProvides = isProvides + } + public func verify() { key.verify() - - if key.kind == .sourceFileProvide { + + if case .sourceFileProvide = key.designator { switch key.aspect { case .interface: assert(sequenceNumber == SourceFileDependencyGraph.sourceFileProvidesInterfaceSequenceNumber) @@ -109,8 +92,6 @@ extension SourceFileDependencyGraph { } extension SourceFileDependencyGraph { - private static let recordBlockId = 8 - private enum RecordKind: UInt64 { case metadata = 1 case sourceFileDepGraphNode @@ -118,8 +99,8 @@ extension SourceFileDependencyGraph { case dependsOnDefinitionNode case identifierNode } - - private enum ReadError: Error { + + fileprivate enum ReadError: Error { case badMagic case noRecordBlock case malformedMetadataRecord @@ -130,88 +111,193 @@ extension SourceFileDependencyGraph { case malformedSourceFileDepGraphNodeRecord case unknownRecord case unexpectedSubblock + case bogusNameOrContext + case unknownKind } - public init(data: Data) throws { - // FIXME: visit blocks and records incrementally instead of reading the - // entire file up front. - let bitcode = try Bitcode(data: data) - guard bitcode.signature == .init(string: "DEPS") else { throw ReadError.badMagic } - - guard bitcode.elements.count == 1, - case .block(let recordBlock) = bitcode.elements.first, - recordBlock.id == Self.recordBlockId else { throw ReadError.noRecordBlock } - - guard case .record(let metadataRecord) = recordBlock.elements.first, - RecordKind(rawValue: metadataRecord.id) == .metadata, - metadataRecord.fields.count == 2, - case .blob(let compilerVersionBlob) = metadataRecord.payload, - let compilerVersionString = String(data: compilerVersionBlob, encoding: .utf8) - else { throw ReadError.malformedMetadataRecord } - - self.majorVersion = metadataRecord.fields[0] - self.minorVersion = metadataRecord.fields[1] - self.compilerVersionString = compilerVersionString - - var nodes: [Node] = [] - var node: Node? = nil - var identifiers: [String] = [""] // The empty string is hardcoded as identifiers[0] - var sequenceNumber = 0 - for element in recordBlock.elements.dropFirst() { - guard case .record(let record) = element else { throw ReadError.unexpectedSubblock } - guard let kind = RecordKind(rawValue: record.id) else { throw ReadError.unknownRecord } - switch kind { - case .metadata: - throw ReadError.unexpectedMetadataRecord - case .sourceFileDepGraphNode: - if let node = node { - nodes.append(node) + // FIXME: This should accept a FileSystem parameter. + static func read(from swiftDeps: ModuleDependencyGraph.SwiftDeps + ) throws -> Self { + try self.init(pathString: swiftDeps.file.name) + } + + /*@_spi(Testing)*/ public init(nodesForTesting: [Node]) { + majorVersion = 0 + minorVersion = 0 + compilerVersionString = "" + allNodes = nodesForTesting + } + + // FIXME: This should accept a FileSystem parameter. + /*@_spi(Testing)*/ public init(pathString: String) throws { + let data = try Data(contentsOf: URL(fileURLWithPath: pathString)) + try self.init(data: data) + } + + /*@_spi(Testing)*/ public init(data: Data, + fromSwiftModule extractFromSwiftModule: Bool = false) throws { + struct Visitor: BitstreamVisitor { + let extractFromSwiftModule: Bool + + init(extractFromSwiftModule: Bool) { + self.extractFromSwiftModule = extractFromSwiftModule + } + + var nodes: [Node] = [] + var majorVersion: UInt64? + var minorVersion: UInt64? + var compilerVersionString: String? + + private var node: Node? = nil + private var identifiers: [String] = [""] // The empty string is hardcoded as identifiers[0] + private var sequenceNumber = 0 + + func validate(signature: Bitcode.Signature) throws { + if extractFromSwiftModule { + guard signature == .init(value: 0x0EA89CE2) else { throw ReadError.badMagic } + } else { + guard signature == .init(string: "DEPS") else { throw ReadError.badMagic } } - guard record.fields.count == 5, - let nodeKind = DependencyKey.Kind(rawValue: record.fields[0]), - let declAspect = DeclAspect(rawValue: record.fields[1]), - record.fields[2] < identifiers.count, - record.fields[3] < identifiers.count else { - throw ReadError.malformedSourceFileDepGraphNodeRecord + } + + mutating func shouldEnterBlock(id: UInt64) throws -> Bool { + if extractFromSwiftModule { + // Enter the top-level module block, and the incremental info + // subblock, ignoring the rest of the file. + return id == /*Module block*/ 8 || id == /*Incremental record block*/ 196 + } else { + guard id == /*Incremental record block*/ 8 else { + throw ReadError.unexpectedSubblock + } + return true } - let context = identifiers[Int(record.fields[2])] - let identifier = identifiers[Int(record.fields[3])] - let isProvides = record.fields[4] != 0 - node = Node(key: .init(kind: nodeKind, - aspect: declAspect, - context: context, - name: identifier), - fingerprint: nil, - sequenceNumber: sequenceNumber, - defsIDependUpon: [], - isProvides: isProvides) - sequenceNumber += 1 - case .fingerprintNode: - guard node != nil, - record.fields.count == 0, - case .blob(let fingerprintBlob) = record.payload, - let fingerprint = String(data: fingerprintBlob, encoding: .utf8) else { - throw ReadError.malformedFingerprintRecord + } + + mutating func didExitBlock() throws { + // Finalize the current node if needed. + if let node = node { + nodes.append(node) + self.node = nil } - node?.fingerprint = fingerprint - case .dependsOnDefinitionNode: - guard node != nil, - record.fields.count == 1 else { throw ReadError.malformedDependsOnDefinitionRecord } - node?.defsIDependUpon.append(Int(record.fields[0])) - case .identifierNode: - guard record.fields.count == 0, - case .blob(let identifierBlob) = record.payload, - let identifier = String(data: identifierBlob, encoding: .utf8) else { - throw ReadError.malformedIdentifierRecord + } + + mutating func visit(record: BitcodeElement.Record) throws { + guard let kind = RecordKind(rawValue: record.id) else { throw ReadError.unknownRecord } + switch kind { + case .metadata: + // If we've already read metadata, this is an unexpected duplicate. + guard majorVersion == nil, minorVersion == nil, compilerVersionString == nil else { + throw ReadError.unexpectedMetadataRecord + } + guard record.fields.count == 2, + case .blob(let compilerVersionBlob) = record.payload, + let compilerVersionString = String(data: compilerVersionBlob, encoding: .utf8) + else { throw ReadError.malformedMetadataRecord } + + self.majorVersion = record.fields[0] + self.minorVersion = record.fields[1] + self.compilerVersionString = compilerVersionString + case .sourceFileDepGraphNode: + if let node = node { + nodes.append(node) + } + let kindCode = record.fields[0] + guard record.fields.count == 5, + let declAspect = DependencyKey.DeclAspect(record.fields[1]), + record.fields[2] < identifiers.count, + record.fields[3] < identifiers.count else { + throw ReadError.malformedSourceFileDepGraphNodeRecord + } + let context = identifiers[Int(record.fields[2])] + let identifier = identifiers[Int(record.fields[3])] + let isProvides = record.fields[4] != 0 + let designator = try DependencyKey.Designator( + kindCode: kindCode, context: context, name: identifier) + let key = DependencyKey(aspect: declAspect, designator: designator) + node = Node(key: key, + fingerprint: nil, + sequenceNumber: sequenceNumber, + defsIDependUpon: [], + isProvides: isProvides) + sequenceNumber += 1 + case .fingerprintNode: + guard node != nil, + record.fields.count == 0, + case .blob(let fingerprintBlob) = record.payload, + let fingerprint = String(data: fingerprintBlob, encoding: .utf8) else { + throw ReadError.malformedFingerprintRecord + } + node?.fingerprint = fingerprint + case .dependsOnDefinitionNode: + guard node != nil, + record.fields.count == 1 else { throw ReadError.malformedDependsOnDefinitionRecord } + node?.defsIDependUpon.append(Int(record.fields[0])) + case .identifierNode: + guard record.fields.count == 0, + case .blob(let identifierBlob) = record.payload, + let identifier = String(data: identifierBlob, encoding: .utf8) else { + throw ReadError.malformedIdentifierRecord + } + identifiers.append(identifier) } - identifiers.append(identifier) } } - if let node = node { - nodes.append(node) + var visitor = Visitor(extractFromSwiftModule: extractFromSwiftModule) + try Bitcode.read(stream: data, using: &visitor) + guard let major = visitor.majorVersion, + let minor = visitor.minorVersion, + let versionString = visitor.compilerVersionString else { + throw ReadError.malformedMetadataRecord + } + self.majorVersion = major + self.minorVersion = minor + self.compilerVersionString = versionString + self.allNodes = visitor.nodes + } +} + +fileprivate extension DependencyKey.DeclAspect { + init?(_ c: UInt64) { + switch c { + case 0: self = .interface + case 1: self = .implementation + default: return nil } + } +} - self.allNodes = nodes +fileprivate extension DependencyKey.Designator { + init(kindCode: UInt64, + context: String, + name: String) throws { + func mustBeEmpty(_ s: String) throws { + guard s.isEmpty else { throw SourceFileDependencyGraph.ReadError.bogusNameOrContext } + } + switch kindCode { + case 0: + try mustBeEmpty(context) + self = .topLevel(name: name) + case 1: + try mustBeEmpty(name) + self = .nominal(context: context) + case 2: + try mustBeEmpty(name) + self = .potentialMember(context: context) + case 3: + self = .member(context: context, name: name) + case 4: + try mustBeEmpty(context) + self = .dynamicLookup(name: name) + case 5: + try mustBeEmpty(context) + self = .externalDepend(ExternalDependency(name)) + case 6: + try mustBeEmpty(context) + self = .sourceFileProvide(name: name) + + default: throw SourceFileDependencyGraph.ReadError.unknownKind + } } } + diff --git a/Sources/SwiftDriver/Incremental Compilation/TwoDMap.swift b/Sources/SwiftDriver/Incremental Compilation/TwoDMap.swift new file mode 100644 index 000000000..c21430083 --- /dev/null +++ b/Sources/SwiftDriver/Incremental Compilation/TwoDMap.swift @@ -0,0 +1,96 @@ +//===---------------- TwoDMap.swift ---------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + + +/// A map with 2 keys that can iterate in a number of ways +public struct TwoDMap: MutableCollection { + + private var map1 = DictionaryOfDictionaries() + private var map2 = DictionaryOfDictionaries() + + public typealias Key = (Key1, Key2) + public typealias Element = (Key, Value) + public typealias Index = DictionaryOfDictionaries.Index + + + + public init() {} + + public subscript(position: Index) -> Element { + get { map1[position] } + set { + map1[newValue.0] = newValue.1 + map2[(newValue.0.1, newValue.0.0)] = newValue.1 + } + } + + public subscript(key: Key) -> Value? { + get { map1[key] } + } + + public subscript(key: Key1) -> [Key2: Value]? { + get { map1[key] } + } + + public subscript(key: Key2) -> [Key1: Value]? { + get { map2[key] } + } + + public var startIndex: Index { + map1.startIndex + } + + public var endIndex: Index { + map1.endIndex + } + + public func index(after i: Index) -> Index { + map1.index(after: i) + } + + @discardableResult + public mutating func updateValue(_ v: Value, forKey keys: (Key1, Key2)) -> Value? { + let inserted1 = map1.updateValue(v, forKey: keys ) + let inserted2 = map2.updateValue(v, forKey: (keys.1, keys.0)) + assert(inserted1 == inserted2) + return inserted1 + } + + @discardableResult + public mutating func removeValue(forKey keys: (Key1, Key2)) -> Value? { + let v1 = map1.removeValue(forKey: keys ) + let v2 = map2.removeValue(forKey: (keys.1, keys.0)) + assert(v1 == v2) + return v1 + } + + /// Verify the integrity of each map and the cross-map consistency. + /// Then call \p verifyFn for each entry found in each of the two maps, + /// passing an index so that the verifyFn knows which map is being tested. + @discardableResult + public func verify(_ fn: ((Key1, Key2), Value, Int) -> Void) -> Bool { + map1.forEach { (kv: ((Key1, Key2), Value)) in + assert(kv.1 == map2[(kv.0.1, kv.0.0)]) + fn(kv.0, kv.1, 0) + } + map2.forEach { (kv: ((Key2, Key1), Value)) in + assert(kv.1 == map1[(kv.0.1, kv.0.0)]) + fn((kv.0.1, kv.0.0), kv.1, 1) + } + return true + } + + public func compactMap(_ fn: ((Key1, Key2), Value) -> R?) -> [R] + { + map1.compactMap(fn) + } +} diff --git a/Sources/SwiftDriver/Jobs/AutolinkExtractJob.swift b/Sources/SwiftDriver/Jobs/AutolinkExtractJob.swift index f4cf206f1..2ce4558dd 100644 --- a/Sources/SwiftDriver/Jobs/AutolinkExtractJob.swift +++ b/Sources/SwiftDriver/Jobs/AutolinkExtractJob.swift @@ -11,18 +11,30 @@ //===----------------------------------------------------------------------===// import TSCBasic +// On ELF/WASM platforms there's no built in autolinking mechanism, so we +// pull the info we need from the .o files directly and pass them as an +// argument input file to the linker. +// FIXME: Also handle Cygwin and MinGW extension Driver { + /*@_spi(Testing)*/ public var isAutolinkExtractJobNeeded: Bool { + [.elf, .wasm].contains(targetTriple.objectFormat) && lto == nil + } + mutating func autolinkExtractJob(inputs: [TypedVirtualPath]) throws -> Job? { - // On ELF platforms there's no built in autolinking mechanism, so we - // pull the info we need from the .o files directly and pass them as an - // argument input file to the linker. - // FIXME: Also handle Cygwin and MinGW - guard inputs.count > 0 && targetTriple.objectFormat == .elf else { + guard let firstInput = inputs.first, isAutolinkExtractJobNeeded else { return nil } var commandLine = [Job.ArgTemplate]() - let output = VirtualPath.temporary(RelativePath("\(moduleOutputInfo.name).autolink")) + // Put output in same place as first .o, following legacy driver. + // (See `constructInvocation(const AutolinkExtractJobAction` in `UnixToolChains.cpp`.) + let outputBasename = "\(moduleOutputInfo.name).autolink" + let dir = firstInput.file.parentDirectory + // Go through a bit of extra rigmarole to keep the "./" out of the name for + // the sake of the tests. + let output: VirtualPath = dir == .temporary(RelativePath(".")) + ? .temporary(RelativePath(outputBasename)) + : dir.appending(component: outputBasename) commandLine.append(contentsOf: inputs.map { .path($0.file) }) commandLine.appendFlag(.o) @@ -34,6 +46,7 @@ extension Driver { tool: .absolute(try toolchain.getToolPath(.swiftAutolinkExtract)), commandLine: commandLine, inputs: inputs, + primaryInputs: [], outputs: [.init(file: output, type: .autolink)], supportsResponseFiles: true ) diff --git a/Sources/SwiftDriver/Jobs/BackendJob.swift b/Sources/SwiftDriver/Jobs/BackendJob.swift index a44c1d2c7..1c68b2742 100644 --- a/Sources/SwiftDriver/Jobs/BackendJob.swift +++ b/Sources/SwiftDriver/Jobs/BackendJob.swift @@ -15,7 +15,8 @@ import Foundation extension Driver { /// Form a backend job. mutating func backendJob(input: TypedVirtualPath, - allOutputs: inout [TypedVirtualPath]) throws -> Job { + addJobOutputs: ([TypedVirtualPath]) -> Void) + throws -> Job { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } var inputs = [TypedVirtualPath]() var outputs = [TypedVirtualPath]() @@ -62,7 +63,7 @@ extension Driver { outputs.append(output) } - allOutputs += outputs + addJobOutputs(outputs) return Job( moduleName: moduleOutputInfo.name, @@ -71,6 +72,7 @@ extension Driver { commandLine: commandLine, displayInputs: inputs, inputs: inputs, + primaryInputs: [], outputs: outputs, supportsResponseFiles: true ) diff --git a/Sources/SwiftDriver/Jobs/CommandLineArguments.swift b/Sources/SwiftDriver/Jobs/CommandLineArguments.swift index b560d4f8b..91f88939c 100644 --- a/Sources/SwiftDriver/Jobs/CommandLineArguments.swift +++ b/Sources/SwiftDriver/Jobs/CommandLineArguments.swift @@ -162,7 +162,7 @@ extension Array where Element == Job.ArgTemplate { } /// A shell-escaped string representation of the arguments, as they would appear on the command line. - var joinedArguments: String { + public var joinedArguments: String { return self.map { switch $0 { case .flag(let string): diff --git a/Sources/SwiftDriver/Jobs/CompileJob.swift b/Sources/SwiftDriver/Jobs/CompileJob.swift index 5842bff90..8547194ed 100644 --- a/Sources/SwiftDriver/Jobs/CompileJob.swift +++ b/Sources/SwiftDriver/Jobs/CompileJob.swift @@ -38,6 +38,8 @@ extension Driver { return TypedVirtualPath(file: baseOutputPath, type: outputType) } else if compilerOutputType?.isTextual == true { return TypedVirtualPath(file: .standardOutput, type: outputType) + } else if outputType == .swiftModule, let moduleOutput = moduleOutputInfo.output { + return TypedVirtualPath(file: moduleOutput.outputPath, type: outputType) } } @@ -52,8 +54,7 @@ extension Driver { return TypedVirtualPath(file:VirtualPath.temporary(.init(baseName.appendingFileTypeExtension(outputType))), type: outputType) } - - return TypedVirtualPath(file: .relative(.init(baseName.appendingFileTypeExtension(outputType))), type: outputType) + return TypedVirtualPath(file: useWorkingDirectory(.init(baseName.appendingFileTypeExtension(outputType))), type: outputType) } /// Is this compile job top-level @@ -64,10 +65,17 @@ extension Driver { return true case .object: return (linkerOutputType == nil) + case .llvmBitcode: + if compilerOutputType != .llvmBitcode { + // The compiler output isn't bitcode, so bitcode isn't top-level (-embed-bitcode). + return false + } else { + // When -lto is set, .bc will be used for linking. Otherwise, .bc is + // top-level output (-emit-bc) + return lto == nil + } case .swiftModule: return compilerMode.isSingleCompilation && moduleOutputInfo.output?.isTopLevel ?? false - case .llvmBitcode: - return compilerOutputType == type case .swift, .image, .dSYM, .dependencies, .autolink, .swiftDocumentation, .swiftInterface, .privateSwiftInterface, .swiftSourceInfoFile, .diagnostics, .objcHeader, .swiftDeps, .remap, .tbd, .moduleTrace, .yamlOptimizationRecord, .bitstreamOptimizationRecord, .pcm, @@ -79,6 +87,7 @@ extension Driver { /// Add the compiler inputs for a frontend compilation job, and return the /// corresponding primary set of outputs. mutating func addCompileInputs(primaryInputs: [TypedVirtualPath], + indexFilePath: TypedVirtualPath?, inputs: inout [TypedVirtualPath], inputOutputMap: inout [TypedVirtualPath: TypedVirtualPath], outputType: FileType?, @@ -107,9 +116,21 @@ extension Driver { // If we will be passing primary files via -primary-file, form a set of primary input files so // we can check more quickly. - let usesPrimaryFileInputs = compilerMode.usesPrimaryFileInputs - assert(!usesPrimaryFileInputs || !primaryInputs.isEmpty) - let primaryInputFiles = usesPrimaryFileInputs ? Set(primaryInputs) : Set() + let usesPrimaryFileInputs: Bool + let primaryInputFiles: Set + if compilerMode.usesPrimaryFileInputs { + assert(!primaryInputs.isEmpty) + usesPrimaryFileInputs = true + primaryInputFiles = Set(primaryInputs) + } else if let path = indexFilePath { + // If -index-file is used, we perform a single compile but pass the + // -index-file-path as a primary input file. + usesPrimaryFileInputs = true + primaryInputFiles = [path] + } else { + usesPrimaryFileInputs = false + primaryInputFiles = [] + } let isMultithreaded = numThreads > 0 @@ -157,8 +178,11 @@ extension Driver { } /// Form a compile job, which executes the Swift frontend to produce various outputs. - mutating func compileJob(primaryInputs: [TypedVirtualPath], outputType: FileType?, - allOutputs: inout [TypedVirtualPath]) throws -> Job { + mutating func compileJob(primaryInputs: [TypedVirtualPath], + outputType: FileType?, + addJobOutputs: ([TypedVirtualPath]) -> Void, + emitModuleTrace: Bool) + throws -> Job { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } var inputs: [TypedVirtualPath] = [] var outputs: [TypedVirtualPath] = [] @@ -167,7 +191,17 @@ extension Driver { commandLine.appendFlag("-frontend") addCompileModeOption(outputType: outputType, commandLine: &commandLine) + + let indexFilePath: TypedVirtualPath? + if let indexFileArg = parsedOptions.getLastArgument(.indexFilePath)?.asSingle { + let path = try VirtualPath(path: indexFileArg) + indexFilePath = inputFiles.first { $0.file == path } + } else { + indexFilePath = nil + } + let primaryOutputs = addCompileInputs(primaryInputs: primaryInputs, + indexFilePath: indexFilePath, inputs: &inputs, inputOutputMap: &inputOutputMap, outputType: outputType, @@ -183,9 +217,11 @@ extension Driver { try commandLine.appendLast(.saveOptimizationRecordEQ, from: &parsedOptions) try commandLine.appendLast(.saveOptimizationRecordPasses, from: &parsedOptions) - outputs += try addFrontendSupplementaryOutputArguments(commandLine: &commandLine, - primaryInputs: primaryInputs, - inputOutputMap: inputOutputMap) + outputs += try addFrontendSupplementaryOutputArguments( + commandLine: &commandLine, + primaryInputs: primaryInputs, + inputOutputMap: inputOutputMap, + includeModuleTracePath: emitModuleTrace) // Forward migrator flags. try commandLine.appendLast(.apiDiffDataFile, from: &parsedOptions) @@ -249,20 +285,30 @@ extension Driver { try commandLine.appendLast(.runtimeCompatibilityVersion, from: &parsedOptions) try commandLine.appendLast(.disableAutolinkingRuntimeCompatibilityDynamicReplacements, from: &parsedOptions) - allOutputs += outputs + addJobOutputs(outputs) - // If we're creating emit module job, order the compile jobs after that. - if shouldCreateEmitModuleJob { + // If we're prioritizing the emit module job, schedule the compile jobs + // after that. + if forceEmitModuleBeforeCompile { inputs.append(TypedVirtualPath(file: moduleOutputInfo.output!.outputPath, type: .swiftModule)) } + var displayInputs = primaryInputs + + // Bridging header is needed for compiling these .swift sources. + if let pchPath = bridgingPrecompiledHeader { + let pchInput = TypedVirtualPath(file: pchPath, type: .pch) + inputs.append(pchInput) + displayInputs.append(pchInput) + } return Job( moduleName: moduleOutputInfo.name, kind: .compile, tool: .absolute(try toolchain.getToolPath(.swiftCompiler)), commandLine: commandLine, - displayInputs: primaryInputs, + displayInputs: displayInputs, inputs: inputs, + primaryInputs: primaryInputs, outputs: outputs, supportsResponseFiles: true ) diff --git a/Sources/SwiftDriver/Jobs/DarwinToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/DarwinToolchain+LinkerSupport.swift index 513ab5c1b..4aef4cd33 100644 --- a/Sources/SwiftDriver/Jobs/DarwinToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/DarwinToolchain+LinkerSupport.swift @@ -14,11 +14,11 @@ import TSCUtility import SwiftOptions extension DarwinToolchain { - internal func findARCLiteLibPath() throws -> AbsolutePath? { + internal func findXcodeClangLibPath(_ additionalPath: String) throws -> AbsolutePath? { let path = try getToolPath(.swiftCompiler) .parentDirectory // 'swift' .parentDirectory // 'bin' - .appending(components: "lib", "arc") + .appending(components: "lib", additionalPath) if fileSystem.exists(path) { return path } @@ -28,15 +28,26 @@ extension DarwinToolchain { return clangPath .parentDirectory // 'clang' .parentDirectory // 'bin' - .appending(components: "lib", "arc") + .appending(components: "lib", additionalPath) } return nil } + internal func findARCLiteLibPath() throws -> AbsolutePath? { + return try findXcodeClangLibPath("arc") + } + + internal func addLTOLibArgs(to commandLine: inout [Job.ArgTemplate]) throws { + if let path = try findXcodeClangLibPath("libLTO.dylib") { + commandLine.appendFlag("-lto_library") + commandLine.appendPath(path) + } + } + func addLinkRuntimeLibraryRPath( to commandLine: inout [Job.ArgTemplate], parsedOptions: inout ParsedOptions, - targetTriple: Triple, + targetInfo: FrontendTargetInfo, darwinLibName: String ) throws { // Adding the rpaths might negatively interact when other rpaths are involved, @@ -55,7 +66,7 @@ extension DarwinToolchain { let clangPath = try clangLibraryPath( - for: targetTriple, + for: targetInfo, parsedOptions: &parsedOptions) commandLine.appendFlag("-rpath") commandLine.appendPath(clangPath) @@ -64,7 +75,7 @@ extension DarwinToolchain { func addLinkSanitizerLibArgsForDarwin( to commandLine: inout [Job.ArgTemplate], parsedOptions: inout ParsedOptions, - targetTriple: Triple, + targetInfo: FrontendTargetInfo, sanitizer: Sanitizer, isShared: Bool ) throws { @@ -76,13 +87,13 @@ extension DarwinToolchain { let sanitizerName = try runtimeLibraryName( for: sanitizer, - targetTriple: targetTriple, + targetTriple: targetInfo.target.triple, isShared: isShared ) try addLinkRuntimeLibrary( named: sanitizerName, to: &commandLine, - for: targetTriple, + for: targetInfo, parsedOptions: &parsedOptions ) @@ -90,7 +101,7 @@ extension DarwinToolchain { try addLinkRuntimeLibraryRPath( to: &commandLine, parsedOptions: &parsedOptions, - targetTriple: targetTriple, + targetInfo: targetInfo, darwinLibName: sanitizerName ) } @@ -99,18 +110,17 @@ extension DarwinToolchain { private func addProfileGenerationArgs( to commandLine: inout [Job.ArgTemplate], parsedOptions: inout ParsedOptions, - targetTriple: Triple + targetInfo: FrontendTargetInfo ) throws { guard parsedOptions.hasArgument(.profileGenerate) else { return } - let clangPath = try clangLibraryPath(for: targetTriple, + let clangPath = try clangLibraryPath(for: targetInfo, parsedOptions: &parsedOptions) - let runtime = targetTriple.darwinPlatform!.libraryNameSuffix - - let clangRTPath = clangPath - .appending(component: "libclang_rt.profile_\(runtime).a") - - commandLine.appendPath(clangRTPath) + for runtime in targetInfo.target.triple.darwinPlatform!.profileLibraryNameSuffixes { + let clangRTPath = clangPath + .appending(component: "libclang_rt.profile_\(runtime).a") + commandLine.appendPath(clangRTPath) + } } private func addPlatformVersionArg(to commandLine: inout [Job.ArgTemplate], @@ -183,131 +193,61 @@ extension DarwinToolchain { inputs: [TypedVirtualPath], outputFile: VirtualPath, shouldUseInputFileList: Bool, - sdkPath: String?, + lto: LTOKind?, sanitizers: Set, targetInfo: FrontendTargetInfo ) throws -> AbsolutePath { - - // FIXME: If we used Clang as a linker instead of going straight to ld, - // we wouldn't have to replicate a bunch of Clang's logic here. - - // Always link the regular compiler_rt if it's present. Note that the - // regular libclang_rt.a uses a fat binary for device and simulator; this is - // not true for all compiler_rt build products. - // - // Note: Normally we'd just add this unconditionally, but it's valid to build - // Swift and use it as a linker without building compiler_rt. - let targetTriple = targetInfo.target.triple - let darwinPlatformSuffix = - targetTriple.darwinPlatform!.with(.device)!.libraryNameSuffix - let compilerRTPath = - try clangLibraryPath( - for: targetTriple, - parsedOptions: &parsedOptions) - .appending(component: "libclang_rt.\(darwinPlatformSuffix).a") - if fileSystem.exists(compilerRTPath) { - commandLine.append(.path(.absolute(compilerRTPath))) - } - // Set up for linking. let linkerTool: Tool switch linkerOutputType { case .dynamicLibrary: // Same as an executable, but with the -dylib flag + linkerTool = .dynamicLinker commandLine.appendFlag("-dylib") - fallthrough + addLinkInputs(shouldUseInputFileList: shouldUseInputFileList, + commandLine: &commandLine, + inputs: inputs, + linkerOutputType: linkerOutputType) + try addDynamicLinkerFlags(targetInfo: targetInfo, + parsedOptions: &parsedOptions, + commandLine: &commandLine, + sanitizers: sanitizers, + linkerOutputType: linkerOutputType, + lto: lto) + case .executable: linkerTool = .dynamicLinker - let fSystemArgs = parsedOptions.arguments(for: .F, .Fsystem) - for opt in fSystemArgs { - commandLine.appendFlag(.F) - commandLine.appendPath(try VirtualPath(path: opt.argument.asSingle)) - } - - // Linking sanitizers will add rpaths, which might negatively interact when - // other rpaths are involved, so we should make sure we add the rpaths after - // all user-specified rpaths. - for sanitizer in sanitizers { - if sanitizer == .fuzzer { - guard linkerOutputType == .executable else { continue } - } - try addLinkSanitizerLibArgsForDarwin( - to: &commandLine, - parsedOptions: &parsedOptions, - targetTriple: targetTriple, - sanitizer: sanitizer, - isShared: sanitizer != .fuzzer - ) - } - - commandLine.appendFlag("-arch") - commandLine.appendFlag(targetTriple.archName) - - try addArgsToLinkStdlib( - to: &commandLine, - parsedOptions: &parsedOptions, - sdkPath: sdkPath, - targetInfo: targetInfo, - linkerOutputType: linkerOutputType, - fileSystem: fileSystem - ) - - // These custom arguments should be right before the object file at the - // end. - try commandLine.append( - contentsOf: parsedOptions.arguments(in: .linkerOption) - ) - try commandLine.appendAllArguments(.Xlinker, from: &parsedOptions) + addLinkInputs(shouldUseInputFileList: shouldUseInputFileList, + commandLine: &commandLine, + inputs: inputs, + linkerOutputType: linkerOutputType) + try addDynamicLinkerFlags(targetInfo: targetInfo, + parsedOptions: &parsedOptions, + commandLine: &commandLine, + sanitizers: sanitizers, + linkerOutputType: linkerOutputType, + lto: lto) case .staticLibrary: - linkerTool = .staticLinker + linkerTool = .staticLinker(lto) commandLine.appendFlag(.static) + addLinkInputs(shouldUseInputFileList: shouldUseInputFileList, + commandLine: &commandLine, + inputs: inputs, + linkerOutputType: linkerOutputType) } - try addArgsToLinkARCLite( - to: &commandLine, - parsedOptions: &parsedOptions, - targetTriple: targetTriple - ) - let targetVariantTriple = targetInfo.targetVariant?.triple - addDeploymentTargetArgs( - to: &commandLine, - targetTriple: targetTriple, - targetVariantTriple: targetVariantTriple, - sdkPath: try sdkPath.map(VirtualPath.init(path:)) - ) - try addProfileGenerationArgs( - to: &commandLine, - parsedOptions: &parsedOptions, - targetTriple: targetTriple - ) - - commandLine.appendFlags( - "-lobjc", - "-lSystem", - "-no_objc_category_merging" - ) - - // Add the SDK path - if let sdkPath = sdkPath { - commandLine.appendFlag("-syslibroot") - commandLine.appendPath(try VirtualPath(path: sdkPath)) - } - - if parsedOptions.contains(.embedBitcode) || - parsedOptions.contains(.embedBitcodeMarker) { - commandLine.appendFlag("-bitcode_bundle") - } - - if parsedOptions.contains(.enableAppExtension) { - commandLine.appendFlag("-application_extension") - } + // Add the output + commandLine.appendFlag("-o") + commandLine.appendPath(outputFile) - // On Darwin, we only support libc++. - if parsedOptions.contains(.enableExperimentalCxxInterop) { - commandLine.appendFlag("-lc++") - } + return try getToolPath(linkerTool) + } + private func addLinkInputs(shouldUseInputFileList: Bool, + commandLine: inout [Job.ArgTemplate], + inputs: [TypedVirtualPath], + linkerOutputType: LinkOutputType) { // inputs LinkFileList if shouldUseInputFileList { commandLine.appendFlag(.filelist) @@ -315,17 +255,21 @@ extension DarwinToolchain { var inputPaths = [VirtualPath]() var inputModules = [VirtualPath]() for input in inputs { - if input.type == .swiftModule { + if input.type == .swiftModule && linkerOutputType != .staticLibrary { inputPaths.append(input.file) inputModules.append(input.file) } else if input.type == .object { inputPaths.append(input.file) + } else if input.type == .llvmBitcode { + inputPaths.append(input.file) } } commandLine.appendPath(.fileList(path, .list(inputPaths))) - for module in inputModules { - commandLine.append(.flag("-add_ast_path")) - commandLine.append(.path(module)) + if linkerOutputType != .staticLibrary { + for module in inputModules { + commandLine.append(.flag("-add_ast_path")) + commandLine.append(.path(module)) + } } // FIXME: Primary inputs need to check -index-file-path @@ -333,20 +277,156 @@ extension DarwinToolchain { // Add inputs. commandLine.append(contentsOf: inputs.flatMap { (path: TypedVirtualPath) -> [Job.ArgTemplate] in - if path.type == .swiftModule { + if path.type == .swiftModule && linkerOutputType != .staticLibrary { return [.flag("-add_ast_path"), .path(path.file)] } else if path.type == .object { return [.path(path.file)] + } else if path.type == .llvmBitcode { + return [.path(path.file)] } else { return [] } }) } + } - // Add the output - commandLine.appendFlag("-o") - commandLine.appendPath(outputFile) + private func addDynamicLinkerFlags(targetInfo: FrontendTargetInfo, + parsedOptions: inout ParsedOptions, + commandLine: inout [Job.ArgTemplate], + sanitizers: Set, + linkerOutputType: LinkOutputType, + lto: LTOKind?) throws { + // FIXME: If we used Clang as a linker instead of going straight to ld, + // we wouldn't have to replicate a bunch of Clang's logic here. - return try getToolPath(linkerTool) + // Always link the regular compiler_rt if it's present. Note that the + // regular libclang_rt.a uses a fat binary for device and simulator; this is + // not true for all compiler_rt build products. + // + // Note: Normally we'd just add this unconditionally, but it's valid to build + // Swift and use it as a linker without building compiler_rt. + let targetTriple = targetInfo.target.triple + let darwinPlatformSuffix = + targetTriple.darwinPlatform!.with(.device)!.libraryNameSuffix + let compilerRTPath = + try clangLibraryPath( + for: targetInfo, + parsedOptions: &parsedOptions) + .appending(component: "libclang_rt.\(darwinPlatformSuffix).a") + if try fileSystem.exists(compilerRTPath) { + commandLine.append(.path(compilerRTPath)) + } + + try addArgsToLinkARCLite( + to: &commandLine, + parsedOptions: &parsedOptions, + targetTriple: targetTriple + ) + + if lto != nil { + try addLTOLibArgs(to: &commandLine) + } + + let fSystemArgs = parsedOptions.arguments(for: .F, .Fsystem) + for opt in fSystemArgs { + commandLine.appendFlag(.F) + commandLine.appendPath(try VirtualPath(path: opt.argument.asSingle)) + } + + if parsedOptions.contains(.enableAppExtension) { + commandLine.appendFlag("-application_extension") + } + + // Linking sanitizers will add rpaths, which might negatively interact when + // other rpaths are involved, so we should make sure we add the rpaths after + // all user-specified rpaths. + for sanitizer in sanitizers { + if sanitizer == .fuzzer { + guard linkerOutputType == .executable else { continue } + } + try addLinkSanitizerLibArgsForDarwin( + to: &commandLine, + parsedOptions: &parsedOptions, + targetInfo: targetInfo, + sanitizer: sanitizer, + isShared: sanitizer != .fuzzer + ) + } + + if parsedOptions.contains(.embedBitcode) || + parsedOptions.contains(.embedBitcodeMarker) { + commandLine.appendFlag("-bitcode_bundle") + } + + // Add the SDK path + if let sdkPath = targetInfo.sdkPath?.path { + commandLine.appendFlag("-syslibroot") + commandLine.appendPath(sdkPath) + } + + commandLine.appendFlags( + "-lobjc", + "-lSystem" + ) + + commandLine.appendFlag("-arch") + commandLine.appendFlag(targetTriple.archName) + + // On Darwin, we only support libc++. + if parsedOptions.contains(.enableExperimentalCxxInterop) { + commandLine.appendFlag("-lc++") + } + + try addArgsToLinkStdlib( + to: &commandLine, + parsedOptions: &parsedOptions, + targetInfo: targetInfo, + linkerOutputType: linkerOutputType, + fileSystem: fileSystem + ) + + try addProfileGenerationArgs( + to: &commandLine, + parsedOptions: &parsedOptions, + targetInfo: targetInfo + ) + + let targetVariantTriple = targetInfo.targetVariant?.triple + addDeploymentTargetArgs( + to: &commandLine, + targetTriple: targetTriple, + targetVariantTriple: targetVariantTriple, + sdkPath: targetInfo.sdkPath?.path + ) + + commandLine.appendFlag("-no_objc_category_merging") + + // These custom arguments should be right before the object file at the + // end. + try commandLine.append( + contentsOf: parsedOptions.arguments(in: .linkerOption) + ) + try commandLine.appendAllArguments(.Xlinker, from: &parsedOptions) + } +} + +private extension DarwinPlatform { + var profileLibraryNameSuffixes: [String] { + switch self { + case .macOS, .iOS(.catalyst): + return ["osx"] + case .iOS(.device): + return ["ios"] + case .iOS(.simulator): + return ["iossim", "ios"] + case .tvOS(.device): + return ["tvos"] + case .tvOS(.simulator): + return ["tvossim", "tvos"] + case .watchOS(.device): + return ["watchos"] + case .watchOS(.simulator): + return ["watchossim", "watchos"] + } } } diff --git a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift index e0dc822e0..14f3fe47c 100644 --- a/Sources/SwiftDriver/Jobs/EmitModuleJob.swift +++ b/Sources/SwiftDriver/Jobs/EmitModuleJob.swift @@ -14,7 +14,8 @@ extension Driver { /// options for the paths of various module files. mutating func addCommonModuleOptions( commandLine: inout [Job.ArgTemplate], - outputs: inout [TypedVirtualPath] + outputs: inout [TypedVirtualPath], + isMergeModule: Bool ) { // Add supplemental outputs. func addSupplementalOutput(path: VirtualPath?, flag: String, type: FileType) { @@ -28,10 +29,15 @@ extension Driver { addSupplementalOutput(path: moduleDocOutputPath, flag: "-emit-module-doc-path", type: .swiftDocumentation) addSupplementalOutput(path: moduleSourceInfoPath, flag: "-emit-module-source-info-path", type: .swiftSourceInfoFile) addSupplementalOutput(path: swiftInterfacePath, flag: "-emit-module-interface-path", type: .swiftInterface) - addSupplementalOutput(path: serializedDiagnosticsFilePath, flag: "-serialize-diagnostics-path", type: .diagnostics) + addSupplementalOutput(path: swiftPrivateInterfacePath, flag: "-emit-private-module-interface-path", type: .privateSwiftInterface) addSupplementalOutput(path: objcGeneratedHeaderPath, flag: "-emit-objc-header-path", type: .objcHeader) addSupplementalOutput(path: tbdPath, flag: "-emit-tbd-path", type: .tbd) + if isMergeModule || shouldCreateEmitModuleJob { + return + } + // Add outputs that can't be merged + addSupplementalOutput(path: serializedDiagnosticsFilePath, flag: "-serialize-diagnostics-path", type: .diagnostics) if let dependenciesFilePath = dependenciesFilePath { var path = dependenciesFilePath // FIXME: Hack to workaround the fact that SwiftPM/Xcode don't pass this path right now. @@ -51,7 +57,7 @@ extension Driver { TypedVirtualPath(file: moduleOutputPath, type: .swiftModule) ] - commandLine.appendFlags("-frontend", "-emit-module") + commandLine.appendFlags("-frontend", "-emit-module", "-experimental-skip-non-inlinable-function-bodies-without-types") let swiftInputFiles = inputFiles.filter { $0.type.isPartOfSwiftCompilation } @@ -61,10 +67,14 @@ extension Driver { inputs.append(input) } + if let pchPath = bridgingPrecompiledHeader { + inputs.append(TypedVirtualPath(file: pchPath, type: .pch)) + } + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) // FIXME: Add MSVC runtime library flags - addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs) + addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs, isMergeModule: false) commandLine.appendFlag(.o) commandLine.appendPath(moduleOutputPath) @@ -75,14 +85,14 @@ extension Driver { tool: .absolute(try toolchain.getToolPath(.swiftCompiler)), commandLine: commandLine, inputs: inputs, + primaryInputs: [], outputs: outputs ) } /// Returns true if the emit module job should be created. var shouldCreateEmitModuleJob: Bool { - return forceEmitModuleInSingleInvocation - && compilerOutputType != .swiftModule - && moduleOutputInfo.output != nil + return forceEmitModuleBeforeCompile + || parsedOptions.hasArgument(.emitModuleSeparately) } } diff --git a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift index 9a01a39e5..256c691eb 100644 --- a/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift +++ b/Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift @@ -103,9 +103,9 @@ extension Driver { // Handle the CPU and its preferences. try commandLine.appendLast(.targetCpu, from: &parsedOptions) - if let sdkPath = sdkPath { + if let sdkPath = frontendTargetInfo.sdkPath?.path { commandLine.appendFlag(.sdk) - commandLine.append(.path(try .init(path: sdkPath))) + commandLine.append(.path(sdkPath)) } try commandLine.appendAll(.I, from: &parsedOptions) @@ -136,7 +136,6 @@ extension Driver { try commandLine.appendLast(.moduleLinkName, from: &parsedOptions) try commandLine.appendLast(.nostdimport, from: &parsedOptions) try commandLine.appendLast(.parseStdlib, from: &parsedOptions) - try commandLine.appendLast(.resourceDir, from: &parsedOptions) try commandLine.appendLast(.solverMemoryThreshold, from: &parsedOptions) try commandLine.appendLast(.valueRecursionThreshold, from: &parsedOptions) try commandLine.appendLast(.warnSwift3ObjcInference, from: &parsedOptions) @@ -162,21 +161,18 @@ extension Driver { try commandLine.appendLast(.packageDescriptionVersion, from: &parsedOptions) try commandLine.appendLast(.serializeDiagnosticsPath, from: &parsedOptions) try commandLine.appendLast(.debugDiagnosticNames, from: &parsedOptions) - try commandLine.appendLast(.enableAstscopeLookup, from: &parsedOptions) - try commandLine.appendLast(.disableAstscopeLookup, from: &parsedOptions) try commandLine.appendLast(.disableParserLookup, from: &parsedOptions) try commandLine.appendLast(.sanitizeRecoverEQ, from: &parsedOptions) try commandLine.appendLast(.scanDependencies, from: &parsedOptions) try commandLine.appendLast(.scanClangDependencies, from: &parsedOptions) - try commandLine.appendLast(.fineGrainedDependencyIncludeIntrafile, from: &parsedOptions) try commandLine.appendLast(.enableExperimentalConcisePoundFile, from: &parsedOptions) try commandLine.appendLast(.printEducationalNotes, from: &parsedOptions) try commandLine.appendLast(.diagnosticStyle, from: &parsedOptions) - try commandLine.appendLast(.enableDirectIntramoduleDependencies, .disableDirectIntramoduleDependencies, from: &parsedOptions) try commandLine.appendLast(.locale, from: &parsedOptions) try commandLine.appendLast(.localizationPath, from: &parsedOptions) try commandLine.appendLast(.requireExplicitAvailability, from: &parsedOptions) try commandLine.appendLast(.requireExplicitAvailabilityTarget, from: &parsedOptions) + try commandLine.appendLast(.lto, from: &parsedOptions) try commandLine.appendAll(.D, from: &parsedOptions) try commandLine.appendAll(.sanitizeEQ, from: &parsedOptions) try commandLine.appendAll(.debugPrefixMap, from: &parsedOptions) @@ -195,15 +191,9 @@ extension Driver { // Resource directory. commandLine.appendFlag(.resourceDir) - commandLine.appendPath( - try AbsolutePath(validating: frontendTargetInfo.paths.runtimeResourcePath)) - - if parsedOptions.hasFlag(positive: .staticExecutable, - negative: .noStaticExecutable, - default: false) || - parsedOptions.hasFlag(positive: .staticStdlib, - negative: .noStaticStdlib, - default: false) { + commandLine.appendPath(frontendTargetInfo.runtimeResourcePath.path) + + if self.useStaticResourceDir { commandLine.appendFlag("-use-static-resource-dir") } @@ -259,7 +249,8 @@ extension Driver { mutating func addFrontendSupplementaryOutputArguments(commandLine: inout [Job.ArgTemplate], primaryInputs: [TypedVirtualPath], - inputOutputMap: [TypedVirtualPath: TypedVirtualPath]) throws -> [TypedVirtualPath] { + inputOutputMap: [TypedVirtualPath: TypedVirtualPath], + includeModuleTracePath: Bool) throws -> [TypedVirtualPath] { var flaggedInputOutputPairs: [(flag: String, input: TypedVirtualPath?, output: TypedVirtualPath)] = [] /// Add output of a particular type, if needed. @@ -295,7 +286,7 @@ extension Driver { /// Add all of the outputs needed for a given input. func addAllOutputsFor(input: TypedVirtualPath?) { - if !forceEmitModuleInSingleInvocation { + if !shouldCreateEmitModuleJob { addOutputOfType( outputType: .swiftModule, finalOutputPath: moduleOutputInfo.output?.outputPath, @@ -311,11 +302,22 @@ extension Driver { finalOutputPath: moduleSourceInfoPath, input: input, flag: "-emit-module-source-info-path") + } + + addOutputOfType( + outputType: .dependencies, + finalOutputPath: dependenciesFilePath, + input: input, + flag: "-emit-dependencies-path") + + if let input = input, let outputFileMap = outputFileMap { + let referenceDependenciesPath = + outputFileMap.existingOutput(inputFile: input.file, outputType: .swiftDeps) addOutputOfType( - outputType: .dependencies, - finalOutputPath: dependenciesFilePath, + outputType: .swiftDeps, + finalOutputPath: referenceDependenciesPath, input: input, - flag: "-emit-dependencies-path") + flag: "-emit-reference-dependencies-path") } addOutputOfType( @@ -357,6 +359,12 @@ extension Driver { input: nil, flag: "-emit-module-interface-path") + addOutputOfType( + outputType: .privateSwiftInterface, + finalOutputPath: swiftPrivateInterfacePath, + input: nil, + flag: "-emit-private-module-interface-path") + addOutputOfType( outputType: .tbd, finalOutputPath: tbdPath, @@ -364,8 +372,15 @@ extension Driver { flag: "-emit-tbd-path") } + if includeModuleTracePath, let tracePath = loadedModuleTracePath { + flaggedInputOutputPairs.append((flag: "-emit-loaded-module-trace-path", + input: nil, + output: TypedVirtualPath(file: tracePath, type: .moduleTrace))) + } + + let allInputsCount = primaryInputs.count + flaggedInputOutputPairs.count // Question: outputs.count > fileListThreshold makes sense, but c++ does the following: - if primaryInputs.count * FileType.allCases.count > fileListThreshold { + if allInputsCount * FileType.allCases.count > fileListThreshold { var entries = [VirtualPath: [FileType: VirtualPath]]() for input in primaryInputs { if let output = inputOutputMap[input] { @@ -404,19 +419,16 @@ extension Driver { /// to inputs and command line arguments of a compile job. func addExplicitModuleBuildArguments(inputs: inout [TypedVirtualPath], commandLine: inout [Job.ArgTemplate]) throws { - guard var handler = explicitModuleBuildHandler else { - fatalError("No handler in Explicit Module Build mode.") + guard var dependencyPlanner = explicitDependencyBuildPlanner else { + fatalError("No dependency planner in Explicit Module Build mode.") } - try handler.resolveMainModuleDependencies(inputs: &inputs, commandLine: &commandLine) + try dependencyPlanner.resolveMainModuleDependencies(inputs: &inputs, commandLine: &commandLine) } /// In Explicit Module Build mode, distinguish between main module jobs and intermediate dependency build jobs, /// such as Swift modules built from .swiftmodule files and Clang PCMs. public func isExplicitMainModuleJob(job: Job) -> Bool { - guard let handler = explicitModuleBuildHandler else { - fatalError("No handler in Explicit Module Build mode.") - } - return job.moduleName == handler.dependencyGraph.mainModuleName + return job.moduleName == moduleOutputInfo.name } } diff --git a/Sources/SwiftDriver/Jobs/GenerateDSYMJob.swift b/Sources/SwiftDriver/Jobs/GenerateDSYMJob.swift index fc60e79b8..e971bb297 100644 --- a/Sources/SwiftDriver/Jobs/GenerateDSYMJob.swift +++ b/Sources/SwiftDriver/Jobs/GenerateDSYMJob.swift @@ -29,6 +29,7 @@ extension Driver { commandLine: commandLine, displayInputs: [], inputs: inputs, + primaryInputs: [], outputs: [.init(file: outputPath, type: .dSYM)] ) } diff --git a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift index 2e5522f45..176e5f620 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCHJob.swift @@ -68,6 +68,7 @@ extension Driver { commandLine: commandLine, displayInputs: [], inputs: inputs, + primaryInputs: [], outputs: outputs, supportsResponseFiles: true ) diff --git a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift index 8416cff41..1a0eff3d8 100644 --- a/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift +++ b/Sources/SwiftDriver/Jobs/GeneratePCMJob.swift @@ -59,6 +59,7 @@ extension Driver { commandLine: commandLine, displayInputs: [], inputs: inputs, + primaryInputs: [], outputs: outputs ) } diff --git a/Sources/SwiftDriver/Jobs/GenericUnixToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/GenericUnixToolchain+LinkerSupport.swift index 888376969..1a1f4908d 100644 --- a/Sources/SwiftDriver/Jobs/GenericUnixToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/GenericUnixToolchain+LinkerSupport.swift @@ -50,7 +50,7 @@ extension GenericUnixToolchain { inputs: [TypedVirtualPath], outputFile: VirtualPath, shouldUseInputFileList: Bool, - sdkPath: String?, + lto: LTOKind?, sanitizers: Set, targetInfo: FrontendTargetInfo ) throws -> AbsolutePath { @@ -70,6 +70,8 @@ extension GenericUnixToolchain { var linker: String? if let arg = parsedOptions.getLastArgument(.useLd) { linker = arg.asSingle + } else if lto != nil { + linker = "lld" } else { linker = defaultLinker(for: targetTriple) } @@ -130,9 +132,9 @@ extension GenericUnixToolchain { let hasRuntimeArgs = !(staticStdlib || staticExecutable) let runtimePaths = try runtimeLibraryPaths( - for: targetTriple, + for: targetInfo, parsedOptions: &parsedOptions, - sdkPath: sdkPath, + sdkPath: targetInfo.sdkPath?.path, isShared: hasRuntimeArgs ) @@ -147,15 +149,11 @@ extension GenericUnixToolchain { } } - let sharedResourceDirPath = try computeResourceDirPath( - for: targetTriple, - parsedOptions: &parsedOptions, - isShared: true - ) - - let swiftrtPath = sharedResourceDirPath + let swiftrtPath = targetInfo.runtimeResourcePath.path .appending( - components: String(majorArchitectureName(for: targetTriple)), "swiftrt.o" + components: targetTriple.platformName() ?? "", + String(majorArchitectureName(for: targetTriple)), + "swiftrt.o" ) commandLine.appendPath(swiftrtPath) @@ -165,6 +163,8 @@ extension GenericUnixToolchain { return .responseFilePath(input.file) } else if input.type == .object { return .path(input.file) + } else if lto != nil && input.type == .llvmBitcode { + return .path(input.file) } else { return nil } @@ -181,9 +181,9 @@ extension GenericUnixToolchain { commandLine.appendPath(try VirtualPath(path: opt.argument.asSingle)) } - if let path = sdkPath { + if let path = targetInfo.sdkPath?.path { commandLine.appendFlag("--sysroot") - commandLine.appendFlag(path) + commandLine.appendPath(path) } // Add the runtime library link paths. @@ -195,11 +195,8 @@ extension GenericUnixToolchain { // Link the standard library. In two paths, we do this using a .lnk file // if we're going that route, we'll set `linkFilePath` to the path to that // file. - var linkFilePath: AbsolutePath? = try computeResourceDirPath( - for: targetTriple, - parsedOptions: &parsedOptions, - isShared: false - ) + var linkFilePath: VirtualPath? = targetInfo.runtimeResourcePath.path + .appending(component: targetTriple.platformName() ?? "") if staticExecutable { linkFilePath = linkFilePath?.appending(component: "static-executable-args.lnk") @@ -211,10 +208,10 @@ extension GenericUnixToolchain { } if let linkFile = linkFilePath { - guard fileSystem.isFile(linkFile) else { - fatalError("\(linkFile.pathString) not found") + guard try fileSystem.exists(linkFile) else { + fatalError("\(linkFile) not found") } - commandLine.append(.responseFilePath(.absolute(linkFile))) + commandLine.append(.responseFilePath(linkFile)) } // Explicitly pass the target to the linker @@ -237,16 +234,24 @@ extension GenericUnixToolchain { } if parsedOptions.hasArgument(.profileGenerate) { - let libProfile = sharedResourceDirPath - .parentDirectory // remove platform name + let libProfile = targetInfo.runtimeResourcePath.path .appending(components: "clang", "lib", targetTriple.osName, - "libclangrt_profile-\(targetTriple.archName).a") + "libclang_rt.profile-\(targetTriple.archName).a") commandLine.appendPath(libProfile) // HACK: Hard-coded from llvm::getInstrProfRuntimeHookVarName() commandLine.appendFlag("-u__llvm_profile_runtime") } + if let lto = lto { + switch lto { + case .llvmFull: + commandLine.appendFlag("-flto=full") + case .llvmThin: + commandLine.appendFlag("-flto=thin") + } + } + // Run clang++ in verbose mode if "-v" is set try commandLine.appendLast(.v, from: &parsedOptions) @@ -268,7 +273,7 @@ extension GenericUnixToolchain { commandLine.appendPath(outputFile) commandLine.append(contentsOf: inputs.map { .path($0.file) }) - return try getToolPath(.staticLinker) + return try getToolPath(.staticLinker(lto)) } } diff --git a/Sources/SwiftDriver/Jobs/InterpretJob.swift b/Sources/SwiftDriver/Jobs/InterpretJob.swift index 90453fa38..ad029269f 100644 --- a/Sources/SwiftDriver/Jobs/InterpretJob.swift +++ b/Sources/SwiftDriver/Jobs/InterpretJob.swift @@ -37,8 +37,8 @@ extension Driver { try commandLine.appendLast(.DASHDASH, from: &parsedOptions) let extraEnvironment = try toolchain.platformSpecificInterpreterEnvironmentVariables( - env: self.env, parsedOptions: &parsedOptions, sdkPath: self.sdkPath, - targetTriple: self.targetTriple) + env: self.env, parsedOptions: &parsedOptions, + sdkPath: frontendTargetInfo.sdkPath?.path, targetInfo: self.frontendTargetInfo) return Job( moduleName: moduleOutputInfo.name, @@ -46,6 +46,7 @@ extension Driver { tool: .absolute(try toolchain.getToolPath(.swiftCompiler)), commandLine: commandLine, inputs: inputs, + primaryInputs: [], outputs: [], extraEnvironment: extraEnvironment, requiresInPlaceExecution: true, diff --git a/Sources/SwiftDriver/Jobs/Job.swift b/Sources/SwiftDriver/Jobs/Job.swift index b8734d2c8..14139357d 100644 --- a/Sources/SwiftDriver/Jobs/Job.swift +++ b/Sources/SwiftDriver/Jobs/Job.swift @@ -19,7 +19,7 @@ public struct Job: Codable, Equatable, Hashable { case backend case mergeModule = "merge-module" case link - case generateDSYM = "generate-dsym" + case generateDSYM = "generate-dSYM" case autolinkExtract = "autolink-extract" case emitModule = "emit-module" case generatePCH = "generate-pch" @@ -70,6 +70,9 @@ public struct Job: Codable, Equatable, Hashable { /// The list of inputs for this job. public var inputs: [TypedVirtualPath] + /// The primary inputs for compile jobs + public var primaryInputs: [TypedVirtualPath] + /// The outputs produced by the job. public var outputs: [TypedVirtualPath] @@ -89,6 +92,7 @@ public struct Job: Codable, Equatable, Hashable { commandLine: [ArgTemplate], displayInputs: [TypedVirtualPath]? = nil, inputs: [TypedVirtualPath], + primaryInputs: [TypedVirtualPath], outputs: [TypedVirtualPath], extraEnvironment: [String: String] = [:], requiresInPlaceExecution: Bool = false, @@ -100,6 +104,7 @@ public struct Job: Codable, Equatable, Hashable { self.commandLine = commandLine self.displayInputs = displayInputs ?? [] self.inputs = inputs + self.primaryInputs = primaryInputs self.outputs = outputs self.extraEnvironment = extraEnvironment self.requiresInPlaceExecution = requiresInPlaceExecution @@ -191,6 +196,15 @@ extension Job : CustomStringConvertible { return "Verifying emitted module interface for module \(moduleName)" } } + + public var descriptionForLifecycle: String { + switch kind { + case .compile: + return "Compiling \(displayInputs.map {$0.file.basename}.joined(separator: ", "))" + default: + return description + } + } } extension Job.Kind { diff --git a/Sources/SwiftDriver/Jobs/LinkJob.swift b/Sources/SwiftDriver/Jobs/LinkJob.swift index 9497913cb..2322fbb5b 100644 --- a/Sources/SwiftDriver/Jobs/LinkJob.swift +++ b/Sources/SwiftDriver/Jobs/LinkJob.swift @@ -12,17 +12,31 @@ import TSCBasic extension Driver { - - /// Compute the output file for an image output. - private var outputFileForImage: VirtualPath { + private var relativeOutputFileForImage: RelativePath { if inputFiles.count == 1 && moduleOutputInfo.nameIsFallback && inputFiles[0].file != .standardInput { - return .relative(RelativePath(inputFiles[0].file.basenameWithoutExt)) + return RelativePath(inputFiles[0].file.basenameWithoutExt) } let outputName = toolchain.makeLinkerOutputFilename(moduleName: moduleOutputInfo.name, type: linkerOutputType!) - return .relative(RelativePath(outputName)) + return RelativePath(outputName) + } + + /// Compute the output file for an image output. + private var outputFileForImage: VirtualPath { + return useWorkingDirectory(relativeOutputFileForImage) + } + + func useWorkingDirectory(_ relative: RelativePath) -> VirtualPath { + return Driver.useWorkingDirectory(relative, workingDirectory) + } + + static func useWorkingDirectory(_ relative: RelativePath, _ workingDirectory: AbsolutePath?) -> VirtualPath { + if let wd = workingDirectory { + return .absolute(AbsolutePath(relative.pathString, relativeTo: wd)) + } + return .relative(relative) } /// Link the given inputs. @@ -45,7 +59,7 @@ extension Driver { inputs: inputs, outputFile: outputFile, shouldUseInputFileList: shouldUseInputFileList, - sdkPath: sdkPath, + lto: lto, sanitizers: enabledSanitizers, targetInfo: frontendTargetInfo ) @@ -56,7 +70,9 @@ extension Driver { kind: .link, tool: .absolute(toolPath), commandLine: commandLine, + displayInputs: inputs, inputs: inputs, + primaryInputs: [], outputs: [.init(file: outputFile, type: .image)] ) } diff --git a/Sources/SwiftDriver/Jobs/MergeModuleJob.swift b/Sources/SwiftDriver/Jobs/MergeModuleJob.swift index 87bc2da9d..297e0feba 100644 --- a/Sources/SwiftDriver/Jobs/MergeModuleJob.swift +++ b/Sources/SwiftDriver/Jobs/MergeModuleJob.swift @@ -53,10 +53,10 @@ extension Driver { commandLine.appendFlag(.disableDiagnosticPasses) commandLine.appendFlag(.disableSilPerfOptzns) - try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) + try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs, bridgingHeaderHandling: .parsed) // FIXME: Add MSVC runtime library flags - addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs) + addCommonModuleOptions(commandLine: &commandLine, outputs: &outputs, isMergeModule: true) commandLine.appendFlag(.o) commandLine.appendPath(moduleOutputInfo.output!.outputPath) @@ -67,6 +67,7 @@ extension Driver { tool: .absolute(try toolchain.getToolPath(.swiftCompiler)), commandLine: commandLine, inputs: inputs, + primaryInputs: [], outputs: outputs, supportsResponseFiles: true ) diff --git a/Sources/SwiftDriver/Jobs/ModuleWrapJob.swift b/Sources/SwiftDriver/Jobs/ModuleWrapJob.swift index 596bcecf6..068eaa16c 100644 --- a/Sources/SwiftDriver/Jobs/ModuleWrapJob.swift +++ b/Sources/SwiftDriver/Jobs/ModuleWrapJob.swift @@ -32,6 +32,7 @@ extension Driver { tool: .absolute(try toolchain.getToolPath(.swiftCompiler)), commandLine: commandLine, inputs: [moduleInput], + primaryInputs: [], outputs: [.init(file: outputPath, type: .object)], supportsResponseFiles: true ) diff --git a/Sources/SwiftDriver/Jobs/Planning.swift b/Sources/SwiftDriver/Jobs/Planning.swift index e8a91754d..bdc652222 100644 --- a/Sources/SwiftDriver/Jobs/Planning.swift +++ b/Sources/SwiftDriver/Jobs/Planning.swift @@ -27,251 +27,370 @@ public enum PlanningError: Error, DiagnosticData { } } -/// Planning for builds +/// // MARK: Standard build planning extension Driver { /// Plan a standard compilation, which produces jobs for compiling separate /// primary files. private mutating func planStandardCompile() throws -> [Job] { - var jobs = [Job]() + precondition(compilerMode.isStandardCompilationForPlanning, + "compiler mode \(compilerMode) is handled elsewhere") + + // Centralize job accumulation here. + // For incremental compilation, must separate jobs happening before and + // during compilation from those happening after. + + // When emitting bitcode, if the first compile job is scheduled, the + // second must be. Thus, job-groups. + var preAndCompileJobGroups = [[Job]]() + func addPreOrCompileJobGroup(_ group: [Job]) { + preAndCompileJobGroups.append(group) + } + func addPreOrCompileJob(_ j: Job) { + addPreOrCompileJobGroup([j]) + } + // need to buffer these to dodge shared ownership + var postCompileJobs = [Job]() + func addPostCompileJob(_ j: Job) { + postCompileJobs.append(j) + } + var allJobs: [Job] { + preAndCompileJobGroups.joined() + postCompileJobs + } - // Keep track of the various outputs we care about from the jobs we build. - var linkerInputs = [TypedVirtualPath]() - var moduleInputs = [TypedVirtualPath]() - var moduleInputsFromJobOutputs = [TypedVirtualPath]() - func addJobOutputs(_ jobOutputs: [TypedVirtualPath]) { - for jobOutput in jobOutputs { - switch jobOutput.type { - case .object, .autolink: - linkerInputs.append(jobOutput) + try addPrecompileModuleDependenciesJobs(addJob: addPreOrCompileJob) + try addPrecompileBridgingHeaderJob(addJob: addPreOrCompileJob) + try addEmitModuleJob(addJob: addPreOrCompileJob) + let linkerInputs = try addJobsFeedingLinker( + addJobGroup: addPreOrCompileJobGroup, + addPostCompileJob: addPostCompileJob) + try addLinkAndPostLinkJobs(linkerInputs: linkerInputs, + debugInfo: debugInfo, + addJob: addPostCompileJob) + + try incrementalCompilationState?.addPreOrCompileJobGroups(preAndCompileJobGroups) { + try formBatchedJobs($0, forIncremental: true) + } + incrementalCompilationState?.addPostCompileJobs(postCompileJobs) - case .swiftModule: - moduleInputsFromJobOutputs.append(jobOutput) + return try formBatchedJobs(allJobs, forIncremental: false) + } - default: - break - } - } - } + private mutating func addPrecompileModuleDependenciesJobs(addJob: (Job) -> Void) throws { // If asked, add jobs to precompile module dependencies - if parsedOptions.contains(.driverExplicitModuleBuild) || - parsedOptions.contains(.driverPrintModuleDependenciesJobs) { - let modulePrebuildJobs = try generateExplicitModuleBuildJobs() + guard parsedOptions.contains(.driverExplicitModuleBuild) else { return } + let modulePrebuildJobs = try generateExplicitModuleDependenciesJobs() + modulePrebuildJobs.forEach(addJob) + } - if parsedOptions.contains(.driverExplicitModuleBuild) { - jobs.append(contentsOf: modulePrebuildJobs) - } - // If we've been asked to prebuild module dependencies, - // for the time being, just print the jobs' compile commands. - if parsedOptions.contains(.driverPrintModuleDependenciesJobs) { - let forceResponseFiles = parsedOptions.contains(.driverForceResponseFiles) - for job in modulePrebuildJobs { - print(try executor.description(of: job, forceResponseFiles: forceResponseFiles)) - } - } - } + private mutating func addPrecompileBridgingHeaderJob(addJob: (Job) -> Void) throws { + guard + let importedObjCHeader = importedObjCHeader, + let bridgingPrecompiledHeader = bridgingPrecompiledHeader + else { return } - // Precompile the bridging header if needed. - if let importedObjCHeader = importedObjCHeader, - let bridgingPrecompiledHeader = bridgingPrecompiledHeader { - jobs.append(try generatePCHJob(input: .init(file: importedObjCHeader, type: .objcHeader), - output: .init(file: bridgingPrecompiledHeader, type: .pch))) - } + addJob( + try generatePCHJob(input: .init(file: importedObjCHeader, + type: .objcHeader), + output: .init(file: bridgingPrecompiledHeader, + type: .pch)) + ) + } - // If we should create emit module job, do so. + private mutating func addEmitModuleJob(addJob: (Job) -> Void) throws { if shouldCreateEmitModuleJob { - jobs.append(try emitModuleJob()) + addJob( try emitModuleJob() ) } + } - var backendJobs = [Job]() + private mutating func addJobsFeedingLinker( + addJobGroup: ([Job]) -> Void, + addPostCompileJob: (Job) -> Void + ) throws -> [TypedVirtualPath] { - let partitions: BatchPartitions? - switch compilerMode { - case .batchCompile(let batchInfo): - partitions = batchPartitions(batchInfo) - - case .immediate, .repl, .compilePCM: - fatalError("compiler mode \(compilerMode) is handled elsewhere") - - case .singleCompile: - if parsedOptions.hasArgument(.embedBitcode), - inputFiles.allSatisfy({ $0.type.isPartOfSwiftCompilation }) { - var jobOutputs: [TypedVirtualPath] = [] - let job = try compileJob(primaryInputs: [], outputType: .llvmBitcode, allOutputs: &jobOutputs) - jobs.append(job) - for input in job.outputs.filter({ $0.type == .llvmBitcode }) { - let job = try backendJob(input: input, allOutputs: &jobOutputs) - backendJobs.append(job) - } - addJobOutputs(jobOutputs) - } else { - // Create a single compile job for all of the files, none of which - // are primary. - var jobOutputs: [TypedVirtualPath] = [] - let job = try compileJob(primaryInputs: [], outputType: compilerOutputType, allOutputs: &jobOutputs) - jobs.append(job) - addJobOutputs(jobOutputs) - } + var linkerInputs = [TypedVirtualPath]() + func addLinkerInput(_ li: TypedVirtualPath) { linkerInputs.append(li) } - partitions = nil + var moduleInputs = [TypedVirtualPath]() + let acceptBitcodeAsLinkerInput = lto == .llvmThin || lto == .llvmFull + func addModuleInput(_ mi: TypedVirtualPath) { moduleInputs.append(mi) } + var moduleInputsFromJobOutputs = [TypedVirtualPath]() + func addModuleInputFromJobOutputs(_ mis: TypedVirtualPath) { + moduleInputsFromJobOutputs.append(mis) } - case .standardCompile: - partitions = nil + func addJobOutputs(_ jobOutputs: [TypedVirtualPath]) { + for jobOutput in jobOutputs { + switch jobOutput.type { + case .object, .autolink: + addLinkerInput(jobOutput) + case .llvmBitcode where acceptBitcodeAsLinkerInput: + addLinkerInput(jobOutput) + case .swiftModule: + addModuleInputFromJobOutputs(jobOutput) + + default: + break + } + } } - for input in inputFiles { - switch input.type { - case .swift, .sil, .sib: - // Generate a compile job for primary inputs here. - guard compilerMode.usesPrimaryFileInputs else { break } - - var primaryInputs: [TypedVirtualPath] - if let partitions = partitions, let partitionIdx = partitions.assignment[input] { - // We have a partitioning for batch mode. If this input file isn't the first - // file in the partition, skip it: it's been accounted for already. - let partition = partitions.partitions[partitionIdx] - if partition[0] != input { - continue - } - - if parsedOptions.hasArgument(.driverShowJobLifecycle) { - stdoutStream.write("Forming batch job from \(partition.count) constituents\n") - } + func addJob(_ j: Job) { addJobGroup([j]) } + + try addSingleCompileJobs(addJob: addJob, + addJobOutputs: addJobOutputs) + + try addJobsForPrimaryInputs( + addJobGroup: addJobGroup, + addModuleInput: addModuleInput, + addLinkerInput: addLinkerInput, + addJobOutputs: addJobOutputs) + + try addAutolinkExtractJob(linkerInputs: linkerInputs, + addLinkerInput: addLinkerInput, + addJob: addPostCompileJob) + + if let mergeJob = try mergeModuleJob( + moduleInputs: moduleInputs, + moduleInputsFromJobOutputs: moduleInputsFromJobOutputs) { + addPostCompileJob(mergeJob) + try addVerifyJobs(mergeJob: mergeJob, addJob: addPostCompileJob) + try addWrapJobOrMergeOutputs( + mergeJob: mergeJob, + debugInfo: debugInfo, + addJob: addPostCompileJob, + addLinkerInput: addLinkerInput) + } + return linkerInputs + } - primaryInputs = partitions.partitions[partitionIdx] - } else { - primaryInputs = [input] - } + private mutating func addSingleCompileJobs( + addJob: (Job) -> Void, + addJobOutputs: ([TypedVirtualPath]) -> Void + ) throws { + guard case .singleCompile = compilerMode + else { return } + + if parsedOptions.hasArgument(.embedBitcode), + inputFiles.allSatisfy({ $0.type.isPartOfSwiftCompilation }) + { + let job = try compileJob(primaryInputs: [], + outputType: .llvmBitcode, + addJobOutputs: addJobOutputs, + emitModuleTrace: loadedModuleTracePath != nil) + addJob(job) - if parsedOptions.hasArgument(.embedBitcode) { - var jobOutputs: [TypedVirtualPath] = [] - let job = try compileJob(primaryInputs: primaryInputs, outputType: .llvmBitcode, allOutputs: &jobOutputs) - jobs.append(job) - for input in job.outputs.filter({ $0.type == .llvmBitcode }) { - let job = try backendJob(input: input, allOutputs: &jobOutputs) - backendJobs.append(job) - } - addJobOutputs(jobOutputs) - } else { - var jobOutputs: [TypedVirtualPath] = [] - let job = try compileJob(primaryInputs: primaryInputs, outputType: compilerOutputType, allOutputs: &jobOutputs) - jobs.append(job) - addJobOutputs(jobOutputs) + for input in job.outputs.filter({ $0.type == .llvmBitcode }) { + let job = try backendJob(input: input, addJobOutputs: addJobOutputs) + addJob(job) } + return + } + // Create a single compile job for all of the files, none of which + // are primary. + let job = try compileJob(primaryInputs: [], + outputType: compilerOutputType, + addJobOutputs: addJobOutputs, + emitModuleTrace: loadedModuleTracePath != nil) + addJob(job) + } - case .object, .autolink: - if linkerOutputType != nil { - linkerInputs.append(input) - } else { - diagnosticEngine.emit(.error_unexpected_input_file(input.file)) - } + private mutating func addJobsForPrimaryInputs( + addJobGroup: ([Job]) -> Void, + addModuleInput: (TypedVirtualPath) -> Void, + addLinkerInput: (TypedVirtualPath) -> Void, + addJobOutputs: ([TypedVirtualPath]) -> Void) + throws { + let loadedModuleTraceInputIndex = inputFiles.firstIndex(where: { + $0.type.isPartOfSwiftCompilation && loadedModuleTracePath != nil + }) + for (index, input) in inputFiles.enumerated() { + // Only emit a loaded module trace from the first frontend job. + try addJobForPrimaryInput( + input: input, + addJobGroup: addJobGroup, + addModuleInput: addModuleInput, + addLinkerInput: addLinkerInput, + addJobOutputs: addJobOutputs, + emitModuleTrace: index == loadedModuleTraceInputIndex) + } + } - case .swiftModule, .swiftDocumentation: - if moduleOutputInfo.output != nil && linkerOutputType == nil { - // When generating a .swiftmodule as a top-level output (as opposed - // to, for example, linking an image), treat .swiftmodule files as - // inputs to a MergeModule action. - moduleInputs.append(input) - } else if linkerOutputType != nil { - // Otherwise, if linking, pass .swiftmodule files as inputs to the - // linker, so that their debug info is available. - linkerInputs.append(input) - } else { - diagnosticEngine.emit(.error_unexpected_input_file(input.file)) + private mutating func addJobForPrimaryInput( + input: TypedVirtualPath, + addJobGroup: ([Job]) -> Void, + addModuleInput: (TypedVirtualPath) -> Void, + addLinkerInput: (TypedVirtualPath) -> Void, + addJobOutputs: ([TypedVirtualPath]) -> Void, + emitModuleTrace: Bool + ) throws + { + switch input.type { + case .swift, .sil, .sib: + // Generate a compile job for primary inputs here. + guard compilerMode.usesPrimaryFileInputs else { break } + + let primaryInputs = [input] + if parsedOptions.hasArgument(.embedBitcode) { + let job = try compileJob(primaryInputs: primaryInputs, + outputType: .llvmBitcode, + addJobOutputs: addJobOutputs, + emitModuleTrace: emitModuleTrace) + var jobGroup = [job] + for input in job.outputs.filter({ $0.type == .llvmBitcode }) { + let job = try backendJob(input: input, addJobOutputs: addJobOutputs) + jobGroup.append(job) } + addJobGroup(jobGroup) + } else if !(compilerOutputType == .swiftModule && shouldCreateEmitModuleJob) { + // We can skip the compile jobs if all we want is a module when it's + // built separately. + let job = try compileJob(primaryInputs: primaryInputs, + outputType: compilerOutputType, + addJobOutputs: addJobOutputs, + emitModuleTrace: emitModuleTrace) + addJobGroup([job]) + } - default: + case .object, .autolink, .llvmBitcode: + if linkerOutputType != nil { + addLinkerInput(input) + } else { diagnosticEngine.emit(.error_unexpected_input_file(input.file)) } - } - jobs.append(contentsOf: backendJobs) + case .swiftModule: + if moduleOutputInfo.output != nil && linkerOutputType == nil { + // When generating a .swiftmodule as a top-level output (as opposed + // to, for example, linking an image), treat .swiftmodule files as + // inputs to a MergeModule action. + addModuleInput(input) + } else if linkerOutputType != nil { + // Otherwise, if linking, pass .swiftmodule files as inputs to the + // linker, so that their debug info is available. + addLinkerInput(input) + } else { + diagnosticEngine.emit(.error_unexpected_input_file(input.file)) + } - // Plan the merge-module job, if there are module inputs. - var mergeJob: Job? - if moduleOutputInfo.output != nil && !(moduleInputs.isEmpty && moduleInputsFromJobOutputs.isEmpty) && compilerMode.usesPrimaryFileInputs { - let mergeModule = try mergeModuleJob(inputs: moduleInputs, inputsFromOutputs: moduleInputsFromJobOutputs) - jobs.append(mergeModule) - mergeJob = mergeModule + default: + diagnosticEngine.emit(.error_unexpected_input_file(input.file)) } + } - if let mergeJob = mergeJob, + /// Need a merge module job if there are module inputs + private mutating func mergeModuleJob( + moduleInputs: [TypedVirtualPath], + moduleInputsFromJobOutputs: [TypedVirtualPath] + ) throws -> Job? { + guard moduleOutputInfo.output != nil, + !(moduleInputs.isEmpty && moduleInputsFromJobOutputs.isEmpty), + compilerMode.usesPrimaryFileInputs + else { return nil } + return try mergeModuleJob(inputs: moduleInputs, inputsFromOutputs: moduleInputsFromJobOutputs) + } + + private mutating func addVerifyJobs(mergeJob: Job, addJob: (Job) -> Void ) + throws { + guard parsedOptions.hasArgument(.enableLibraryEvolution), parsedOptions.hasFlag(positive: .verifyEmittedModuleInterface, negative: .noVerifyEmittedModuleInterface, - default: false) { - if parsedOptions.hasArgument(.emitModuleInterface) || - parsedOptions.hasArgument(.emitModuleInterfacePath) { - let mergeInterfaceOutputs = mergeJob.outputs.filter { $0.type == .swiftInterface } - assert(mergeInterfaceOutputs.count == 1, - "Merge module job should only have one swiftinterface output") - let verifyJob = try verifyModuleInterfaceJob(interfaceInput: mergeInterfaceOutputs[0]) - jobs.append(verifyJob) - } - if parsedOptions.hasArgument(.emitPrivateModuleInterfacePath) { - let mergeInterfaceOutputs = mergeJob.outputs.filter { $0.type == .privateSwiftInterface } - assert(mergeInterfaceOutputs.count == 1, - "Merge module job should only have one private swiftinterface output") - let verifyJob = try verifyModuleInterfaceJob(interfaceInput: mergeInterfaceOutputs[0]) - jobs.append(verifyJob) - } + default: false) + else { return } + + func addVerifyJob(forPrivate: Bool) throws { + let isNeeded = + forPrivate + ? parsedOptions.hasArgument(.emitPrivateModuleInterfacePath) + : parsedOptions.hasArgument(.emitModuleInterface, .emitModuleInterfacePath) + guard isNeeded else { return } + + let outputType: FileType = + forPrivate ? .privateSwiftInterface : .swiftInterface + let mergeInterfaceOutputs = mergeJob.outputs.filter { $0.type == outputType } + assert(mergeInterfaceOutputs.count == 1, + "Merge module job should only have one swiftinterface output") + let job = try verifyModuleInterfaceJob(interfaceInput: mergeInterfaceOutputs[0]) + addJob(job) } + try addVerifyJob(forPrivate: false) + try addVerifyJob(forPrivate: true ) + } - // If we need to autolink-extract, do so. + private mutating func addAutolinkExtractJob( + linkerInputs: [TypedVirtualPath], + addLinkerInput: (TypedVirtualPath) -> Void, + addJob: (Job) -> Void) + throws + { let autolinkInputs = linkerInputs.filter { $0.type == .object } if let autolinkExtractJob = try autolinkExtractJob(inputs: autolinkInputs) { - linkerInputs.append(contentsOf: autolinkExtractJob.outputs) - jobs.append(autolinkExtractJob) - } - - if let mergeJob = mergeJob, debugInfo.level == .astTypes { - if targetTriple.objectFormat != .macho { - // Module wrapping is required. - let mergeModuleOutputs = mergeJob.outputs.filter { $0.type == .swiftModule } - assert(mergeModuleOutputs.count == 1, - "Merge module job should only have one swiftmodule output") - let wrapJob = try moduleWrapJob(moduleInput: mergeModuleOutputs[0]) - linkerInputs.append(contentsOf: wrapJob.outputs) - jobs.append(wrapJob) - } else { - linkerInputs.append(contentsOf: mergeJob.outputs) - } + addJob(autolinkExtractJob) + autolinkExtractJob.outputs.forEach(addLinkerInput) } + } - // If we should link, do so. - var link: Job? - if linkerOutputType != nil && !linkerInputs.isEmpty { - link = try linkJob(inputs: linkerInputs) - jobs.append(link!) + private mutating func addWrapJobOrMergeOutputs(mergeJob: Job, + debugInfo: DebugInfo, + addJob: (Job) -> Void, + addLinkerInput: (TypedVirtualPath) -> Void) + throws { + guard case .astTypes = debugInfo.level + else { return } + if targetTriple.objectFormat != .macho { + // Module wrapping is required. + let mergeModuleOutputs = mergeJob.outputs.filter { $0.type == .swiftModule } + assert(mergeModuleOutputs.count == 1, + "Merge module job should only have one swiftmodule output") + let wrapJob = try moduleWrapJob(moduleInput: mergeModuleOutputs[0]) + addJob(wrapJob) + wrapJob.outputs.forEach(addLinkerInput) + } else { + let mergeModuleOutputs = mergeJob.outputs.filter { $0.type == .swiftModule } + assert(mergeModuleOutputs.count == 1, + "Merge module job should only have one swiftmodule output") + addLinkerInput(mergeModuleOutputs[0]) } + } - // If we should generate a dSYM, do so. - if let linkJob = link, targetTriple.isDarwin, debugInfo.level != nil { - let dsymJob = try generateDSYMJob(inputs: linkJob.outputs) - jobs.append(dsymJob) - if debugInfo.shouldVerify { - jobs.append(try verifyDebugInfoJob(inputs: dsymJob.outputs)) - } + private mutating func addLinkAndPostLinkJobs( + linkerInputs: [TypedVirtualPath], + debugInfo: DebugInfo, + addJob: (Job) -> Void + ) throws { + guard linkerOutputType != nil && !linkerInputs.isEmpty + else { return } + + let linkJ = try linkJob(inputs: linkerInputs) + addJob(linkJ) + guard targetTriple.isDarwin, debugInfo.level != nil + else {return } + + let dsymJob = try generateDSYMJob(inputs: linkJ.outputs) + addJob(dsymJob) + if debugInfo.shouldVerify { + addJob(try verifyDebugInfoJob(inputs: dsymJob.outputs)) } - - return jobs } /// Prescan the source files to produce a module dependency graph and turn it into a set /// of jobs required to build all dependencies. /// Preprocess the graph by resolving placeholder dependencies, if any are present and /// by re-scanning all Clang modules against all possible targets they will be built against. - public mutating func generateExplicitModuleBuildJobs() throws -> [Job] { - let dependencyGraph = try generateInterModuleDependencyGraph() - explicitModuleBuildHandler = - try ExplicitModuleBuildHandler(dependencyGraph: dependencyGraph, - toolchain: toolchain, - fileSystem: fileSystem) - return try explicitModuleBuildHandler!.generateExplicitModuleDependenciesBuildJobs() + public mutating func generateExplicitModuleDependenciesJobs() throws -> [Job] { + // Run the dependency scanner and update the dependency oracle with the results + let dependencyGraph = try gatherModuleDependencies(into: interModuleDependencyOracle) + + // Plan build jobs for all direct and transitive module dependencies of the current target + explicitDependencyBuildPlanner = + try ExplicitDependencyBuildPlanner(dependencyGraph: dependencyGraph, + toolchain: toolchain) + return try explicitDependencyBuildPlanner!.generateExplicitModuleDependenciesBuildJobs() } - private mutating func generateInterModuleDependencyGraph() throws -> InterModuleDependencyGraph { + private mutating func gatherModuleDependencies(into dependencyOracle: InterModuleDependencyOracle) + throws -> InterModuleDependencyGraph { let dependencyScannerJob = try dependencyScanningJob() let forceResponseFiles = parsedOptions.hasArgument(.driverForceResponseFiles) @@ -282,16 +401,21 @@ extension Driver { recordedInputModificationDates: recordedInputModificationDates) // Resolve placeholder dependencies in the dependency graph, if any. - if externalDependencyArtifactMap != nil, !externalDependencyArtifactMap!.isEmpty { - try dependencyGraph.resolvePlaceholderDependencies(using: externalDependencyArtifactMap!) + if externalBuildArtifacts != nil { + try dependencyGraph.resolvePlaceholderDependencies(for: externalBuildArtifacts!, + using: dependencyOracle) } // Re-scan Clang modules at all the targets they will be built against. + // TODO: Should be deprecated once switched over to libSwiftScan try resolveVersionedClangDependencies(dependencyGraph: &dependencyGraph) // Set dependency modules' paths to be saved in the module cache. try updateDependencyModulesWithModuleCachePath(dependencyGraph: &dependencyGraph) + // Update the dependency oracle, adding this new dependency graph to its store + try dependencyOracle.mergeModules(from: dependencyGraph) + return dependencyGraph } @@ -321,28 +445,24 @@ extension Driver { } } +} + +/// MARK: Planning +extension Driver { /// Create a job if needed for simple requests that can be immediately /// forwarded to the frontend. public mutating func immediateForwardingJob() throws -> Job? { if parsedOptions.hasArgument(.printTargetInfo) { let sdkPath = try parsedOptions.getLastArgument(.sdk).map { try VirtualPath(path: $0.asSingle) } let resourceDirPath = try parsedOptions.getLastArgument(.resourceDir).map { try VirtualPath(path: $0.asSingle) } - var useStaticResourceDir = false - if parsedOptions.hasFlag(positive: .staticExecutable, - negative: .noStaticExecutable, - default: false) || - parsedOptions.hasFlag(positive: .staticStdlib, - negative: .noStaticStdlib, - default: false) { - useStaticResourceDir = true - } - + return try toolchain.printTargetInfoJob(target: targetTriple, targetVariant: targetVariantTriple, sdkPath: sdkPath, resourceDirPath: resourceDirPath, requiresInPlaceExecution: true, - useStaticResourceDir: useStaticResourceDir) + useStaticResourceDir: useStaticResourceDir, + swiftCompilerPrefixArgs: swiftCompilerPrefixArgs) } if parsedOptions.hasArgument(.version) || parsedOptions.hasArgument(.version_) { @@ -352,6 +472,7 @@ extension Driver { tool: .absolute(try toolchain.getToolPath(.swiftCompiler)), commandLine: [.flag("--version")], inputs: [], + primaryInputs: [], outputs: [], requiresInPlaceExecution: true) } @@ -367,6 +488,7 @@ extension Driver { tool: .absolute(try toolchain.getToolPath(.swiftHelp)), commandLine: commandLine, inputs: [], + primaryInputs: [], outputs: [], requiresInPlaceExecution: true) } @@ -376,7 +498,6 @@ extension Driver { /// Plan a build by producing a set of jobs to complete the build. public mutating func planBuild() throws -> [Job] { - if let job = try immediateForwardingJob() { return [job] } @@ -421,11 +542,97 @@ extension Diagnostic.Message { // MARK: Batch mode extension Driver { + + /// Given some jobs, merge the compile jobs into batched jobs, as appropriate + /// While it may seem odd to create unbatched jobs, then later disect and rebatch them, + /// there are reasons for doing it this way: + /// 1. For incremental builds, the inputs compiled in the 2nd wave cannot be known in advance, and + /// 2. The code that creates a compile job intermixes command line formation, output gathering, etc. + /// It does this for good reason: these things are connected by consistency requirments, and + /// 3. The outputs of all compilations are needed, not just 1st wave ones, to feed as inputs to the link job. + /// + /// So, in order to avoid making jobs and rebatching, the code would have to just get outputs for each + /// compilation. But `compileJob` intermixes the output computation with other stuff. + mutating func formBatchedJobs(_ jobs: [Job], forIncremental: Bool) throws -> [Job] { + guard compilerMode.isBatchCompile else { + // Don't even go through the logic so as to not print out confusing + // "batched foobar" messages. + return jobs + } + let noncompileJobs = jobs.filter {$0.kind != .compile} + let compileJobs = jobs.filter {$0.kind == .compile} + let inputsAndJobs = compileJobs.flatMap { job in + job.primaryInputs.map {($0, job)} + } + let jobsByInput = Dictionary(uniqueKeysWithValues: inputsAndJobs) + // Try to preserve input order for easier testing + let inputsInOrder = inputFiles.filter {jobsByInput[$0] != nil} + + // For compatibility with swiftpm, the driver produces batched jobs + // for every job, even when run in incremental mode, so that all jobs + // can be returned from `planBuild`. + // But in that case, don't emit lifecycle messages. + let isIncrementalBuild = incrementalCompilationState != nil + let isNotPhoneyBaloneyBatching = isIncrementalBuild == forIncremental + + let partitions = batchPartitions( + inputs: inputsInOrder, + isNotPhoneyBaloneyBatching: isNotPhoneyBaloneyBatching) + let outputType = parsedOptions.hasArgument(.embedBitcode) + ? .llvmBitcode + : compilerOutputType + + let inputsRequiringModuleTrace = Set( + compileJobs.filter { $0.outputs.contains {$0.type == .moduleTrace} } + .flatMap {$0.primaryInputs} + ) + + let batchedCompileJobs = try inputsInOrder.compactMap { anInput -> Job? in + let idx = partitions.assignment[anInput]! + let primaryInputs = partitions.partitions[idx] + guard primaryInputs[0] == anInput + else { + // This input file isn't the first + // file in the partition, skip it: it's been accounted for already. + return nil + } + if showJobLifecycle && isNotPhoneyBaloneyBatching { + // Log life cycle for added batch job + primaryInputs.forEach { + diagnosticEngine + .emit( + .remark( + "Adding {compile: \($0.file.basename)} to batch \(idx)")) + } + + let constituents = primaryInputs.map {$0.file.basename}.joined(separator: ", ") + diagnosticEngine + .emit( + .remark( + "Forming batch job from \(primaryInputs.count) constituents: \(constituents)")) + } + let constituentsEmittedModuleTrace = !inputsRequiringModuleTrace.intersection(primaryInputs).isEmpty + // no need to add job outputs again + return try compileJob(primaryInputs: primaryInputs, + outputType: outputType, + addJobOutputs: {_ in }, + emitModuleTrace: constituentsEmittedModuleTrace) + } + return batchedCompileJobs + noncompileJobs + } + /// Determine the number of partitions we'll use for batch mode. private func numberOfBatchPartitions( - _ info: BatchModeInfo, - swiftInputFiles: [TypedVirtualPath] + _ info: BatchModeInfo?, + numInputFiles: Int ) -> Int { + guard numInputFiles > 0 else { + return 0 + } + guard let info = info else { + return 1 // not batch mode + } + // If the number of partitions was specified by the user, use it if let fixedCount = info.count { return fixedCount @@ -537,7 +744,6 @@ extension Driver { } let defaultSizeLimit = 25 - let numInputFiles = swiftInputFiles.count let sizeLimit = info.sizeLimit ?? defaultSizeLimit let numTasks = numParallelJobs ?? 1 @@ -554,39 +760,53 @@ extension Driver { let partitions: [[TypedVirtualPath]] } - /// Compute the partitions we'll use for batch mode. - private func batchPartitions(_ info: BatchModeInfo) -> BatchPartitions? { - let swiftInputFiles = inputFiles.filter { inputFile in - inputFile.type.isPartOfSwiftCompilation - } - let numPartitions = numberOfBatchPartitions(info, swiftInputFiles: swiftInputFiles) - - if parsedOptions.hasArgument(.driverShowJobLifecycle) { - stdoutStream.write("Found \(swiftInputFiles.count) batchable jobs\n") - stdoutStream.write("Forming into \(numPartitions) batches\n") - stdoutStream.flush() + private func batchPartitions( + inputs: [TypedVirtualPath], + isNotPhoneyBaloneyBatching: Bool + ) -> BatchPartitions { + let numScheduledPartitions = numberOfBatchPartitions( + compilerMode.batchModeInfo, + numInputFiles: inputs.count) + + if showJobLifecycle && inputs.count > 0 && isNotPhoneyBaloneyBatching { + diagnosticEngine + .emit( + .remark( + "Found \(inputs.count) batchable job\(inputs.count != 1 ? "s" : "")" + )) + diagnosticEngine + .emit( + .remark( + "Forming into \(numScheduledPartitions) batch\(numScheduledPartitions != 1 ? "es" : "")" + )) } - // If there is only one partition, fast path. - if numPartitions == 1 { + // If there is at most one partition, fast path. + if numScheduledPartitions <= 1 { var assignment = [TypedVirtualPath: Int]() - for input in swiftInputFiles { + for input in inputs { assignment[input] = 0 } - return BatchPartitions(assignment: assignment, partitions: [swiftInputFiles]) + let partitions = inputs.isEmpty ? [] : [inputs] + return BatchPartitions(assignment: assignment, + partitions: partitions) } // Map each input file to a partition index. Ensure that we evenly // distribute the remainder. - let numInputFiles = swiftInputFiles.count - let remainder = numInputFiles % numPartitions - let targetSize = numInputFiles / numPartitions + let numScheduledInputFiles = inputs.count + let remainder = numScheduledInputFiles % numScheduledPartitions + let targetSize = numScheduledInputFiles / numScheduledPartitions var partitionIndices: [Int] = [] - for partitionIdx in 0..(repeating: [], count: numPartitions) - for (fileIndex, file) in swiftInputFiles.enumerated() { + var partitions = Array<[TypedVirtualPath]>(repeating: [], count: numScheduledPartitions) + for (fileIndex, file) in inputs.enumerated() { let partitionIdx = partitionIndices[fileIndex] assignment[file] = partitionIdx partitions[partitionIdx].append(file) } - return BatchPartitions(assignment: assignment, partitions: partitions) + return BatchPartitions(assignment: assignment, + partitions: partitions) } } diff --git a/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift b/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift index de07e8dff..1666d8b42 100644 --- a/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift +++ b/Sources/SwiftDriver/Jobs/PrintTargetInfoJob.swift @@ -62,7 +62,8 @@ extension SwiftVersion: Codable { } /// Describes information about the target as provided by the Swift frontend. -public struct FrontendTargetInfo: Codable { +@dynamicMemberLookup +@_spi(Testing) public struct FrontendTargetInfo: Codable { struct CompatibilityLibrary: Codable { enum Filter: String, Codable { case all @@ -94,20 +95,51 @@ public struct FrontendTargetInfo: Codable { /// Whether the Swift libraries need to be referenced in their system /// location (/usr/lib/swift) via rpath. let librariesRequireRPath: Bool + + static func dummyForTesting(_ toolchain: Toolchain) -> Self { + let dummyForTestingTriple = Triple.dummyForTesting(toolchain) + return Self( + triple: dummyForTestingTriple, + unversionedTriple: dummyForTestingTriple, + moduleTriple: dummyForTestingTriple, + swiftRuntimeCompatibilityVersion: nil, + compatibilityLibraries: [], + librariesRequireRPath: false) + } } - struct Paths: Codable { + @_spi(Testing) public struct Paths: Codable { /// The path to the SDK, if provided. - let sdkPath: String? - let runtimeLibraryPaths: [String] - let runtimeLibraryImportPaths: [String] - let runtimeResourcePath: String + public let sdkPath: TextualVirtualPath? + public let runtimeLibraryPaths: [TextualVirtualPath] + public let runtimeLibraryImportPaths: [TextualVirtualPath] + public let runtimeResourcePath: TextualVirtualPath + + static let dummyForTesting = Paths( + sdkPath: nil, + runtimeLibraryPaths: [], + runtimeLibraryImportPaths: [], + runtimeResourcePath: .dummyForTesting) } var compilerVersion: String var target: Target var targetVariant: Target? let paths: Paths + + static func dummyForTesting(_ toolchain: Toolchain) -> Self { + Self(compilerVersion: "dummy", + target: .dummyForTesting(toolchain), + targetVariant: nil, + paths: .dummyForTesting) + } +} + +// Make members of `FrontendTargetInfo.Paths` accessible on `FrontendTargetInfo`. +extension FrontendTargetInfo { + @_spi(Testing) public subscript(dynamicMember dynamicMember: KeyPath) -> T { + self.paths[keyPath: dynamicMember] + } } extension Toolchain { @@ -117,9 +149,11 @@ extension Toolchain { resourceDirPath: VirtualPath? = nil, runtimeCompatibilityVersion: String? = nil, requiresInPlaceExecution: Bool = false, - useStaticResourceDir: Bool = false) throws -> Job { - var commandLine: [Job.ArgTemplate] = [.flag("-frontend"), - .flag("-print-target-info")] + useStaticResourceDir: Bool = false, + swiftCompilerPrefixArgs: [String]) throws -> Job { + var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } + commandLine.append(contentsOf: [.flag("-frontend"), + .flag("-print-target-info")]) // If we were given a target, include it. Otherwise, let the frontend // tell us the host target. if let target = target { @@ -157,6 +191,7 @@ extension Toolchain { commandLine: commandLine, displayInputs: [], inputs: [], + primaryInputs: [], outputs: [.init(file: .standardOutput, type: .jsonTargetInfo)], requiresInPlaceExecution: requiresInPlaceExecution, supportsResponseFiles: false diff --git a/Sources/SwiftDriver/Jobs/ReplJob.swift b/Sources/SwiftDriver/Jobs/ReplJob.swift index f82331d86..df6848b37 100644 --- a/Sources/SwiftDriver/Jobs/ReplJob.swift +++ b/Sources/SwiftDriver/Jobs/ReplJob.swift @@ -29,6 +29,7 @@ extension Driver { tool: .absolute(try toolchain.getToolPath(.lldb)), commandLine: [Job.ArgTemplate.flag(lldbArg)], inputs: inputs, + primaryInputs: [], outputs: [], requiresInPlaceExecution: true ) diff --git a/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift b/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift index ab860276e..67a2caa70 100644 --- a/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift +++ b/Sources/SwiftDriver/Jobs/Toolchain+InterpreterSupport.swift @@ -34,16 +34,16 @@ extension DarwinToolchain { public func platformSpecificInterpreterEnvironmentVariables( env: [String : String], parsedOptions: inout ParsedOptions, - sdkPath: String?, - targetTriple: Triple) throws -> [String: String] { + sdkPath: VirtualPath?, + targetInfo: FrontendTargetInfo) throws -> [String: String] { var envVars: [String: String] = [:] let runtimePaths = try runtimeLibraryPaths( - for: targetTriple, + for: targetInfo, parsedOptions: &parsedOptions, sdkPath: sdkPath, isShared: true - ).map { $0.pathString } + ).map { $0.name } addPathEnvironmentVariableIfNeeded("DYLD_LIBRARY_PATH", to: &envVars, currentEnv: env, option: .L, @@ -62,16 +62,16 @@ extension GenericUnixToolchain { public func platformSpecificInterpreterEnvironmentVariables( env: [String : String], parsedOptions: inout ParsedOptions, - sdkPath: String?, - targetTriple: Triple) throws -> [String: String] { + sdkPath: VirtualPath?, + targetInfo: FrontendTargetInfo) throws -> [String: String] { var envVars: [String: String] = [:] let runtimePaths = try runtimeLibraryPaths( - for: targetTriple, + for: targetInfo, parsedOptions: &parsedOptions, sdkPath: sdkPath, isShared: true - ).map { $0.pathString } + ).map { $0.name } addPathEnvironmentVariableIfNeeded("LD_LIBRARY_PATH", to: &envVars, currentEnv: env, option: .L, @@ -86,8 +86,8 @@ extension WindowsToolchain { public func platformSpecificInterpreterEnvironmentVariables( env: [String : String], parsedOptions: inout ParsedOptions, - sdkPath: String?, - targetTriple: Triple) throws -> [String: String] { + sdkPath: VirtualPath?, + targetInfo: FrontendTargetInfo) throws -> [String: String] { // TODO: See whether Windows needs `platformSpecificInterpreterEnvironmentVariables` return [:] } diff --git a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift index f495f09d5..79e00e88c 100644 --- a/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/Toolchain+LinkerSupport.swift @@ -2,7 +2,7 @@ // // This source file is part of the Swift.org open source project // -// Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors // Licensed under Apache License v2.0 with Runtime Library Exception // // See https://swift.org/LICENSE.txt for license information @@ -44,42 +44,29 @@ extension Toolchain { } return resourceDirBase.appending(components: triple.platformName() ?? "") } - - func computeSecondaryResourceDirPath(for triple: Triple, primaryPath: AbsolutePath) -> AbsolutePath? { + + func computeSecondaryResourceDirPath(for triple: Triple, primaryPath: VirtualPath) -> VirtualPath? { guard triple.isMacCatalyst else { return nil } return primaryPath.parentDirectory.appending(component: "macosx") } func clangLibraryPath( - for triple: Triple, + for targetInfo: FrontendTargetInfo, parsedOptions: inout ParsedOptions - ) throws -> AbsolutePath { - #if os(Windows) - return try getToolPath(.swiftCompiler) - .parentDirectory // remove /swift - .parentDirectory // remove /bin - .appending(components: "lib", "swift", "clang", "lib", - triple.platformName(conflatingDarwin: true)!) - #else - return try computeResourceDirPath(for: triple, - parsedOptions: &parsedOptions, - isShared: true) - .parentDirectory // Remove platform name. - .appending(components: "clang", "lib", - triple.platformName(conflatingDarwin: true)!) - #endif + ) throws -> VirtualPath { + return targetInfo.runtimeResourcePath.path + .appending(components: "clang", "lib", + targetInfo.target.triple.platformName(conflatingDarwin: true)!) } func runtimeLibraryPaths( - for triple: Triple, + for targetInfo: FrontendTargetInfo, parsedOptions: inout ParsedOptions, - sdkPath: String?, + sdkPath: VirtualPath?, isShared: Bool - ) throws -> [AbsolutePath] { - let resourceDirPath = try computeResourceDirPath( - for: triple, - parsedOptions: &parsedOptions, - isShared: isShared) + ) throws -> [VirtualPath] { + let triple = targetInfo.target.triple + let resourceDirPath = targetInfo.runtimeResourcePath.path.appending(component: triple.platformName() ?? "") var result = [resourceDirPath] let secondaryResourceDir = computeSecondaryResourceDirPath(for: triple, primaryPath: resourceDirPath) @@ -87,14 +74,17 @@ extension Toolchain { result.append(path) } - if let path = sdkPath { - let sdkPath = AbsolutePath(path) + if let sdkPath = sdkPath { // If we added the secondary resource dir, we also need the iOSSupport directory. if secondaryResourceDir != nil { result.append(sdkPath.appending(components: "System", "iOSSupport", "usr", "lib", "swift")) } - result.append(sdkPath.appending(RelativePath("usr/lib/swift"))) + if (triple.isWindows) { + result.append(sdkPath.appending(components: "usr", "lib", "swift", "windows")) + } else { + result.append(sdkPath.appending(components: "usr", "lib", "swift")) + } } return result @@ -103,11 +93,11 @@ extension Toolchain { func addLinkRuntimeLibrary( named name: String, to commandLine: inout [Job.ArgTemplate], - for triple: Triple, + for targetInfo: FrontendTargetInfo, parsedOptions: inout ParsedOptions ) throws { let path = try clangLibraryPath( - for: triple, + for: targetInfo, parsedOptions: &parsedOptions) .appending(component: name) commandLine.appendPath(path) @@ -115,20 +105,20 @@ extension Toolchain { func runtimeLibraryExists( for sanitizer: Sanitizer, - targetTriple: Triple, + targetInfo: FrontendTargetInfo, parsedOptions: inout ParsedOptions, isShared: Bool ) throws -> Bool { let runtimeName = try runtimeLibraryName( for: sanitizer, - targetTriple: targetTriple, + targetTriple: targetInfo.target.triple, isShared: isShared ) let path = try clangLibraryPath( - for: targetTriple, + for: targetInfo, parsedOptions: &parsedOptions ).appending(component: runtimeName) - return fileSystem.exists(path) + return try fileSystem.exists(path) } } @@ -138,7 +128,6 @@ extension DarwinToolchain { func addArgsToLinkStdlib( to commandLine: inout [Job.ArgTemplate], parsedOptions: inout ParsedOptions, - sdkPath: String?, targetInfo: FrontendTargetInfo, linkerOutputType: LinkOutputType, fileSystem: FileSystem @@ -147,12 +136,10 @@ extension DarwinToolchain { // Link compatibility libraries, if we're deploying back to OSes that // have an older Swift runtime. - let resourceDirPath = try computeResourceDirPath(for: targetTriple, - parsedOptions: &parsedOptions, - isShared: true) - func addArgsForBackDeployLib(_ libName: String) { - let backDeployLibPath = resourceDirPath.appending(component: libName) - if fileSystem.exists(backDeployLibPath) { + func addArgsForBackDeployLib(_ libName: String) throws { + let backDeployLibPath = targetInfo.runtimeResourcePath.path + .appending(components: targetTriple.platformName() ?? "", libName) + if try fileSystem.exists(backDeployLibPath) { commandLine.append(.flag("-force_load")) commandLine.appendPath(backDeployLibPath) } @@ -170,16 +157,16 @@ extension DarwinToolchain { } if shouldLink { - addArgsForBackDeployLib("lib" + compatibilityLib.libraryName + ".a") + try addArgsForBackDeployLib("lib" + compatibilityLib.libraryName + ".a") } } // Add the runtime library link path, which is platform-specific and found // relative to the compiler. let runtimePaths = try runtimeLibraryPaths( - for: targetTriple, + for: targetInfo, parsedOptions: &parsedOptions, - sdkPath: sdkPath, + sdkPath: targetInfo.sdkPath?.path, isShared: true ) for path in runtimePaths { @@ -259,12 +246,12 @@ extension DarwinToolchain { } } - func paths(runtimeLibraryPaths: [AbsolutePath]) -> [AbsolutePath] { + func paths(runtimeLibraryPaths: [VirtualPath]) -> [VirtualPath] { switch self { case .toolchain: return runtimeLibraryPaths case .os: - return [AbsolutePath("/usr/lib/swift")] + return [.absolute(.init("/usr/lib/swift"))] case .none: return [] } diff --git a/Sources/SwiftDriver/Jobs/VerifyDebugInfoJob.swift b/Sources/SwiftDriver/Jobs/VerifyDebugInfoJob.swift index 33705d7dd..86268cbb4 100644 --- a/Sources/SwiftDriver/Jobs/VerifyDebugInfoJob.swift +++ b/Sources/SwiftDriver/Jobs/VerifyDebugInfoJob.swift @@ -27,6 +27,7 @@ extension Driver { commandLine: commandLine, displayInputs: [], inputs: inputs, + primaryInputs: [], outputs: [] ) } diff --git a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift index 93d876e09..ecd816bfa 100644 --- a/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift +++ b/Sources/SwiftDriver/Jobs/VerifyModuleInterfaceJob.swift @@ -13,8 +13,9 @@ extension Driver { mutating func verifyModuleInterfaceJob(interfaceInput: TypedVirtualPath) throws -> Job { var commandLine: [Job.ArgTemplate] = swiftCompilerPrefixArgs.map { Job.ArgTemplate.flag($0) } - var inputs: [TypedVirtualPath] = [] + var inputs: [TypedVirtualPath] = [interfaceInput] commandLine.appendFlags("-frontend", "-typecheck-module-from-interface") + commandLine.appendPath(interfaceInput.file) try addCommonFrontendOptions(commandLine: &commandLine, inputs: &inputs) // FIXME: MSVC runtime flags @@ -32,8 +33,9 @@ extension Driver { kind: .verifyModuleInterface, tool: .absolute(try toolchain.getToolPath(.swiftCompiler)), commandLine: commandLine, - displayInputs: [], - inputs: [interfaceInput], + displayInputs: [interfaceInput], + inputs: inputs, + primaryInputs: [], outputs: [outputFile] ) } diff --git a/Sources/SwiftDriver/Jobs/WebAssemblyToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/WebAssemblyToolchain+LinkerSupport.swift new file mode 100644 index 000000000..1c07f6632 --- /dev/null +++ b/Sources/SwiftDriver/Jobs/WebAssemblyToolchain+LinkerSupport.swift @@ -0,0 +1,161 @@ +//===--------------- WebAssemblyToolchain+LinkerSupport.swift -------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import TSCBasic +import SwiftOptions + +extension WebAssemblyToolchain { + public func addPlatformSpecificLinkerArgs( + to commandLine: inout [Job.ArgTemplate], + parsedOptions: inout ParsedOptions, + linkerOutputType: LinkOutputType, + inputs: [TypedVirtualPath], + outputFile: VirtualPath, + shouldUseInputFileList: Bool, + lto: LTOKind?, + sanitizers: Set, + targetInfo: FrontendTargetInfo + ) throws -> AbsolutePath { + let targetTriple = targetInfo.target.triple + switch linkerOutputType { + case .dynamicLibrary: + throw Error.dynamicLibrariesUnsupportedForTarget(targetTriple.triple) + case .executable: + if !targetTriple.triple.isEmpty { + commandLine.appendFlag("-target") + commandLine.appendFlag(targetTriple.triple) + } + + // Select the linker to use. + if let linkerArg = parsedOptions.getLastArgument(.useLd)?.asSingle { + commandLine.appendFlag("-fuse-ld=\(linkerArg)") + } + + // Configure the toolchain. + // + // By default use the system `clang` to perform the link. We use `clang` for + // the driver here because we do not wish to select a particular C++ runtime. + // Furthermore, until C++ interop is enabled, we cannot have a dependency on + // C++ code from pure Swift code. If linked libraries are C++ based, they + // should properly link C++. In the case of static linking, the user can + // explicitly specify the C++ runtime to link against. This is particularly + // important for platforms like android where as it is a Linux platform, the + // default C++ runtime is `libstdc++` which is unsupported on the target but + // as the builds are usually cross-compiled from Linux, libstdc++ is going to + // be present. This results in linking the wrong version of libstdc++ + // generating invalid binaries. It is also possible to use different C++ + // runtimes than the default C++ runtime for the platform (e.g. libc++ on + // Windows rather than msvcprt). When C++ interop is enabled, we will need to + // surface this via a driver flag. For now, opt for the simpler approach of + // just using `clang` and avoid a dependency on the C++ runtime. + var clangPath = try getToolPath(.clang) + if let toolsDirPath = parsedOptions.getLastArgument(.toolsDirectory) { + // FIXME: What if this isn't an absolute path? + let toolsDir = try AbsolutePath(validating: toolsDirPath.asSingle) + + // If there is a clang in the toolchain folder, use that instead. + if let tool = lookupExecutablePath(filename: "clang", searchPaths: [toolsDir]) { + clangPath = tool + } + } + + guard !parsedOptions.hasArgument(.noStaticStdlib, .noStaticExecutable) else { + throw Error.dynamicLibrariesUnsupportedForTarget(targetTriple.triple) + } + + let runtimePaths = try runtimeLibraryPaths( + for: targetInfo, + parsedOptions: &parsedOptions, + sdkPath: targetInfo.sdkPath?.path, + isShared: false + ) + + let swiftrtPath = targetInfo.runtimeResourcePath.path + .appending( + components: targetTriple.platformName() ?? "", + targetTriple.archName, + "swiftrt.o" + ) + commandLine.appendPath(swiftrtPath) + + let inputFiles: [Job.ArgTemplate] = inputs.compactMap { input in + // Autolink inputs are handled specially + if input.type == .autolink { + return .responseFilePath(input.file) + } else if input.type == .object { + return .path(input.file) + } else { + return nil + } + } + commandLine.append(contentsOf: inputFiles) + + if let path = targetInfo.sdkPath?.path { + commandLine.appendFlag("--sysroot") + commandLine.appendPath(path) + } + + // Add the runtime library link paths. + for path in runtimePaths { + commandLine.appendFlag(.L) + commandLine.appendPath(path) + } + + // Link the standard library. + let linkFilePath: VirtualPath = targetInfo.runtimeResourcePath.path + .appending( + components: targetTriple.platformName() ?? "", + "static-executable-args.lnk" + ) + + guard try fileSystem.exists(linkFilePath) else { + fatalError("\(linkFilePath) not found") + } + commandLine.append(.responseFilePath(linkFilePath)) + + // Explicitly pass the target to the linker + commandLine.appendFlag("--target=\(targetTriple.triple)") + + // Delegate to Clang for sanitizers. It will figure out the correct linker + // options. + guard sanitizers.isEmpty else { + fatalError("WebAssembly does not support sanitizers, but a runtime library was found") + } + + guard !parsedOptions.hasArgument(.profileGenerate) else { + throw Error.profilingUnsupportedForTarget(targetTriple.triple) + } + + // Run clang++ in verbose mode if "-v" is set + try commandLine.appendLast(.v, from: &parsedOptions) + + // These custom arguments should be right before the object file at the + // end. + try commandLine.append( + contentsOf: parsedOptions.arguments(in: .linkerOption) + ) + try commandLine.appendAllArguments(.Xlinker, from: &parsedOptions) + try commandLine.appendAllArguments(.XclangLinker, from: &parsedOptions) + + // This should be the last option, for convenience in checking output. + commandLine.appendFlag(.o) + commandLine.appendPath(outputFile) + return clangPath + case .staticLibrary: + // We're using 'ar' as a linker + commandLine.appendFlag("crs") + commandLine.appendPath(outputFile) + + commandLine.append(contentsOf: inputs.map { .path($0.file) }) + return try getToolPath(.staticLinker(lto)) + } + } +} diff --git a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift index 02a75498d..11e1f9a2d 100644 --- a/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift +++ b/Sources/SwiftDriver/Jobs/WindowsToolchain+LinkerSupport.swift @@ -20,7 +20,7 @@ extension WindowsToolchain { inputs: [TypedVirtualPath], outputFile: VirtualPath, shouldUseInputFileList: Bool, - sdkPath: String?, + lto: LTOKind?, sanitizers: Set, targetInfo: FrontendTargetInfo ) throws -> AbsolutePath { @@ -75,18 +75,16 @@ extension WindowsToolchain { } commandLine.appendFlag("-fuse-ld=\(linker)") - // FIXME: Do we really need `oldnames`? - commandLine.appendFlags("-autolink-library", "oldnames") if let crt = parsedOptions.getLastArgument(.libc) { switch crt.asSingle { - case "MT": commandLine.appendFlags("-autolink-library", "libcmt") - case "MTd": commandLine.appendFlags("-autolink-library", "libcmtd") - case "MD": commandLine.appendFlags("-autolink-library", "msvcrt") - case "MDd": commandLine.appendFlags("-autolink-library", "msvcrtd") + case "MT": commandLine.appendFlags("-Xlinker", "-defaultlib:libcmt") + case "MTd": commandLine.appendFlags("-Xlinker", "-defaultlib:libcmtd") + case "MD": commandLine.appendFlags("-Xlinker", "-defaultlib:msvcrt") + case "MDd": commandLine.appendFlags("-Xlinker", "-defaultlib:msvcrtd") default: fatalError("Invalid C runtime value should be filtered") } } else { - commandLine.appendFlags("-autolink-library", "msvcrt") + commandLine.appendFlags("-Xlinker", "-defaultlib:msvcrt") } let staticStdlib = parsedOptions.hasFlag(positive: .staticStdlib, @@ -98,9 +96,9 @@ extension WindowsToolchain { let hasRuntimeArgs = !(staticStdlib || staticExecutable) let runtimePaths = try runtimeLibraryPaths( - for: targetTriple, + for: targetInfo, parsedOptions: &parsedOptions, - sdkPath: sdkPath, + sdkPath: targetInfo.sdkPath?.path, isShared: hasRuntimeArgs ) @@ -111,7 +109,7 @@ extension WindowsToolchain { ) let swiftrtPath = sharedResourceDirPath.appending( - components: targetTriple.archName, "swiftrt.obj" + components: archName(for: targetTriple), "swiftrt.obj" ) commandLine.appendPath(swiftrtPath) @@ -140,7 +138,7 @@ extension WindowsToolchain { // Add the runtime library link paths. for path in runtimePaths { commandLine.appendFlag(.L) - commandLine.appendPath(path) + commandLine.appendPath(path.appending(component: archName(for: targetTriple))) } if hasRuntimeArgs { @@ -184,7 +182,7 @@ extension WindowsToolchain { case .staticLibrary: commandLine.append(.joinedOptionAndPath("-out:", outputFile)) commandLine.append(contentsOf: inputs.map { .path($0.file) }) - return try getToolPath(.staticLinker) + return try getToolPath(.staticLinker(lto)) } } } diff --git a/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift b/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift index 9469bc3f1..b4701399b 100644 --- a/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/DarwinToolchain.swift @@ -17,7 +17,7 @@ import SwiftOptions /// Toolchain for Darwin-based platforms, such as macOS and iOS. /// /// FIXME: This class is not thread-safe. -public final class DarwinToolchain: Toolchain { +@_spi(Testing) public final class DarwinToolchain: Toolchain { public let env: [String: String] /// The executor used to run processes used to find tools and retrieve target info. @@ -29,10 +29,16 @@ public final class DarwinToolchain: Toolchain { /// Doubles as path cache and point for overriding normal lookup private var toolPaths = [Tool: AbsolutePath]() - public init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem = localFileSystem) { + // An externally provided path from where we should find tools like ld + public let toolDirectory: AbsolutePath? + + public let dummyForTestingObjectFormat = Triple.ObjectFormat.macho + + public init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem = localFileSystem, toolDirectory: AbsolutePath? = nil) { self.env = env self.executor = executor self.fileSystem = fileSystem + self.toolDirectory = toolDirectory } /// Retrieve the absolute path for a given tool. @@ -290,7 +296,7 @@ public final class DarwinToolchain: Toolchain { inputs: inout [TypedVirtualPath], frontendTargetInfo: FrontendTargetInfo ) throws { - guard let sdkPath = try frontendTargetInfo.paths.sdkPath.map(VirtualPath.init(path:)), + guard let sdkPath = frontendTargetInfo.sdkPath?.path, let sdkInfo = getTargetSDKInfo(sdkPath: sdkPath) else { return } commandLine.append(.flag("-target-sdk-version")) diff --git a/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift b/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift index 09ca4d00a..3e30cb856 100644 --- a/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/GenericUnixToolchain.swift @@ -12,7 +12,7 @@ import TSCBasic /// Toolchain for Unix-like systems. -public final class GenericUnixToolchain: Toolchain { +@_spi(Testing) public final class GenericUnixToolchain: Toolchain { public let env: [String: String] /// The executor used to run processes used to find tools and retrieve target info. @@ -24,10 +24,15 @@ public final class GenericUnixToolchain: Toolchain { /// Doubles as path cache and point for overriding normal lookup private var toolPaths = [Tool: AbsolutePath]() - public init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem = localFileSystem) { + public let toolDirectory: AbsolutePath? + + public let dummyForTestingObjectFormat = Triple.ObjectFormat.elf + + public init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem = localFileSystem, toolDirectory: AbsolutePath? = nil) { self.env = env self.executor = executor self.fileSystem = fileSystem + self.toolDirectory = toolDirectory } public func makeLinkerOutputFilename(moduleName: String, type: LinkOutputType) -> String { @@ -54,8 +59,11 @@ public final class GenericUnixToolchain: Toolchain { switch tool { case .swiftCompiler: return try lookup(executable: "swift-frontend") - case .staticLinker: + case .staticLinker(nil): return try lookup(executable: "ar") + case .staticLinker(.llvmFull), + .staticLinker(.llvmThin): + return try lookup(executable: "llvm-ar") case .dynamicLinker: // FIXME: This needs to look in the tools_directory first. return try lookup(executable: "clang") diff --git a/Sources/SwiftDriver/Toolchains/Toolchain.swift b/Sources/SwiftDriver/Toolchains/Toolchain.swift index ebfe27107..4ebd2715f 100644 --- a/Sources/SwiftDriver/Toolchains/Toolchain.swift +++ b/Sources/SwiftDriver/Toolchains/Toolchain.swift @@ -13,9 +13,9 @@ import Foundation import TSCBasic import SwiftOptions -public enum Tool { +public enum Tool: Hashable { case swiftCompiler - case staticLinker + case staticLinker(LTOKind?) case dynamicLinker case clang case swiftAutolinkExtract @@ -27,8 +27,8 @@ public enum Tool { /// Describes a toolchain, which includes information about compilers, linkers /// and other tools required to build Swift code. -public protocol Toolchain { - init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem) +@_spi(Testing) public protocol Toolchain { + init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem, toolDirectory: AbsolutePath?) var env: [String: String] { get } @@ -38,6 +38,8 @@ public protocol Toolchain { var executor: DriverExecutor { get } + var toolDirectory: AbsolutePath? { get } + /// Retrieve the absolute path to a particular tool. func getToolPath(_ tool: Tool) throws -> AbsolutePath @@ -66,7 +68,7 @@ public protocol Toolchain { inputs: [TypedVirtualPath], outputFile: VirtualPath, shouldUseInputFileList: Bool, - sdkPath: String?, + lto: LTOKind?, sanitizers: Set, targetInfo: FrontendTargetInfo ) throws -> AbsolutePath @@ -80,14 +82,16 @@ public protocol Toolchain { func platformSpecificInterpreterEnvironmentVariables( env: [String: String], parsedOptions: inout ParsedOptions, - sdkPath: String?, - targetTriple: Triple) throws -> [String: String] + sdkPath: VirtualPath?, + targetInfo: FrontendTargetInfo) throws -> [String: String] func addPlatformSpecificCommonFrontendOptions( commandLine: inout [Job.ArgTemplate], inputs: inout [TypedVirtualPath], frontendTargetInfo: FrontendTargetInfo ) throws + + var dummyForTestingObjectFormat: Triple.ObjectFormat {get} } extension Toolchain { @@ -137,6 +141,10 @@ extension Toolchain { #endif if let overrideString = envVar(forExecutable: executable) { return try AbsolutePath(validating: overrideString) + } else if let toolDir = toolDirectory, + let path = lookupExecutablePath(filename: filename, searchPaths: [toolDir]) { + // Looking for tools from the tools directory. + return path } else if let path = lookupExecutablePath(filename: filename, searchPaths: [executableDir]) { return path } else if let path = try? xcrunFind(executable: executable) { @@ -184,6 +192,6 @@ extension Toolchain { ) throws {} } -public enum ToolchainError: Swift.Error { +@_spi(Testing) public enum ToolchainError: Swift.Error { case unableToFind(tool: String) } diff --git a/Sources/SwiftDriver/Toolchains/WebAssemblyToolchain.swift b/Sources/SwiftDriver/Toolchains/WebAssemblyToolchain.swift new file mode 100644 index 000000000..9b3814ba3 --- /dev/null +++ b/Sources/SwiftDriver/Toolchains/WebAssemblyToolchain.swift @@ -0,0 +1,134 @@ +//===-------- WebAssemblyToolchain.swift - Swift WASM Toolchain -----------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +import TSCBasic +import SwiftOptions + +/// Toolchain for WebAssembly-based systems. +@_spi(Testing) public final class WebAssemblyToolchain: Toolchain { + @_spi(Testing) public enum Error: Swift.Error, DiagnosticData { + case interactiveModeUnsupportedForTarget(String) + case dynamicLibrariesUnsupportedForTarget(String) + case sanitizersUnsupportedForTarget(String) + case profilingUnsupportedForTarget(String) + + public var description: String { + switch self { + case .interactiveModeUnsupportedForTarget(let triple): + return "interactive mode is unsupported for target '\(triple)'; use 'swiftc' instead" + case .dynamicLibrariesUnsupportedForTarget(let triple): + return "dynamic libraries are unsupported for target '\(triple)'" + case .sanitizersUnsupportedForTarget(let triple): + return "sanitizers are unsupported for target '\(triple)'" + case .profilingUnsupportedForTarget(let triple): + return "profiling is unsupported for target '\(triple)'" + } + } + } + + public let env: [String: String] + + /// The executor used to run processes used to find tools and retrieve target info. + public let executor: DriverExecutor + + /// The file system to use for queries. + public let fileSystem: FileSystem + + /// Doubles as path cache and point for overriding normal lookup + private var toolPaths = [Tool: AbsolutePath]() + + public let toolDirectory: AbsolutePath? + + public let dummyForTestingObjectFormat = Triple.ObjectFormat.wasm + + public init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem = localFileSystem, toolDirectory: AbsolutePath? = nil) { + self.env = env + self.executor = executor + self.fileSystem = fileSystem + self.toolDirectory = toolDirectory + } + + public func makeLinkerOutputFilename(moduleName: String, type: LinkOutputType) -> String { + switch type { + case .executable: + return moduleName + case .dynamicLibrary: + // WASM doesn't support dynamic libraries yet, but we'll report the error later. + return "" + case .staticLibrary: + return "lib\(moduleName).a" + } + } + + /// Retrieve the absolute path for a given tool. + public func getToolPath(_ tool: Tool) throws -> AbsolutePath { + // Check the cache + if let toolPath = toolPaths[tool] { + return toolPath + } + let path = try lookupToolPath(tool) + // Cache the path + toolPaths[tool] = path + return path + } + + private func lookupToolPath(_ tool: Tool) throws -> AbsolutePath { + switch tool { + case .swiftCompiler: + return try lookup(executable: "swift-frontend") + case .staticLinker(nil): + return try lookup(executable: "ar") + case .staticLinker(.llvmFull), + .staticLinker(.llvmThin): + return try lookup(executable: "llvm-ar") + case .dynamicLinker: + // FIXME: This needs to look in the tools_directory first. + return try lookup(executable: "clang") + case .clang: + return try lookup(executable: "clang") + case .swiftAutolinkExtract: + return try lookup(executable: "swift-autolink-extract") + case .dsymutil: + return try lookup(executable: "dsymutil") + case .lldb: + return try lookup(executable: "lldb") + case .dwarfdump: + return try lookup(executable: "dwarfdump") + case .swiftHelp: + return try lookup(executable: "swift-help") + } + } + + public func overrideToolPath(_ tool: Tool, path: AbsolutePath) { + toolPaths[tool] = path + } + + public func defaultSDKPath(_ target: Triple?) throws -> AbsolutePath? { + return nil + } + + public var shouldStoreInvocationInDebugInfo: Bool { false } + + public func runtimeLibraryName( + for sanitizer: Sanitizer, + targetTriple: Triple, + isShared: Bool + ) throws -> String { + throw Error.sanitizersUnsupportedForTarget(targetTriple.triple) + } + + public func platformSpecificInterpreterEnvironmentVariables(env: [String : String], + parsedOptions: inout ParsedOptions, + sdkPath: VirtualPath?, + targetInfo: FrontendTargetInfo) throws -> [String : String] { + throw Error.interactiveModeUnsupportedForTarget(targetInfo.target.triple.triple) + } +} diff --git a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift index 1659e9ca4..3b32f0aba 100644 --- a/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift +++ b/Sources/SwiftDriver/Toolchains/WindowsToolchain.swift @@ -13,7 +13,7 @@ import TSCBasic import SwiftOptions /// Toolchain for Windows. -public final class WindowsToolchain: Toolchain { +@_spi(Testing) public final class WindowsToolchain: Toolchain { public let env: [String: String] /// The executor used to run processes used to find tools and retrieve target info. @@ -24,6 +24,11 @@ public final class WindowsToolchain: Toolchain { /// Doubles as path cache and point for overriding normal lookup private var toolPaths = [Tool: AbsolutePath]() + + // An externally provided path from where we should find tools like ld + public let toolDirectory: AbsolutePath? + + public let dummyForTestingObjectFormat = Triple.ObjectFormat.coff public func archName(for triple: Triple) -> String { switch triple.arch { @@ -35,19 +40,14 @@ public final class WindowsToolchain: Toolchain { } } - public init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem = localFileSystem) { - self.env = env - self.executor = executor - self.fileSystem = fileSystem - } - - public func makeLinkerOutputFilename(moduleName: String, type: LinkOutputType) -> String { - switch type { - case .executable: return "\(moduleName).exe" - case .dynamicLibrary: return "\(moduleName).dll" - case .staticLibrary: return "lib\(moduleName).lib" + public init(env: [String: String], executor: DriverExecutor, fileSystem: FileSystem = localFileSystem, toolDirectory: AbsolutePath? = nil) { + self.env = env + self.executor = executor + self.fileSystem = fileSystem + self.toolDirectory = toolDirectory } - } + + /// Retrieve the absolute path for a given tool. public func getToolPath(_ tool: Tool) throws -> AbsolutePath { @@ -88,6 +88,19 @@ public final class WindowsToolchain: Toolchain { public func overrideToolPath(_ tool: Tool, path: AbsolutePath) { toolPaths[tool] = path } + + /// Path to the StdLib inside the SDK. + public func sdkStdlib(sdk: AbsolutePath, triple: Triple) -> AbsolutePath { + sdk.appending(RelativePath("usr/lib/swift/windows")).appending(component: archName(for: triple)) + } + + public func makeLinkerOutputFilename(moduleName: String, type: LinkOutputType) -> String { + switch type { + case .executable: return "\(moduleName).exe" + case .dynamicLibrary: return "\(moduleName).dll" + case .staticLibrary: return "lib\(moduleName).lib" + } + } public func defaultSDKPath(_ target: Triple?) throws -> AbsolutePath? { return nil diff --git a/Sources/SwiftDriver/Utilities/Bits.swift b/Sources/SwiftDriver/Utilities/Bits.swift deleted file mode 100644 index e2b9e5185..000000000 --- a/Sources/SwiftDriver/Utilities/Bits.swift +++ /dev/null @@ -1,98 +0,0 @@ -//===----------------- Bits.swift - Collection of bits --------------------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2020 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import Foundation - -struct Bits: RandomAccessCollection { - var buffer: Data - - var startIndex: Int { return 0 } - var endIndex: Int { return buffer.count * 8 } - - subscript(index: Int) -> UInt8 { - let byte = buffer[index / 8] - return (byte >> UInt8(index % 8)) & 1 - } - - func readBits(atOffset offset: Int, count: Int) -> UInt64 { - precondition(count >= 0 && count <= 64) - precondition(offset >= 0) - precondition(offset &+ count >= offset) - precondition(offset &+ count <= self.endIndex) - - return buffer.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in - let upperBound = offset &+ count - let topByteIndex = upperBound >> 3 - var result: UInt64 = 0 - if upperBound & 7 != 0 { - let mask: UInt8 = (1 << UInt8(upperBound & 7)) &- 1 - result = UInt64(bytes[topByteIndex] & mask) - } - for i in ((offset >> 3)..<(upperBound >> 3)).reversed() { - result <<= 8 - result |= UInt64(bytes[i]) - } - if offset & 7 != 0 { - result >>= UInt64(offset & 7) - } - return result - } - } - - struct Cursor { - enum Error: Swift.Error { case bufferOverflow } - - let buffer: Bits - private var offset: Int = 0 - - init(buffer: Bits) { - self.buffer = buffer - } - - init(buffer: Data) { - self.init(buffer: Bits(buffer: buffer)) - } - - var isAtEnd: Bool { - return offset == buffer.count - } - - func peek(_ count: Int) throws -> UInt64 { - if buffer.count - offset < count { throw Error.bufferOverflow } - return buffer.readBits(atOffset: offset, count: count) - } - - mutating func read(_ count: Int) throws -> UInt64 { - defer { offset += count } - return try peek(count) - } - - mutating func read(bytes count: Int) throws -> Data { - precondition(count >= 0) - precondition(offset & 0b111 == 0) - let newOffset = offset &+ (count << 3) - precondition(newOffset >= offset) - if newOffset > buffer.count { throw Error.bufferOverflow } - defer { offset = newOffset } - return buffer.buffer.dropFirst(offset >> 3).prefix((newOffset - offset) >> 3) - } - - mutating func advance(toBitAlignment align: Int) throws { - precondition(align > 0) - precondition(offset &+ (align&-1) >= offset) - precondition(align & (align &- 1) == 0) - if offset % align == 0 { return } - offset = (offset &+ align) & ~(align &- 1) - if offset > buffer.count { throw Error.bufferOverflow } - } - } -} diff --git a/Sources/SwiftDriver/Utilities/Bitstream.swift b/Sources/SwiftDriver/Utilities/Bitstream.swift deleted file mode 100644 index e3786251c..000000000 --- a/Sources/SwiftDriver/Utilities/Bitstream.swift +++ /dev/null @@ -1,371 +0,0 @@ -//===---- Bitstream.swift - LLVM Bitstream Container Format Reader --------===// -// -// This source file is part of the Swift.org open source project -// -// Copyright (c) 2020 Apple Inc. and the Swift project authors -// Licensed under Apache License v2.0 with Runtime Library Exception -// -// See https://swift.org/LICENSE.txt for license information -// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -// -//===----------------------------------------------------------------------===// - -import Foundation - -enum BitcodeElement { - struct Block { - var id: UInt64 - var elements: [BitcodeElement] - } - - struct Record { - enum Payload { - case none - case array([UInt64]) - case char6String(String) - case blob(Data) - } - - var id: UInt64 - var fields: [UInt64] - var payload: Payload - } - - case block(Block) - case record(Record) -} - -struct BlockInfo { - var name: String = "" - var recordNames: [UInt64: String] = [:] -} - -extension Bitcode { - struct Signature: Equatable { - private var value: UInt32 - - init(value: UInt32) { - self.value = value - } - - init(string: String) { - precondition(string.utf8.count == 4) - var result: UInt32 = 0 - for byte in string.utf8.reversed() { - result <<= 8 - result |= UInt32(byte) - } - self.value = result - } - } -} - -struct Bitcode { - let signature: Signature - let elements: [BitcodeElement] - let blockInfo: [UInt64: BlockInfo] -} - -private extension Bits.Cursor { - enum BitcodeError: Swift.Error { - case vbrOverflow - } - - mutating func readVBR(_ width: Int) throws -> UInt64 { - precondition(width > 1) - let testBit = UInt64(1 << (width &- 1)) - let mask = testBit &- 1 - - var result: UInt64 = 0 - var offset: UInt64 = 0 - var next: UInt64 - repeat { - next = try self.read(width) - result |= (next & mask) << offset - offset += UInt64(width &- 1) - if offset > 64 { throw BitcodeError.vbrOverflow } - } while next & testBit != 0 - - return result - } -} - -private struct BitstreamReader { - struct Abbrev { - enum Operand { - case literal(UInt64) - case fixed(Int) - case vbr(Int) - indirect case array(Operand) - case char6 - case blob - - var isPayload: Bool { - switch self { - case .array, .blob: return true - case .literal, .fixed, .vbr, .char6: return false - } - } - } - - var operands: [Operand] = [] - } - - enum Error: Swift.Error { - case invalidAbbrev - case nestedBlockInBlockInfo - case missingSETBID - case invalidBlockInfoRecord(recordID: UInt64) - case abbrevWidthTooSmall(width: Int) - case noSuchAbbrev(blockID: UInt64, abbrevID: Int) - case missingEndBlock(blockID: UInt64) - } - - var cursor: Bits.Cursor - var blockInfo: [UInt64: BlockInfo] = [:] - var globalAbbrevs: [UInt64: [Abbrev]] = [:] - - init(buffer: Data) { - cursor = Bits.Cursor(buffer: buffer) - } - - mutating func readAbbrevOp() throws -> Abbrev.Operand { - let isLiteralFlag = try cursor.read(1) - if isLiteralFlag == 1 { - return .literal(try cursor.readVBR(8)) - } - - switch try cursor.read(3) { - case 0: - throw Error.invalidAbbrev - case 1: - return .fixed(Int(try cursor.readVBR(5))) - case 2: - return .vbr(Int(try cursor.readVBR(5))) - case 3: - return .array(try readAbbrevOp()) - case 4: - return .char6 - case 5: - return .blob - case 6, 7: - throw Error.invalidAbbrev - default: - fatalError() - } - } - - mutating func readAbbrev(numOps: Int) throws -> Abbrev { - guard numOps > 0 else { throw Error.invalidAbbrev } - - var operands: [Abbrev.Operand] = [] - for i in 0.. UInt64 { - switch operand { - case .char6: - let value = try cursor.read(6) - switch value { - case 0...25: - return value + UInt64(("a" as UnicodeScalar).value) - case 26...51: - return value + UInt64(("A" as UnicodeScalar).value) - 26 - case 52...61: - return value + UInt64(("0" as UnicodeScalar).value) - 52 - case 62: - return UInt64(("." as UnicodeScalar).value) - case 63: - return UInt64(("_" as UnicodeScalar).value) - default: - fatalError() - } - case .literal(let value): - return value - case .fixed(let width): - return try cursor.read(width) - case .vbr(let width): - return try cursor.readVBR(width) - case .array, .blob: - fatalError() - } - } - - mutating func readAbbreviatedRecord(_ abbrev: Abbrev) throws -> BitcodeElement.Record { - let code = try readSingleAbbreviatedRecordOperand(abbrev.operands.first!) - - let lastOperand = abbrev.operands.last! - let lastRegularOperandIndex: Int = abbrev.operands.endIndex - (lastOperand.isPayload ? 1 : 0) - - var fields = [UInt64]() - for op in abbrev.operands[1.." - case 3: - guard let blockID = currentBlockID else { - throw Error.missingSETBID - } - if blockInfo[blockID] == nil { blockInfo[blockID] = BlockInfo() } - guard let recordID = operands.first else { - throw Error.invalidBlockInfoRecord(recordID: code) - } - blockInfo[blockID]!.recordNames[recordID] = String(bytes: operands.dropFirst().map { UInt8($0) }, encoding: .utf8) ?? "" - default: - throw Error.invalidBlockInfoRecord(recordID: code) - } - - case let abbrevID: - throw Error.noSuchAbbrev(blockID: 0, abbrevID: Int(abbrevID)) - } - } - } - - mutating func readBlock(id: UInt64, abbrevWidth: Int, abbrevInfo: [Abbrev]) throws -> [BitcodeElement] { - var abbrevInfo = abbrevInfo - var elements = [BitcodeElement]() - - while !cursor.isAtEnd { - switch try cursor.read(abbrevWidth) { - case 0: // END_BLOCK - try cursor.advance(toBitAlignment: 32) - // FIXME: check expected length - return elements - - case 1: // ENTER_SUBBLOCK - let blockID = try cursor.readVBR(8) - let newAbbrevWidth = Int(try cursor.readVBR(4)) - try cursor.advance(toBitAlignment: 32) - _ = try cursor.read(32) // FIXME: use expected length - - switch blockID { - case 0: - try readBlockInfoBlock(abbrevWidth: newAbbrevWidth) - case 1...7: - // Metadata blocks we don't understand yet - fallthrough - default: - let innerElements = try readBlock( - id: blockID, abbrevWidth: newAbbrevWidth, abbrevInfo: globalAbbrevs[blockID] ?? []) - elements.append(.block(.init(id: blockID, elements: innerElements))) - } - - case 2: // DEFINE_ABBREV - let numOps = Int(try cursor.readVBR(5)) - abbrevInfo.append(try readAbbrev(numOps: numOps)) - - case 3: // UNABBREV_RECORD - let code = try cursor.readVBR(6) - let numOps = try cursor.readVBR(6) - var operands = [UInt64]() - for _ in 0.. 4) - let signatureValue = UInt32(Bits(buffer: data).readBits(atOffset: 0, count: 32)) - let bitstreamData = data[4..() let jobs: [Job] diff --git a/Sources/SwiftDriver/Utilities/DateAdditions.swift b/Sources/SwiftDriver/Utilities/DateAdditions.swift index 88c651787..7355898dd 100644 --- a/Sources/SwiftDriver/Utilities/DateAdditions.swift +++ b/Sources/SwiftDriver/Utilities/DateAdditions.swift @@ -24,4 +24,10 @@ public extension Date { init(legacyDriverSecs secs: Int, nanos: Int) { self = Date(timeIntervalSince1970: Double(secs) + Double(nanos) / 1e9) } + var legacyDriverSecsAndNanos: [Int] { + let totalSecs = timeIntervalSince1970 + let secs = Int( floor(totalSecs)) + let nanos = Int((totalSecs - floor(totalSecs)) * 1e9) + return [secs, nanos] + } } diff --git a/Sources/SwiftDriver/Utilities/Diagnostics.swift b/Sources/SwiftDriver/Utilities/Diagnostics.swift index 4967d419a..84aed8f1b 100644 --- a/Sources/SwiftDriver/Utilities/Diagnostics.swift +++ b/Sources/SwiftDriver/Utilities/Diagnostics.swift @@ -12,26 +12,35 @@ import TSCBasic import SwiftOptions -public typealias Diagnostic = TSCBasic.Diagnostic -public typealias DiagnosticData = TSCBasic.DiagnosticData - extension Diagnostic.Message { static var error_static_emit_executable_disallowed: Diagnostic.Message { .error("-static may not be used with -emit-executable") } - static func error_option_missing_required_argument(option: Option, requiredArg: Option) -> Diagnostic.Message { - .error("option '\(option.spelling)' is missing a required argument (\(requiredArg.spelling))") + static func error_option_missing_required_argument(option: Option, requiredArg: String) -> Diagnostic.Message { + .error("option '\(option.spelling)' is missing a required argument (\(requiredArg))") } static func error_opt_invalid_mapping(option: Option, value: String) -> Diagnostic.Message { .error("values for '\(option.spelling)' must be in the format 'original=remapped', but '\(value)' was provided") } + static func error_unsupported_argument(argument: String, option: Option) -> Diagnostic.Message { + .error("unsupported argument '\(argument)' to option '\(option.spelling)'") + } + + static func error_option_requires_sanitizer(option: Option) -> Diagnostic.Message { + .error("option '\(option.spelling)' requires a sanitizer to be enabled. Use -sanitize= to enable a sanitizer") + } + static func error_invalid_arg_value(arg: Option, value: String) -> Diagnostic.Message { .error("invalid value '\(value)' in '\(arg.spelling)'") } + static func warning_inferring_simulator_target(originalTriple: Triple, inferredTriple: Triple) -> Diagnostic.Message { + .warning("inferring simulator environment for target '\(originalTriple.triple)'; use '-target \(inferredTriple.triple)' instead") + } + static func error_argument_not_allowed_with(arg: String, other: String) -> Diagnostic.Message { .error("argument '\(arg)' is not allowed with '\(other)'") } @@ -44,6 +53,10 @@ extension Diagnostic.Message { .error("this mode does not support emitting modules") } + static func error_cannot_read_swiftdeps(file: VirtualPath, reason: String) -> Diagnostic.Message { + .error("cannot read swiftdeps: \(reason), file: \(file)") + } + static func error_bad_module_name( moduleName: String, explicitModuleName: Bool @@ -79,4 +92,8 @@ extension Diagnostic.Message { static func error_unknown_target(_ target: String) -> Diagnostic.Message { .error("unknown target '\(target)'") } + + static func warning_option_overrides_another(overridingOption: Option, overridenOption: Option) -> Diagnostic.Message { + .warning("ignoring '\(overridenOption.spelling)' because '\(overridingOption.spelling)' was also specified") + } } diff --git a/Sources/SwiftDriver/Utilities/PredictableRandomNumberGenerator.swift b/Sources/SwiftDriver/Utilities/PredictableRandomNumberGenerator.swift index 2b3339737..3448a66b7 100644 --- a/Sources/SwiftDriver/Utilities/PredictableRandomNumberGenerator.swift +++ b/Sources/SwiftDriver/Utilities/PredictableRandomNumberGenerator.swift @@ -16,7 +16,7 @@ /// The generator uses the [xoshiro256**](http://prng.di.unimi.it/xoshiro256starstar.c) /// algorithm to produce its output. It is initialized using the /// [splitmix64](http://prng.di.unimi.it/splitmix64.c) algorithm. -public struct PredictableRandomNumberGenerator: RandomNumberGenerator { +@_spi(Testing) public struct PredictableRandomNumberGenerator: RandomNumberGenerator { var state: (UInt64, UInt64, UInt64, UInt64) diff --git a/Sources/SwiftDriver/Utilities/RelativePathAdditions.swift b/Sources/SwiftDriver/Utilities/RelativePathAdditions.swift index b92ddd8be..41e2104d3 100644 --- a/Sources/SwiftDriver/Utilities/RelativePathAdditions.swift +++ b/Sources/SwiftDriver/Utilities/RelativePathAdditions.swift @@ -12,17 +12,6 @@ import TSCBasic extension RelativePath { - /// Retrieve the basename of the relative path without the extension. - /// - /// FIXME: Probably belongs in TSC - var basenameWithoutExt: String { - if let ext = self.extension { - return String(basename.dropLast(ext.count + 1)) - } - - return basename - } - /// Retrieve the basename of the relative path without any extensions, /// even if there are several, and without any leading dots. Roughly /// equivalent to the regex `/[^.]+/`. diff --git a/Sources/SwiftDriver/Utilities/Triple+Platforms.swift b/Sources/SwiftDriver/Utilities/Triple+Platforms.swift index 8cb27a6af..aa90d10c2 100644 --- a/Sources/SwiftDriver/Utilities/Triple+Platforms.swift +++ b/Sources/SwiftDriver/Utilities/Triple+Platforms.swift @@ -289,13 +289,15 @@ extension Triple { return "ps4" case .haiku: return "haiku" + case .wasi: + return "wasi" // Explicitly spell out the remaining cases to force a compile error when // Triple updates case .ananas, .cloudABI, .dragonFly, .fuchsia, .kfreebsd, .lv2, .netbsd, .openbsd, .solaris, .minix, .rtems, .nacl, .cnk, .aix, .cuda, .nvcl, .amdhsa, .elfiamcu, .mesa3d, .contiki, .amdpal, .hermitcore, .hurd, - .wasi, .emscripten: + .emscripten: return nil } } diff --git a/Sources/SwiftDriver/Utilities/Triple.swift b/Sources/SwiftDriver/Utilities/Triple.swift index bb3629584..4bb5f28dd 100644 --- a/Sources/SwiftDriver/Utilities/Triple.swift +++ b/Sources/SwiftDriver/Utilities/Triple.swift @@ -176,6 +176,20 @@ public struct Triple { os: parsedOS?.value) } } + + private init(dummyForTesting toolchain: Toolchain) { + self.triple = "" + self.arch = nil + self.subArch = nil + self.vendor = nil + self.os = nil + self.environment = nil + self.objectFormat = toolchain.dummyForTestingObjectFormat + } + + static func dummyForTesting(_ toolchain: Toolchain) -> Self { + Self(dummyForTesting: toolchain) + } } extension Triple: Codable { diff --git a/Sources/SwiftDriver/Utilities/VirtualPath.swift b/Sources/SwiftDriver/Utilities/VirtualPath.swift index b5a5e3968..ca8b9710c 100644 --- a/Sources/SwiftDriver/Utilities/VirtualPath.swift +++ b/Sources/SwiftDriver/Utilities/VirtualPath.swift @@ -10,6 +10,7 @@ // //===----------------------------------------------------------------------===// import TSCBasic +import Foundation /// A virtual path. public enum VirtualPath: Hashable { @@ -29,6 +30,9 @@ public enum VirtualPath: Hashable { /// A temporary file with the given name. case temporary(RelativePath) + /// A temporary file with the given name and contents. + case temporaryWithKnownContents(RelativePath, Data) + /// A temporary file that holds a list of paths. case fileList(RelativePath, FileList) @@ -48,7 +52,8 @@ public enum VirtualPath: Hashable { /// The extension of this path, for relative or absolute paths. public var `extension`: String? { switch self { - case .relative(let path), .temporary(let path), .fileList(let path, _): + case .relative(let path), .temporary(let path), + .temporaryWithKnownContents(let path, _), .fileList(let path, _): return path.extension case .absolute(let path): return path.extension @@ -62,7 +67,7 @@ public enum VirtualPath: Hashable { switch self { case .relative, .absolute, .standardInput, .standardOutput: return false - case .temporary, .fileList: + case .temporary, .temporaryWithKnownContents, .fileList: return true } } @@ -71,7 +76,25 @@ public enum VirtualPath: Hashable { switch self { case let .absolute(absolutePath): return absolutePath - case .relative, .temporary, .fileList, .standardInput, .standardOutput: + case .relative, .temporary, .temporaryWithKnownContents, .fileList, .standardInput, .standardOutput: + return nil + } + } + + public var relativePath: RelativePath? { + guard case .relative(let relativePath) = self else { return nil } + return relativePath + } + + /// If the path is some kind of temporary file, returns the `RelativePath` + /// representing its name. + public var temporaryFileName: RelativePath? { + switch self { + case .temporary(let name), + .fileList(let name, _), + .temporaryWithKnownContents(let name, _): + return name + case .absolute, .relative, .standardInput, .standardOutput: return nil } } @@ -81,7 +104,7 @@ public enum VirtualPath: Hashable { switch self { case .absolute(let path): return path.basename - case .relative(let path), .temporary(let path), .fileList(let path, _): + case .relative(let path), .temporary(let path), .temporaryWithKnownContents(let path, _), .fileList(let path, _): return path.basename case .standardInput, .standardOutput: return "" @@ -93,7 +116,7 @@ public enum VirtualPath: Hashable { switch self { case .absolute(let path): return path.basenameWithoutExt - case .relative(let path), .temporary(let path), .fileList(let path, _): + case .relative(let path), .temporary(let path), .temporaryWithKnownContents(let path, _), .fileList(let path, _): return path.basenameWithoutExt case .standardInput, .standardOutput: return "" @@ -107,7 +130,7 @@ public enum VirtualPath: Hashable { return .absolute(path.parentDirectory) case .relative(let path): return .relative(RelativePath(path.dirname)) - case .temporary(let path): + case .temporary(let path), .temporaryWithKnownContents(let path, _): return .temporary(RelativePath(path.dirname)) case .fileList(let path, _): return .temporary(RelativePath(path.dirname)) @@ -128,6 +151,8 @@ public enum VirtualPath: Hashable { return .relative(path.appending(component: component)) case .temporary(let path): return .temporary(path.appending(component: component)) + case let .temporaryWithKnownContents(path, contents): + return .temporaryWithKnownContents(path.appending(component: component), contents) case .fileList(let path, let content): return .fileList(path.appending(component: component), content) case .standardInput, .standardOutput: @@ -136,6 +161,24 @@ public enum VirtualPath: Hashable { } } + public func appending(components: String...) -> VirtualPath { + switch self { + case .absolute(let path): + return .absolute(path.appending(components: components)) + case .relative(let path): + return .relative(path.appending(components: components)) + case .temporary(let path): + return .temporary(path.appending(components: components)) + case let .temporaryWithKnownContents(path, contents): + return .temporaryWithKnownContents(path.appending(components: components), contents) + case .fileList(let path, let content): + return .fileList(path.appending(components: components), content) + case .standardInput, .standardOutput: + assertionFailure("Can't append path component to standard in/out") + return self + } + } + /// Returns the virtual path with an additional suffix appended to base name. /// /// This should not be used with `.standardInput` or `.standardOutput`. @@ -147,6 +190,8 @@ public enum VirtualPath: Hashable { return .relative(RelativePath(path.pathString + suffix)) case let .temporary(path): return .temporary(RelativePath(path.pathString + suffix)) + case let .temporaryWithKnownContents(path, contents): + return .temporaryWithKnownContents(RelativePath(path.pathString + suffix), contents) case let .fileList(path, content): return .fileList(RelativePath(path.pathString + suffix), content) case .standardInput, .standardOutput: @@ -158,7 +203,8 @@ public enum VirtualPath: Hashable { extension VirtualPath: Codable { private enum CodingKeys: String, CodingKey { - case relative, absolute, standardInput, standardOutput, temporary, fileList + case relative, absolute, standardInput, standardOutput, temporary, + temporaryWithKnownContents, fileList } public func encode(to encoder: Encoder) throws { @@ -177,6 +223,10 @@ extension VirtualPath: Codable { case .temporary(let a1): var unkeyedContainer = container.nestedUnkeyedContainer(forKey: .temporary) try unkeyedContainer.encode(a1) + case let .temporaryWithKnownContents(path, contents): + var unkeyedContainer = container.nestedUnkeyedContainer(forKey: .temporaryWithKnownContents) + try unkeyedContainer.encode(path) + try unkeyedContainer.encode(contents) case .fileList(let path, let fileList): var unkeyedContainer = container.nestedUnkeyedContainer(forKey: .fileList) try unkeyedContainer.encode(path) @@ -206,6 +256,11 @@ extension VirtualPath: Codable { var unkeyedValues = try values.nestedUnkeyedContainer(forKey: key) let a1 = try unkeyedValues.decode(RelativePath.self) self = .temporary(a1) + case .temporaryWithKnownContents: + var unkeyedValues = try values.nestedUnkeyedContainer(forKey: key) + let path = try unkeyedValues.decode(RelativePath.self) + let contents = try unkeyedValues.decode(Data.self) + self = .temporaryWithKnownContents(path, contents) case .fileList: var unkeyedValues = try values.nestedUnkeyedContainer(forKey: key) let path = try unkeyedValues.decode(RelativePath.self) @@ -215,6 +270,35 @@ extension VirtualPath: Codable { } } +/// A wrapper for easier decoding of absolute or relative VirtualPaths from strings. +@_spi(Testing) public struct TextualVirtualPath: Codable { + public var path: VirtualPath + + public init(from decoder: Decoder) throws { + let container = try decoder.singleValueContainer() + path = try VirtualPath(path: container.decode(String.self)) + } + + private init(path: VirtualPath) { + self.path = path + } + + static let dummyForTesting = Self(path: try! .init(path: "")) + + public func encode(to encoder: Encoder) throws { + var container = encoder.singleValueContainer() + switch path { + case .absolute(let path): + try container.encode(path.pathString) + case .relative(let path): + try container.encode(path.pathString) + case .temporary, .temporaryWithKnownContents, .standardInput, + .standardOutput, .fileList: + preconditionFailure("Path does not have a round-trippable textual representation") + } + } +} + extension VirtualPath: CustomStringConvertible { public var description: String { switch self { @@ -227,12 +311,34 @@ extension VirtualPath: CustomStringConvertible { case .standardInput, .standardOutput: return "-" - case .temporary(let path), .fileList(let path, _): + case .temporary(let path), .temporaryWithKnownContents(let path, _), + .fileList(let path, _): return path.pathString } } } +extension VirtualPath: CustomDebugStringConvertible { + public var debugDescription: String { + switch self { + case .relative(let path): + return ".relative(\(path.pathString))" + case .absolute(let path): + return ".absolute(\(path.pathString))" + case .standardInput: + return ".standardInput" + case .standardOutput: + return ".standardOutput" + case .temporary(let path): + return ".temporary(\(path.pathString))" + case .temporaryWithKnownContents(let path, _): + return ".temporaryWithKnownContents(\(path.pathString))" + case .fileList(let path, _): + return ".fileList(\(path.pathString))" + } + } +} + extension VirtualPath { /// Replace the extension of the given path with a new one based on the /// specified file type. @@ -244,6 +350,8 @@ extension VirtualPath { return .relative(RelativePath(path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType))) case let .temporary(path): return .temporary(RelativePath(path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType))) + case let .temporaryWithKnownContents(path, contents): + return .temporaryWithKnownContents(RelativePath(path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType)), contents) case let .fileList(path, content): return .fileList(RelativePath(path.pathString.withoutExt(path.extension).appendingFileTypeExtension(fileType)), content) case .standardInput, .standardOutput: @@ -252,6 +360,14 @@ extension VirtualPath { } } +extension VirtualPath { + /// Resolve a relative path into an absolute one, if possible. + public func resolvedRelativePath(base: AbsolutePath) -> VirtualPath { + guard case let .relative(relPath) = self else { return self } + return .absolute(.init(base, relPath)) + } +} + private extension String { func withoutExt(_ ext: String?) -> String { if let ext = ext { @@ -282,7 +398,8 @@ extension TSCBasic.FileSystem { throw FileSystemError.noCurrentWorkingDirectory } return try f(.init(cwd, relPath)) - case let .temporary(relPath), let .fileList(relPath, _): + case let .temporary(relPath), let .temporaryWithKnownContents(relPath, _), + let .fileList(relPath, _): throw FileSystemError.cannotResolveTempPath(relPath) case .standardInput: throw FileSystemError.cannotResolveStandardInput @@ -298,4 +415,8 @@ extension TSCBasic.FileSystem { func getFileInfo(_ path: VirtualPath) throws -> TSCBasic.FileInfo { try resolvingVirtualPath(path, apply: getFileInfo) } + + func exists(_ path: VirtualPath) throws -> Bool { + try resolvingVirtualPath(path, apply: exists) + } } diff --git a/Sources/SwiftDriverExecution/CMakeLists.txt b/Sources/SwiftDriverExecution/CMakeLists.txt new file mode 100644 index 000000000..c42a4decb --- /dev/null +++ b/Sources/SwiftDriverExecution/CMakeLists.txt @@ -0,0 +1,31 @@ +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2020 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for Swift project authors + +add_library(SwiftDriverExecution + llbuild.swift + MultiJobExecutor.swift + SwiftDriverExecutor.swift) + +target_link_libraries(SwiftDriverExecution PUBLIC + TSCBasic + TSCUtility + SwiftOptions + SwiftDriver) +target_link_libraries(SwiftDriverExecution PRIVATE + llbuildSwift) + +set_property(GLOBAL APPEND PROPERTY SWIFTDRIVER_EXPORTS SwiftDriverExecution) + +# NOTE: workaround for CMake not setting up include flags yet +set_target_properties(SwiftDriverExecution PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}) + +install(TARGETS SwiftDriverExecution + ARCHIVE DESTINATION lib + LIBRARY DESTINATION lib + RUNTIME DESTINATION bin) diff --git a/Sources/SwiftDriver/Execution/MultiJobExecutor.swift b/Sources/SwiftDriverExecution/MultiJobExecutor.swift similarity index 56% rename from Sources/SwiftDriver/Execution/MultiJobExecutor.swift rename to Sources/SwiftDriverExecution/MultiJobExecutor.swift index 2cacc4fe7..f66a67eaa 100644 --- a/Sources/SwiftDriver/Execution/MultiJobExecutor.swift +++ b/Sources/SwiftDriverExecution/MultiJobExecutor.swift @@ -1,4 +1,4 @@ -//===--------------- JobExecutor.swift - Swift Job Execution --------------===// +//===------- MultiJobExecutor.swift - LLBuild-powered job executor --------===// // // This source file is part of the Swift.org open source project // @@ -9,22 +9,45 @@ // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors // //===----------------------------------------------------------------------===// + import TSCBasic import enum TSCUtility.Diagnostics import Foundation import Dispatch +import SwiftDriver + +// We either import the llbuildSwift shared library or the llbuild framework. +#if canImport(llbuildSwift) +@_implementationOnly import llbuildSwift +@_implementationOnly import llbuild +#else +@_implementationOnly import llbuild +#endif + public final class MultiJobExecutor { /// The context required during job execution. - struct Context { + /// Must be a class because the producer map can grow as jobs are added. + class Context { /// This contains mapping from an output to the index(in the jobs array) of the job that produces that output. - let producerMap: [VirtualPath: Int] + /// Can grow dynamically as jobs are added. + var producerMap: [VirtualPath: Int] = [:] /// All the jobs being executed. - let jobs: [Job] + var jobs: [Job] = [] + + /// The indices into `jobs` for the primary jobs; those that must be run before the full set of + /// secondaries can be determined. Basically compilations. + let primaryIndices: Range + + /// The indices into `jobs` of the jobs that must run *after* all compilations. + let postCompileIndices: Range + + /// If non-null, the driver is performing an incremental compilation. + let incrementalCompilationState: IncrementalCompilationState? /// The resolver for argument template. let argsResolver: ArgsResolver @@ -33,7 +56,7 @@ public final class MultiJobExecutor { let env: [String: String] /// The file system. - let fileSystem: FileSystem + let fileSystem: TSCBasic.FileSystem /// The job executor delegate. let executorDelegate: JobExecutionDelegate @@ -59,12 +82,23 @@ public final class MultiJobExecutor { /// The type to use when launching new processes. This mostly serves as an override for testing. let processType: ProcessProtocol.Type + /// Records the task build engine for the `ExecuteAllJobs` rule (and task) so that when a + /// mandatory job finishes, and new jobs are discovered, inputs can be added to that rule for + /// any newly-required job. Set only once. + private(set) var executeAllJobsTaskBuildEngine: LLTaskBuildEngine? = nil + + /// If a job fails, the driver needs to stop running jobs. + private(set) var isBuildCancelled = false + + /// The value of the option + let continueBuildingAfterErrors: Bool + + init( argsResolver: ArgsResolver, env: [String: String], - fileSystem: FileSystem, - producerMap: [VirtualPath: Int], - jobs: [Job], + fileSystem: TSCBasic.FileSystem, + workload: DriverExecutorWorkload, executorDelegate: JobExecutionDelegate, jobQueue: OperationQueue, processSet: ProcessSet?, @@ -73,8 +107,15 @@ public final class MultiJobExecutor { diagnosticsEngine: DiagnosticsEngine, processType: ProcessProtocol.Type = Process.self ) { - self.producerMap = producerMap - self.jobs = jobs + ( + jobs: self.jobs, + producerMap: self.producerMap, + primaryIndices: self.primaryIndices, + postCompileIndices: self.postCompileIndices, + incrementalCompilationState: self.incrementalCompilationState, + continueBuildingAfterErrors: self.continueBuildingAfterErrors + ) = Self.fillInJobsAndProducers(workload) + self.argsResolver = argsResolver self.env = env self.fileSystem = fileSystem @@ -86,37 +127,157 @@ public final class MultiJobExecutor { self.diagnosticsEngine = diagnosticsEngine self.processType = processType } + + deinit { + // break a potential cycle + executeAllJobsTaskBuildEngine = nil + } + + private static func fillInJobsAndProducers(_ workload: DriverExecutorWorkload + ) -> (jobs: [Job], + producerMap: [VirtualPath: Int], + primaryIndices: Range, + postCompileIndices: Range, + incrementalCompilationState: IncrementalCompilationState?, + continueBuildingAfterErrors: Bool) + { + var jobs = [Job]() + var producerMap = [VirtualPath: Int]() + let primaryIndices, postCompileIndices: Range + let incrementalCompilationState: IncrementalCompilationState? + switch workload.kind { + case let .incremental(ics): + incrementalCompilationState = ics + primaryIndices = Self.addJobs( + ics.mandatoryPreOrCompileJobsInOrder, + to: &jobs, + producing: &producerMap + ) + postCompileIndices = Self.addJobs( + ics.postCompileJobs, + to: &jobs, + producing: &producerMap) + case let .all(nonincrementalJobs): + incrementalCompilationState = nil + primaryIndices = Self.addJobs( + nonincrementalJobs, + to: &jobs, + producing: &producerMap) + postCompileIndices = 0 ..< 0 + } + return ( jobs: jobs, + producerMap: producerMap, + primaryIndices: primaryIndices, + postCompileIndices: postCompileIndices, + incrementalCompilationState: incrementalCompilationState, + continueBuildingAfterErrors: workload.continueBuildingAfterErrors) + } + + /// Allow for dynamically adding jobs, since some compile jobs are added dynamically. + /// Return the indices into `jobs` of the added jobs. + @discardableResult + fileprivate static func addJobs( + _ js: [Job], + to jobs: inout [Job], + producing producerMap: inout [VirtualPath: Int] + ) -> Range { + let initialCount = jobs.count + for job in js { + addProducts(of: job, index: jobs.count, knownJobs: jobs, to: &producerMap) + jobs.append(job) + } + return initialCount ..< jobs.count + } + + /// Update the producer map when adding a job. + private static func addProducts(of job: Job, + index: Int, + knownJobs: [Job], + to producerMap: inout [VirtualPath: Int] + ) { + for output in job.outputs { + if let otherJobIndex = producerMap.updateValue(index, forKey: output.file) { + fatalError("multiple producers for output \(output): \(job) & \(knownJobs[otherJobIndex])") + } + producerMap[output.file] = index + } + } + + fileprivate func setExecuteAllJobsTaskBuildEngine(_ engine: LLTaskBuildEngine) { + assert(executeAllJobsTaskBuildEngine == nil) + executeAllJobsTaskBuildEngine = engine + } + + /// After a job finishes, an incremental build may discover more jobs are needed, or if all compilations + /// are done, will need to then add in the post-compilation rules. + fileprivate func addRuleBeyondMandatoryCompiles( + finishedJob job: Job, + result: ProcessResult + ) { + guard job.kind.isCompile else { + return + } + if let newJobs = incrementalCompilationState? + .getJobsDiscoveredToBeNeededAfterFinishing(job: job, result: result) { + let newJobIndices = Self.addJobs(newJobs, to: &jobs, producing: &producerMap) + needInputFor(indices: newJobIndices) + } + else { + needInputFor(indices: postCompileIndices) + } + } + fileprivate func needInputFor(indices: Indices) + where Indices.Element == Int + { + for index in indices { + let key = ExecuteJobRule.RuleKey(index: index) + executeAllJobsTaskBuildEngine!.taskNeedsInput(key, inputID: index) + } + } + + fileprivate func cancelBuildIfNeeded(_ result: ProcessResult) { + switch (result.exitStatus, continueBuildingAfterErrors) { + case (.terminated(let code), false) where code != EXIT_SUCCESS: + isBuildCancelled = true + #if !os(Windows) + case (.signalled, _): + isBuildCancelled = true + #endif + default: + break + } + } } - /// The list of jobs that we may need to run. - let jobs: [Job] + /// The work to be done. + private let workload: DriverExecutorWorkload /// The argument resolver. - let argsResolver: ArgsResolver + private let argsResolver: ArgsResolver /// The job executor delegate. - let executorDelegate: JobExecutionDelegate + private let executorDelegate: JobExecutionDelegate /// The number of jobs to run in parallel. - let numParallelJobs: Int + private let numParallelJobs: Int /// The process set to use when launching new processes. - let processSet: ProcessSet? + private let processSet: ProcessSet? /// If true, always use response files to pass command line arguments. - let forceResponseFiles: Bool + private let forceResponseFiles: Bool /// The last time each input file was modified, recorded at the start of the build. - public let recordedInputModificationDates: [TypedVirtualPath: Date] + private let recordedInputModificationDates: [TypedVirtualPath: Date] /// The diagnostics engine to use when reporting errors. - let diagnosticsEngine: DiagnosticsEngine + private let diagnosticsEngine: DiagnosticsEngine /// The type to use when launching new processes. This mostly serves as an override for testing. - let processType: ProcessProtocol.Type + private let processType: ProcessProtocol.Type public init( - jobs: [Job], + workload: DriverExecutorWorkload, resolver: ArgsResolver, executorDelegate: JobExecutionDelegate, diagnosticsEngine: DiagnosticsEngine, @@ -126,7 +287,7 @@ public final class MultiJobExecutor { recordedInputModificationDates: [TypedVirtualPath: Date] = [:], processType: ProcessProtocol.Type = Process.self ) { - self.jobs = jobs + self.workload = workload self.argsResolver = resolver self.executorDelegate = executorDelegate self.diagnosticsEngine = diagnosticsEngine @@ -138,8 +299,8 @@ public final class MultiJobExecutor { } /// Execute all jobs. - public func execute(env: [String: String], fileSystem: FileSystem) throws { - let context = createContext(jobs, env: env, fileSystem: fileSystem) + public func execute(env: [String: String], fileSystem: TSCBasic.FileSystem) throws { + let context = createContext(env: env, fileSystem: fileSystem) let delegate = JobExecutorBuildDelegate(context) let engine = LLBuildEngine(delegate: delegate) @@ -153,15 +314,7 @@ public final class MultiJobExecutor { } /// Create the context required during the execution. - func createContext(_ jobs: [Job], env: [String: String], fileSystem: FileSystem) -> Context { - var producerMap: [VirtualPath: Int] = [:] - for (index, job) in jobs.enumerated() { - for output in job.outputs { - assert(!producerMap.keys.contains(output.file), "multiple producers for output \(output): \(job) \(producerMap[output.file]!)") - producerMap[output.file] = index - } - } - + private func createContext(env: [String: String], fileSystem: TSCBasic.FileSystem) -> Context { let jobQueue = OperationQueue() jobQueue.name = "org.swift.driver.job-execution" jobQueue.maxConcurrentOperationCount = numParallelJobs @@ -170,8 +323,7 @@ public final class MultiJobExecutor { argsResolver: argsResolver, env: env, fileSystem: fileSystem, - producerMap: producerMap, - jobs: jobs, + workload: workload, executorDelegate: executorDelegate, jobQueue: jobQueue, processSet: processSet, @@ -194,7 +346,7 @@ struct JobExecutorBuildDelegate: LLBuildEngineDelegate { func lookupRule(rule: String, key: Key) -> Rule { switch rule { case ExecuteAllJobsRule.ruleName: - return ExecuteAllJobsRule(key, jobs: context.jobs, fileSystem: context.fileSystem) + return ExecuteAllJobsRule(context: context) case ExecuteJobRule.ruleName: return ExecuteJobRule(key, context: context) default: @@ -228,23 +380,20 @@ class ExecuteAllJobsRule: LLBuildRule { override class var ruleName: String { "\(ExecuteAllJobsRule.self)" } - private let key: RuleKey - private let jobs: [Job] + private let context: MultiJobExecutor.Context /// True if any of the inputs had any error. private var allInputsSucceeded: Bool = true - init(_ key: Key, jobs: [Job], fileSystem: FileSystem) { - self.key = RuleKey(key) - self.jobs = jobs - super.init(fileSystem: fileSystem) + + init(context: MultiJobExecutor.Context) { + self.context = context + super.init(fileSystem: context.fileSystem) } override func start(_ engine: LLTaskBuildEngine) { - for index in jobs.indices { - let key = ExecuteJobRule.RuleKey(index: index) - engine.taskNeedsInput(key, inputID: index) - } + context.setExecuteAllJobsTaskBuildEngine(engine) + context.needInputFor(indices: context.primaryIndices) } override func isResultValid(_ priorValue: Value) -> Bool { @@ -288,12 +437,7 @@ class ExecuteJobRule: LLBuildRule { } override func start(_ engine: LLTaskBuildEngine) { - for (idx, input) in context.jobs[key.index].inputs.enumerated() { - if let producingJobIndex = context.producerMap[input.file] { - let key = ExecuteJobRule.RuleKey(index: producingJobIndex) - engine.taskNeedsInput(key, inputID: idx) - } - } + requestInputs(from: engine) } override func isResultValid(_ priorValue: Value) -> Bool { @@ -301,16 +445,11 @@ class ExecuteJobRule: LLBuildRule { } override func provideValue(_ engine: LLTaskBuildEngine, inputID: Int, value: Value) { - do { - let buildValue = try DriverBuildValue(value) - allInputsSucceeded = allInputsSucceeded && buildValue.success - } catch { - allInputsSucceeded = false - } + rememberIfInputSucceeded(engine, value: value) } + /// Called when the build engine thinks all inputs are available in order to run the job. override func inputsAvailable(_ engine: LLTaskBuildEngine) { - // Return early any of the input failed. guard allInputsSucceeded else { return engine.taskIsComplete(DriverBuildValue.jobExecution(success: false)) } @@ -320,10 +459,40 @@ class ExecuteJobRule: LLBuildRule { } } + private var myJob: Job { + context.jobs[key.index] + } + + private var inputKeysAndIDs: [(RuleKey, Int)] { + myJob.inputs.enumerated().compactMap { + (inputIndex, inputFile) in + context.producerMap[inputFile.file] .map { (ExecuteJobRule.RuleKey(index: $0), inputIndex) } + } + } + + private func requestInputs(from engine: LLTaskBuildEngine) { + for (key, ID) in inputKeysAndIDs { + engine.taskNeedsInput(key, inputID: ID) + } + } + + private func rememberIfInputSucceeded(_ engine: LLTaskBuildEngine, value: Value) { + do { + let buildValue = try DriverBuildValue(value) + allInputsSucceeded = allInputsSucceeded && buildValue.success + } catch { + allInputsSucceeded = false + } + } + private func executeJob(_ engine: LLTaskBuildEngine) { + if context.isBuildCancelled { + engine.taskIsComplete(DriverBuildValue.jobExecution(success: false)) + return + } let context = self.context let resolver = context.argsResolver - let job = context.jobs[key.index] + let job = myJob let env = context.env.merging(job.extraEnvironment, uniquingKeysWith: { $1 }) let value: DriverBuildValue @@ -364,13 +533,17 @@ class ExecuteJobRule: LLBuildRule { #endif } } + if case .terminated = result.exitStatus { + context.addRuleBeyondMandatoryCompiles(finishedJob: job, result: result) + } // Inform the delegate about job finishing. context.delegateQueue.async { context.executorDelegate.jobFinished(job: job, result: result, pid: pid) } - value = .jobExecution(success: success) + + context.cancelBuildIfNeeded(result) } catch { if error is DiagnosticData { context.diagnosticsEngine.emit(error) @@ -394,12 +567,12 @@ class ExecuteJobRule: LLBuildRule { extension Job: LLBuildValue { } -private extension Diagnostic.Message { - static func error_command_failed(kind: Job.Kind, code: Int32) -> Diagnostic.Message { +private extension TSCBasic.Diagnostic.Message { + static func error_command_failed(kind: Job.Kind, code: Int32) -> TSCBasic.Diagnostic.Message { .error("\(kind.rawValue) command failed with exit code \(code) (use -v to see invocation)") } - static func error_command_signalled(kind: Job.Kind, signal: Int32) -> Diagnostic.Message { + static func error_command_signalled(kind: Job.Kind, signal: Int32) -> TSCBasic.Diagnostic.Message { .error("\(kind.rawValue) command failed due to signal \(signal) (use -v to see invocation)") } } diff --git a/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift b/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift new file mode 100644 index 000000000..0e7017028 --- /dev/null +++ b/Sources/SwiftDriverExecution/SwiftDriverExecutor.swift @@ -0,0 +1,100 @@ +//===--- MultiJobExecutor.swift - Builtin DriverExecutor implementation ---===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +import SwiftDriver +import TSCBasic +import Foundation + +public final class SwiftDriverExecutor: DriverExecutor { + let diagnosticsEngine: DiagnosticsEngine + let processSet: ProcessSet + let fileSystem: FileSystem + public let resolver: ArgsResolver + let env: [String: String] + + public init(diagnosticsEngine: DiagnosticsEngine, + processSet: ProcessSet, + fileSystem: FileSystem, + env: [String: String]) throws { + self.diagnosticsEngine = diagnosticsEngine + self.processSet = processSet + self.fileSystem = fileSystem + self.env = env + self.resolver = try ArgsResolver(fileSystem: fileSystem) + } + + public func execute(job: Job, + forceResponseFiles: Bool = false, + recordedInputModificationDates: [TypedVirtualPath: Date] = [:]) throws -> ProcessResult { + let arguments: [String] = try resolver.resolveArgumentList(for: job, + forceResponseFiles: forceResponseFiles) + + try job.verifyInputsNotModified(since: recordedInputModificationDates, + fileSystem: fileSystem) + + if job.requiresInPlaceExecution { + for (envVar, value) in job.extraEnvironment { + try ProcessEnv.setVar(envVar, value: value) + } + + try exec(path: arguments[0], args: arguments) + fatalError("unreachable, exec always throws on failure") + } else { + var childEnv = env + childEnv.merge(job.extraEnvironment, uniquingKeysWith: { (_, new) in new }) + + let process = try Process.launchProcess(arguments: arguments, env: childEnv) + return try process.waitUntilExit() + } + } + + public func execute(workload: DriverExecutorWorkload, + delegate: JobExecutionDelegate, + numParallelJobs: Int = 1, + forceResponseFiles: Bool = false, + recordedInputModificationDates: [TypedVirtualPath: Date] = [:] + ) throws { + let llbuildExecutor = MultiJobExecutor( + workload: workload, + resolver: resolver, + executorDelegate: delegate, + diagnosticsEngine: diagnosticsEngine, + numParallelJobs: numParallelJobs, + processSet: processSet, + forceResponseFiles: forceResponseFiles, + recordedInputModificationDates: recordedInputModificationDates) + try llbuildExecutor.execute(env: env, fileSystem: fileSystem) + } + + @discardableResult + public func checkNonZeroExit(args: String..., environment: [String: String] = ProcessEnv.vars) throws -> String { + return try Process.checkNonZeroExit(arguments: args, environment: environment) + } + + public func description(of job: Job, forceResponseFiles: Bool) throws -> String { + let (args, usedResponseFile) = try resolver.resolveArgumentList(for: job, forceResponseFiles: forceResponseFiles) + var result = args.joined(separator: " ") + + if usedResponseFile { + // Print the response file arguments as a comment. + result += " # \(job.commandLine.joinedArguments)" + } + + if !job.extraEnvironment.isEmpty { + result += " #" + for (envVar, val) in job.extraEnvironment { + result += " \(envVar)=\(val)" + } + } + return result + } +} diff --git a/Sources/SwiftDriverExecution/llbuild.swift b/Sources/SwiftDriverExecution/llbuild.swift new file mode 100644 index 000000000..5de035b43 --- /dev/null +++ b/Sources/SwiftDriverExecution/llbuild.swift @@ -0,0 +1,269 @@ +//===--------------- llbuild.swift - Swift LLBuild Interaction ------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +// FIXME: This is slightly modified from the SwiftPM version, +// consider moving this to llbuild. + +import Foundation +import TSCBasic + +// We either import the llbuildSwift shared library or the llbuild framework. +#if canImport(llbuildSwift) +@_implementationOnly import llbuildSwift +@_implementationOnly import llbuild +#else +@_implementationOnly import llbuild +#endif + +/// An llbuild value. +protocol LLBuildValue: Codable { +} + +/// An llbuild key. +protocol LLBuildKey: Codable { + /// The value that this key computes. + associatedtype BuildValue: LLBuildValue + + /// The rule that this key operates on. + associatedtype BuildRule: LLBuildRule +} + +protocol LLBuildEngineDelegate { + func lookupRule(rule: String, key: Key) -> Rule +} + +final class LLBuildEngine { + + enum Error: Swift.Error, CustomStringConvertible { + case failed(errors: [String]) + + var description: String { + switch self { + case .failed(let errors): + return errors.joined(separator: "\n") + } + } + } + + fileprivate final class Delegate: BuildEngineDelegate { + let delegate: LLBuildEngineDelegate + var errors: [String] = [] + + init(_ delegate: LLBuildEngineDelegate) { + self.delegate = delegate + } + + func lookupRule(_ key: Key) -> Rule { + let ruleKey = RuleKey(key) + return delegate.lookupRule( + rule: ruleKey.rule, key: Key(ruleKey.data)) + } + + func error(_ message: String) { + errors.append(message) + } + } + + private let engine: BuildEngine + private let delegate: Delegate + + init(delegate: LLBuildEngineDelegate) { + self.delegate = Delegate(delegate) + engine = BuildEngine(delegate: self.delegate) + } + + deinit { + engine.close() + } + + func build(key: T) throws -> T.BuildValue { + // Clear out any errors from the previous build. + delegate.errors.removeAll() + + let encodedKey = RuleKey( + rule: T.BuildRule.ruleName, data: key.toKey().data).toKey() + let value = engine.build(key: encodedKey) + + // Throw if the engine encountered any fatal error during the build. + if !delegate.errors.isEmpty || value.data.isEmpty { + throw Error.failed(errors: delegate.errors) + } + + return try T.BuildValue(value) + } + + func attachDB(path: String, schemaVersion: Int = 2) throws { + try engine.attachDB(path: path, schemaVersion: schemaVersion) + } + + func close() { + engine.close() + } +} + +// FIXME: Rename to something else. +class LLTaskBuildEngine { + + let engine: TaskBuildEngine + let fileSystem: TSCBasic.FileSystem + + init(_ engine: TaskBuildEngine, fileSystem: TSCBasic.FileSystem) { + self.engine = engine + self.fileSystem = fileSystem + } + + func taskNeedsInput(_ key: T, inputID: Int) { + let encodedKey = RuleKey( + rule: T.BuildRule.ruleName, data: key.toKey().data).toKey() + engine.taskNeedsInput(encodedKey, inputID: inputID) + } + + func taskIsComplete(_ result: T) { + engine.taskIsComplete(result.toValue(), forceChange: false) + } +} + +/// An individual build rule. +class LLBuildRule: Rule, Task { + + /// The name of the rule. + /// + /// This name will be available in the delegate's lookupRule(rule:key:). + class var ruleName: String { + fatalError("subclass responsibility") + } + + let fileSystem: TSCBasic.FileSystem + + init(fileSystem: TSCBasic.FileSystem) { + self.fileSystem = fileSystem + } + + func createTask() -> Task { + return self + } + + func start(_ engine: TaskBuildEngine) { + self.start(LLTaskBuildEngine(engine, fileSystem: fileSystem)) + } + + func provideValue(_ engine: TaskBuildEngine, inputID: Int, value: Value) { + self.provideValue(LLTaskBuildEngine(engine, fileSystem: fileSystem), inputID: inputID, value: value) + } + + func inputsAvailable(_ engine: TaskBuildEngine) { + self.inputsAvailable(LLTaskBuildEngine(engine, fileSystem: fileSystem)) + } + + // MARK:- + + func isResultValid(_ priorValue: Value) -> Bool { + return true + } + + func start(_ engine: LLTaskBuildEngine) { + } + + func provideValue(_ engine: LLTaskBuildEngine, inputID: Int, value: Value) { + } + + func inputsAvailable(_ engine: LLTaskBuildEngine) { + } + + // Not strictly needed, but permits overriding for debugging + func updateStatus(_ status: RuleStatus) { + } +} + +// MARK:- Helpers + +private struct RuleKey: Codable { + + let rule: String + let data: [UInt8] + + init(rule: String, data: [UInt8]) { + self.rule = rule + self.data = data + } + + init(_ key: Key) { + self.init(key.data) + } + + init(_ data: [UInt8]) { + self = try! fromBytes(data) + } + + func toKey() -> Key { + return try! Key(toBytes(self)) + } +} + +extension LLBuildKey { + init(_ key: Key) { + self.init(key.data) + } + + init(_ data: [UInt8]) { + do { + self = try fromBytes(data) + } catch { + let stringValue: String + if let str = String(bytes: data, encoding: .utf8) { + stringValue = str + } else { + stringValue = String(describing: data) + } + fatalError("Please file a bug at https://bugs.swift.org with this info -- LLBuildKey: ###\(error)### ----- ###\(stringValue)###") + } + } + + func toKey() -> Key { + return try! Key(toBytes(self)) + } +} + +extension LLBuildValue { + init(_ value: Value) throws { + do { + self = try fromBytes(value.data) + } catch { + let stringValue: String + if let str = String(bytes: value.data, encoding: .utf8) { + stringValue = str + } else { + stringValue = String(describing: value.data) + } + fatalError("Please file a bug at https://bugs.swift.org with this info -- LLBuildValue: ###\(error)### ----- ###\(stringValue)###") + } + } + + func toValue() -> Value { + return try! Value(toBytes(self)) + } +} + +private func fromBytes(_ bytes: [UInt8]) throws -> T { + var bytes = bytes + let data = Data(bytes: &bytes, count: bytes.count) + return try JSONDecoder().decode(T.self, from: data) +} + +private func toBytes(_ value: T) throws -> [UInt8] { + let encoder = JSONEncoder() + if #available(macOS 10.13, iOS 11.0, watchOS 4.0, tvOS 11.0, *) { + encoder.outputFormatting = [.sortedKeys] + } + let encoded = try encoder.encode(value) + return [UInt8](encoded) +} diff --git a/Sources/SwiftOptions/ExtraOptions.swift b/Sources/SwiftOptions/ExtraOptions.swift index 23e5bdb9b..d6faa8da7 100644 --- a/Sources/SwiftOptions/ExtraOptions.swift +++ b/Sources/SwiftOptions/ExtraOptions.swift @@ -12,15 +12,15 @@ extension Option { public static let driverPrintGraphviz: Option = Option("-driver-print-graphviz", .flag, attributes: [.helpHidden, .doesNotAffectIncrementalBuild], helpText: "Write the job graph as a graphviz file", group: .internalDebug) public static let driverExplicitModuleBuild: Option = Option("-experimental-explicit-module-build", .flag, attributes: [.helpHidden], helpText: "Prebuild module dependencies to make them explicit") - public static let driverPrintModuleDependenciesJobs: Option = Option("-driver-print-module-dependencies-jobs", .flag, attributes: [.helpHidden], helpText: "Print commands to explicitly build module dependencies") public static let driverWarnUnusedOptions: Option = Option("-driver-warn-unused-options", .flag, attributes: [.helpHidden], helpText: "Emit warnings for any provided options which are unused by the driver.") + public static let emitModuleSeparately: Option = Option("-experimental-emit-module-separately", .flag, attributes: [.helpHidden], helpText: "Emit module files as a distinct job") public static var extraOptions: [Option] { return [ Option.driverPrintGraphviz, Option.driverExplicitModuleBuild, - Option.driverPrintModuleDependenciesJobs, - Option.driverWarnUnusedOptions + Option.driverWarnUnusedOptions, + Option.emitModuleSeparately ] } } diff --git a/Sources/SwiftOptions/Option.swift b/Sources/SwiftOptions/Option.swift index e98a8e237..5535d8b6f 100644 --- a/Sources/SwiftOptions/Option.swift +++ b/Sources/SwiftOptions/Option.swift @@ -135,9 +135,9 @@ extension Option { public func isAccepted(by driverKind: DriverKind) -> Bool { switch driverKind { case .batch: - return !attributes.contains(.noBatch) + return attributes.isDisjoint(with: [.noDriver, .noBatch]) case .interactive: - return !attributes.contains(.noInteractive) + return attributes.isDisjoint(with: [.noDriver, .noInteractive]) } } } diff --git a/Sources/SwiftOptions/OptionParsing.swift b/Sources/SwiftOptions/OptionParsing.swift index a81a5122e..6f59e58e4 100644 --- a/Sources/SwiftOptions/OptionParsing.swift +++ b/Sources/SwiftOptions/OptionParsing.swift @@ -38,7 +38,8 @@ extension OptionTable { public func parse(_ arguments: [String], for driverKind: DriverKind) throws -> ParsedOptions { var trie = PrefixTrie