Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[DependencyScanning] Add ability to scan TU with a buffer input #9818

Merged
merged 1 commit into from
Feb 5, 2025
Merged

[DependencyScanning] Add ability to scan TU with a buffer input #9818

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions clang/include/clang/Tooling/DependencyScanning/DependencyScanningTool.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,14 +174,17 @@ class DependencyScanningTool {
/// \param LookupModuleOutput This function is called to fill in
/// "-fmodule-file=", "-o" and other output
/// arguments for dependencies.
/// \param TUBuffer Optional memory buffer for translation unit input. If
/// TUBuffer is nullopt, the input should be included in the
/// Commandline already.
///
/// \returns a \c StringError with the diagnostic output if clang errors
/// occurred, \c TranslationUnitDeps otherwise.
llvm::Expected<TranslationUnitDeps>
getTranslationUnitDependencies(const std::vector<std::string> &CommandLine,
StringRef CWD,
const llvm::DenseSet<ModuleID> &AlreadySeen,
LookupModuleOutputCallback LookupModuleOutput);
llvm::Expected<TranslationUnitDeps> getTranslationUnitDependencies(
const std::vector<std::string> &CommandLine, StringRef CWD,
const llvm::DenseSet<ModuleID> &AlreadySeen,
LookupModuleOutputCallback LookupModuleOutput,
std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt);

/// Given a compilation context specified via the Clang driver command-line,
/// gather modular dependencies of module with the given name, and return the
Expand Down
47 changes: 42 additions & 5 deletions clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
#include "llvm/CAS/CachingOnDiskFileSystem.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBufferRef.h"
#include <optional>
#include <string>

Expand Down Expand Up @@ -119,9 +120,21 @@ class DependencyScanningWorker {
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS);

/// Run the dependency scanning tool for a given clang driver command-line,
/// and report the discovered dependencies to the provided consumer. If \p
/// ModuleName isn't empty, this function reports the dependencies of module
/// \p ModuleName.
/// and report the discovered dependencies to the provided consumer. If
/// TUBuffer is not nullopt, it is used as TU input for the dependency
/// scanning. Otherwise, the input should be included as part of the
/// command-line.
///
/// \returns false if clang errors occurred (with diagnostics reported to
/// \c DiagConsumer), true otherwise.
bool computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &DepConsumer, DependencyActionController &Controller,
DiagnosticConsumer &DiagConsumer,
std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt);

/// Run the dependency scanning tool for a given clang driver command-line
/// for a specific module.
///
/// \returns false if clang errors occurred (with diagnostics reported to
/// \c DiagConsumer), true otherwise.
Expand All @@ -130,13 +143,28 @@ class DependencyScanningWorker {
DependencyConsumer &DepConsumer,
DependencyActionController &Controller,
DiagnosticConsumer &DiagConsumer,
std::optional<StringRef> ModuleName = std::nullopt);
StringRef ModuleName);

/// Run the dependency scanning tool for a given clang driver command-line
/// for a specific translation unit via file system or memory buffer.
///
/// \returns A \c StringError with the diagnostic output if clang errors
/// occurred, success otherwise.
llvm::Error computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
std::optional<StringRef> ModuleName = std::nullopt);
std::optional<llvm::MemoryBufferRef> TUBuffer = std::nullopt);

/// Run the dependency scanning tool for a given clang driver command-line
/// for a specific module.
///
/// \returns A \c StringError with the diagnostic output if clang errors
/// occurred, success otherwise.
llvm::Error computeDependencies(StringRef WorkingDirectory,
const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer,
DependencyActionController &Controller,
StringRef ModuleName);

/// Scan from a compiler invocation.
/// If \p DiagGenerationAsCompilation is true it will generate error
Expand Down Expand Up @@ -186,6 +214,15 @@ class DependencyScanningWorker {
llvm::IntrusiveRefCntPtr<DependencyScanningCASFilesystem> DepCASFS;
CASOptions CASOpts;
std::shared_ptr<cas::ObjectStore> CAS;

/// Private helper functions.
bool scanDependencies(StringRef WorkingDirectory,
const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer,
DependencyActionController &Controller,
DiagnosticConsumer &DC,
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
std::optional<StringRef> ModuleName);
};

} // end namespace dependencies
Expand Down
7 changes: 4 additions & 3 deletions clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -268,11 +268,12 @@ llvm::Expected<TranslationUnitDeps>
DependencyScanningTool::getTranslationUnitDependencies(
const std::vector<std::string> &CommandLine, StringRef CWD,
const llvm::DenseSet<ModuleID> &AlreadySeen,
LookupModuleOutputCallback LookupModuleOutput) {
LookupModuleOutputCallback LookupModuleOutput,
std::optional<llvm::MemoryBufferRef> TUBuffer) {
FullDependencyConsumer Consumer(AlreadySeen);
auto Controller = createActionController(LookupModuleOutput);
llvm::Error Result =
Worker.computeDependencies(CWD, CommandLine, Consumer, *Controller);
llvm::Error Result = Worker.computeDependencies(CWD, CommandLine, Consumer,
*Controller, TUBuffer);
if (Result)
return std::move(Result);
return Consumer.takeTranslationUnitDeps();
Expand Down
175 changes: 119 additions & 56 deletions clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,14 @@
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
#include "clang/Tooling/DependencyScanning/ScanAndUpdateArgs.h"
#include "clang/Tooling/Tooling.h"
#include "llvm/ADT/IntrusiveRefCntPtr.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/CAS/CASProvidingFileSystem.h"
#include "llvm/CAS/CachingOnDiskFileSystem.h"
#include "llvm/CAS/ObjectStore.h"
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/PrefixMapper.h"
#include "llvm/TargetParser/Host.h"
#include <optional>
Expand Down Expand Up @@ -744,20 +746,43 @@ DependencyScanningWorker::getOrCreateFileManager() const {
return new FileManager(FileSystemOptions(), BaseFS);
}

llvm::Error DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
std::optional<StringRef> ModuleName) {
static std::unique_ptr<DiagnosticOptions>
createDiagOptions(const std::vector<std::string> &CommandLine) {
std::vector<const char *> CLI;
for (const std::string &Arg : CommandLine)
CLI.push_back(Arg.c_str());
auto DiagOpts = CreateAndPopulateDiagOpts(CLI);
sanitizeDiagOpts(*DiagOpts);
return DiagOpts;
}

llvm::Error DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
std::optional<llvm::MemoryBufferRef> TUBuffer) {
// Capture the emitted diagnostics and report them to the client
// in the case of a failure.
std::string DiagnosticOutput;
llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
auto DiagOpts = createDiagOptions(CommandLine);
TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.release());

if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller,
DiagPrinter, TUBuffer))
return llvm::Error::success();
return llvm::make_error<llvm::StringError>(DiagnosticsOS.str(),
llvm::inconvertibleErrorCode());
}

llvm::Error DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
StringRef ModuleName) {
// Capture the emitted diagnostics and report them to the client
// in the case of a failure.
std::string DiagnosticOutput;
llvm::raw_string_ostream DiagnosticsOS(DiagnosticOutput);
auto DiagOpts = createDiagOptions(CommandLine);
TextDiagnosticPrinter DiagPrinter(DiagnosticsOS, DiagOpts.release());

if (computeDependencies(WorkingDirectory, CommandLine, Consumer, Controller,
Expand Down Expand Up @@ -828,59 +853,18 @@ static bool createAndRunToolInvocation(
return true;
}

bool DependencyScanningWorker::computeDependencies(
bool DependencyScanningWorker::scanDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
DiagnosticConsumer &DC, std::optional<StringRef> ModuleName) {
// Reset what might have been modified in the previous worker invocation.
BaseFS->setCurrentWorkingDirectory(WorkingDirectory);

std::optional<std::vector<std::string>> ModifiedCommandLine;
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> ModifiedFS;

// If we're scanning based on a module name alone, we don't expect the client
// to provide us with an input file. However, the driver really wants to have
// one. Let's just make it up to make the driver happy.
if (ModuleName) {
auto OverlayFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
auto InMemoryFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);

SmallString<128> FakeInputPath;
// TODO: We should retry the creation if the path already exists.
llvm::sys::fs::createUniquePath(*ModuleName + "-%%%%%%%%.input",
FakeInputPath,
/*MakeAbsolute=*/false);
InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer(""));

llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay =
InMemoryFS;
// If we are using a CAS but not dependency CASFS, we need to provide the
// fake input file in a CASProvidingFS for include-tree.
if (CAS && !DepCASFS)
InMemoryOverlay =
llvm::cas::createCASProvidingFileSystem(CAS, std::move(InMemoryFS));

OverlayFS->pushOverlay(InMemoryOverlay);
ModifiedFS = OverlayFS;
ModifiedCommandLine = CommandLine;
ModifiedCommandLine->emplace_back(FakeInputPath);
}

const std::vector<std::string> &FinalCommandLine =
ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
auto &FinalFS = ModifiedFS ? ModifiedFS : BaseFS;

DiagnosticConsumer &DC, llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
std::optional<StringRef> ModuleName) {
auto FileMgr =
llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOptions{}, FinalFS);
llvm::makeIntrusiveRefCnt<FileManager>(FileSystemOptions{}, FS);

std::vector<const char *> FinalCCommandLine(FinalCommandLine.size(), nullptr);
llvm::transform(FinalCommandLine, FinalCCommandLine.begin(),
std::vector<const char *> CCommandLine(CommandLine.size(), nullptr);
llvm::transform(CommandLine, CCommandLine.begin(),
[](const std::string &Str) { return Str.c_str(); });

auto DiagOpts = CreateAndPopulateDiagOpts(FinalCCommandLine);
auto DiagOpts = CreateAndPopulateDiagOpts(CCommandLine);
sanitizeDiagOpts(*DiagOpts);
IntrusiveRefCntPtr<DiagnosticsEngine> Diags =
CompilerInstance::createDiagnostics(DiagOpts.release(), &DC,
Expand All @@ -902,12 +886,12 @@ bool DependencyScanningWorker::computeDependencies(
/*DiagGenerationAsCompilation=*/false, getCASOpts(),
ModuleName);
bool Success = false;
if (FinalCommandLine[1] == "-cc1") {
Success = createAndRunToolInvocation(FinalCommandLine, Action, *FileMgr,
if (CommandLine[1] == "-cc1") {
Success = createAndRunToolInvocation(CommandLine, Action, *FileMgr,
PCHContainerOps, *Diags, Consumer);
} else {
Success = forEachDriverJob(
FinalCommandLine, *Diags, *FileMgr, [&](const driver::Command &Cmd) {
CommandLine, *Diags, *FileMgr, [&](const driver::Command &Cmd) {
if (StringRef(Cmd.getCreator().getName()) != "clang") {
// Non-clang command. Just pass through to the dependency
// consumer.
Expand Down Expand Up @@ -935,10 +919,89 @@ bool DependencyScanningWorker::computeDependencies(

if (Success && !Action.hasScanned())
Diags->Report(diag::err_fe_expected_compiler_job)
<< llvm::join(FinalCommandLine, " ");
<< llvm::join(CommandLine, " ");
return Success && Action.hasScanned();
}

bool DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
DiagnosticConsumer &DC, std::optional<llvm::MemoryBufferRef> TUBuffer) {
// Reset what might have been modified in the previous worker invocation.
BaseFS->setCurrentWorkingDirectory(WorkingDirectory);

std::optional<std::vector<std::string>> ModifiedCommandLine;
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> ModifiedFS;

// If we're scanning based on a module name alone, we don't expect the client
// to provide us with an input file. However, the driver really wants to have
// one. Let's just make it up to make the driver happy.
if (TUBuffer) {
auto OverlayFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
auto InMemoryFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
auto InputPath = TUBuffer->getBufferIdentifier();
InMemoryFS->addFile(
InputPath, 0,
llvm::MemoryBuffer::getMemBufferCopy(TUBuffer->getBuffer()));
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay =
InMemoryFS;
// If we are using a CAS but not dependency CASFS, we need to provide the
// fake input file in a CASProvidingFS for include-tree.
if (CAS && !DepCASFS)
InMemoryOverlay =
llvm::cas::createCASProvidingFileSystem(CAS, std::move(InMemoryFS));

OverlayFS->pushOverlay(InMemoryOverlay);
ModifiedFS = OverlayFS;
ModifiedCommandLine = CommandLine;
ModifiedCommandLine->emplace_back(InputPath);
}

const std::vector<std::string> &FinalCommandLine =
ModifiedCommandLine ? *ModifiedCommandLine : CommandLine;
auto &FinalFS = ModifiedFS ? ModifiedFS : BaseFS;

return scanDependencies(WorkingDirectory, FinalCommandLine, Consumer,
Controller, DC, FinalFS, /*ModuleName=*/std::nullopt);
}

bool DependencyScanningWorker::computeDependencies(
StringRef WorkingDirectory, const std::vector<std::string> &CommandLine,
DependencyConsumer &Consumer, DependencyActionController &Controller,
DiagnosticConsumer &DC, StringRef ModuleName) {
// Reset what might have been modified in the previous worker invocation.
BaseFS->setCurrentWorkingDirectory(WorkingDirectory);

// If we're scanning based on a module name alone, we don't expect the client
// to provide us with an input file. However, the driver really wants to have
// one. Let's just make it up to make the driver happy.
auto OverlayFS =
llvm::makeIntrusiveRefCnt<llvm::vfs::OverlayFileSystem>(BaseFS);
auto InMemoryFS = llvm::makeIntrusiveRefCnt<llvm::vfs::InMemoryFileSystem>();
InMemoryFS->setCurrentWorkingDirectory(WorkingDirectory);
SmallString<128> FakeInputPath;
// TODO: We should retry the creation if the path already exists.
llvm::sys::fs::createUniquePath(ModuleName + "-%%%%%%%%.input", FakeInputPath,
/*MakeAbsolute=*/false);
InMemoryFS->addFile(FakeInputPath, 0, llvm::MemoryBuffer::getMemBuffer(""));
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> InMemoryOverlay = InMemoryFS;
// If we are using a CAS but not dependency CASFS, we need to provide the
// fake input file in a CASProvidingFS for include-tree.
if (CAS && !DepCASFS)
InMemoryOverlay =
llvm::cas::createCASProvidingFileSystem(CAS, std::move(InMemoryFS));

OverlayFS->pushOverlay(InMemoryOverlay);
auto ModifiedCommandLine = CommandLine;
ModifiedCommandLine.emplace_back(FakeInputPath);

return scanDependencies(WorkingDirectory, ModifiedCommandLine, Consumer,
Controller, DC, OverlayFS, ModuleName);
}

DependencyActionController::~DependencyActionController() {}

void DependencyScanningWorker::computeDependenciesFromCompilerInvocation(
Expand Down
Loading