Skip to content

Commit

Permalink
Implement 'dynout' feature to inform ninja about dynamic outputs
Browse files Browse the repository at this point in the history 
Co-authored-by: Hampus Adolfsson <[email protected]>
  • Loading branch information
@Dragnalith @HampusAdolfsson
Dragnalith and HampusAdolfsson committed May 20, 2024
1 parent 7ab135b commit 10f6f0f
Show file tree
Hide file tree
Showing 16 changed files with 681 additions and 37 deletions.
7 changes: 7 additions & 0 deletions doc/manual.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -900,6 +900,13 @@ keys.
stored as `.ninja_deps` in the `builddir`, see <<ref_toplevel,the
discussion of `builddir`>>.

`dynout`:: path to an optional _dynout file_ that contains extra _implicit
outputs_ generated by the rule. This allows ninja to dynamically discover
output files whose presence is decided during the build, so that for
subsequent builds the edge is re-run if some dynamic output is missing, and
dynamic outputs are cleaned when using the `-t clean` tool. The dynout file
syntax expects one path per line.

`msvc_deps_prefix`:: _(Available since Ninja 1.5.)_ defines the string
which should be stripped from msvc's /showIncludes output. Only
needed when `deps = msvc` and no English Visual Studio version is used.
Expand Down
75 changes: 73 additions & 2 deletions src/build.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
#include "depfile_parser.h"
#include "deps_log.h"
#include "disk_interface.h"
#include "dynout_parser.h"
#include "graph.h"
#include "metrics.h"
#include "state.h"
Expand Down Expand Up @@ -695,6 +696,7 @@ void Builder::Cleanup() {
for (vector<Edge*>::iterator e = active_edges.begin();
e != active_edges.end(); ++e) {
string depfile = (*e)->GetUnescapedDepfile();
string dynout = (*e)->GetUnescapedDynout();
for (vector<Node*>::iterator o = (*e)->outputs_.begin();
o != (*e)->outputs_.end(); ++o) {
// Only delete this output if it was actually modified. This is
Expand All @@ -713,6 +715,8 @@ void Builder::Cleanup() {
}
if (!depfile.empty())
disk_interface_->RemoveFile(depfile);
if (!dynout.empty())
disk_interface_->RemoveFile(dynout);
}
}

Expand Down Expand Up @@ -946,6 +950,18 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) {
}
}

int outputs_count = 0;
string extract_err;
std::string dynout_file = edge->GetUnescapedDynout();
if (!ExtractDynouts(edge, dynout_file, &deps_nodes, &outputs_count,
&extract_err) &&
result->success()) {
if (!result->output.empty())
result->output.append("\n");
result->output.append(extract_err);
result->status = ExitFailure;
}

int64_t start_time_millis, end_time_millis;
RunningEdgeMap::iterator it = running_edges_.find(edge);
start_time_millis = it->second;
Expand Down Expand Up @@ -1012,14 +1028,14 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) {
}
}

if (!deps_type.empty() && !config_.dry_run) {
if ((!deps_type.empty() || !dynout_file.empty()) && !config_.dry_run) {
assert(!edge->outputs_.empty() && "should have been rejected by parser");
for (std::vector<Node*>::const_iterator o = edge->outputs_.begin();
o != edge->outputs_.end(); ++o) {
TimeStamp deps_mtime = disk_interface_->Stat((*o)->path(), err);
if (deps_mtime == -1)
return false;
if (!scan_.deps_log()->RecordDeps(*o, deps_mtime, deps_nodes, 0)) {
if (!scan_.deps_log()->RecordDeps(*o, deps_mtime, deps_nodes, outputs_count)) {
*err = std::string("Error writing to deps log: ") + strerror(errno);
return false;
}
Expand Down Expand Up @@ -1106,3 +1122,58 @@ bool Builder::LoadDyndeps(Node* node, string* err) {

return true;
}

bool Builder::ExtractDynouts(Edge* edge, const std::string& dynout_file,
std::vector<Node*>* nodes, int* outputs_count,
std::string* err) {
if (dynout_file.empty()) {
return true;
}

// Read depfile content. Treat a missing depfile as empty.
std::string content;
switch (disk_interface_->ReadFile(dynout_file, &content, err)) {
case DiskInterface::Okay:
break;
case DiskInterface::NotFound:
if (err != NULL) {
err->clear();
}
break;
case DiskInterface::OtherError:
if (err != NULL) {
*err = "loading '" + dynout_file + "': " + *err;
}
return false;
}

std::vector<StringPiece> output_paths;
std::string parse_err;
if (!DynoutParser::Parse(content, output_paths, &parse_err)) {
if (err != NULL) {
*err = parse_err;
}
return false;
}

int start_size = nodes->size();

for (const StringPiece &p : output_paths) {
uint64_t slash_bits;
std::string canonical = p.AsString();
CanonicalizePath(&canonical, &slash_bits);
Node* new_node = state_->GetNode(canonical, slash_bits);
nodes->push_back(new_node);
}
*outputs_count = (int) nodes->size() - start_size;

if (!g_keep_dynout) {
if (disk_interface_->RemoveFile(dynout_file) < 0) {
if (err != NULL) {
*err = std::string("deleting dynout: ") + strerror(errno) + std::string("\n");
}
return false;
}
}
return true;
}
4 changes: 4 additions & 0 deletions src/build.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,10 @@ struct Builder {
const std::string& deps_prefix,
std::vector<Node*>* deps_nodes, std::string* err);

bool ExtractDynouts(Edge* edge, const std::string& dynout_file,
std::vector<Node*>* nodes, int* outputs_count,
std::string* err);

/// Map of running edge to time the edge started running.
typedef std::map<const Edge*, int> RunningEdgeMap;
RunningEdgeMap running_edges_;
Expand Down
177 changes: 177 additions & 0 deletions src/build_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -660,6 +660,18 @@ bool FakeCommandRunner::StartCommand(Edge* edge) {
if (fs_->ReadFile(edge->inputs_[0]->path(), &content, &err) ==
DiskInterface::Okay)
fs_->WriteFile(edge->outputs_[0]->path(), content);
} else if (edge->rule().name() == "cp-plus-bis") {
assert(!edge->inputs_.empty());
assert(edge->outputs_.size() >= 1);
string content;
string err;
if (fs_->ReadFile(edge->inputs_[0]->path(), &content, &err) ==
DiskInterface::Okay) {
fs_->Tick();
fs_->WriteFile(edge->outputs_[0]->path() + ".bis", content);
fs_->Tick();
fs_->WriteFile(edge->outputs_[0]->path(), content);
}
} else if (edge->rule().name() == "touch-implicit-dep-out") {
string dep = edge->GetBinding("test_dependency");
fs_->Tick();
Expand Down Expand Up @@ -4381,3 +4393,168 @@ TEST_F(BuildTest, ValidationWithCircularDependency) {
EXPECT_FALSE(builder_.AddTarget("out", &err));
EXPECT_EQ("dependency cycle: validate -> validate_in -> validate", err);
}

TEST_F(BuildWithDepsLogTest, RebuildMissingDynamicOutputs) {
string err;
BuildLog build_log;

{
FakeCommandRunner command_runner(&fs_);
State state;
DepsLog deps_log;
ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err));
ASSERT_EQ("", err);
Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner);

ASSERT_NO_FATAL_FAILURE(
AssertParse(&state,
"rule cp-plus-bis\n"
" command = cp $in $out && cp $in $out.bis\n"
" dynout = $out.dynout\n"
"build out: cp-plus-bis in\n"));

fs_.Tick();
fs_.Create("in", "");
fs_.Create("out.dynout", "out.bis\n");

EXPECT_TRUE(builder.AddTarget("out", &err));
EXPECT_TRUE(builder.Build(&err));
ASSERT_EQ("", err);
ASSERT_EQ(1u, command_runner.commands_ran_.size());
ASSERT_GT(fs_.Stat("out", &err), 0);
ASSERT_GT(fs_.Stat("out.bis", &err), 0);
// Make sure the dynout file has been removed after its
// information has been extracted in the deps log.
ASSERT_EQ(fs_.Stat("out.dynout", &err), 0);

// all clean, no rebuild.
command_runner.commands_ran_.clear();
state.Reset();
EXPECT_TRUE(builder.AddTarget("out", &err));
EXPECT_EQ("", err);
EXPECT_TRUE(builder.AlreadyUpToDate());

// Test dynamic output are rebuilt if they are deleted.
fs_.RemoveFile("out.bis");
command_runner.commands_ran_.clear();
state.Reset();

fs_.Create("out.dynout", "out.bis\n");
EXPECT_TRUE(builder.AddTarget("out", &err));
EXPECT_TRUE(builder.Build(&err));
ASSERT_EQ("", err);
ASSERT_EQ(1u, command_runner.commands_ran_.size());

builder.command_runner_.release();
deps_log.Close();
}

// Create a new state to make sure outputs are reset
{
FakeCommandRunner command_runner(&fs_);
State state;
DepsLog deps_log;
ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err));
ASSERT_EQ("", err);
deps_log.Load(deps_log_file_.path(), &state, &err);
ASSERT_EQ("", err);
Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner);

ASSERT_NO_FATAL_FAILURE(
AssertParse(&state,
"rule cp-plus-bis\n"
" command = cp $in $out && cp $in $out.bis\n"
" dynout = $out.dynout\n"
"build out: cp-plus-bis in\n"));

// all clean, no rebuild.
command_runner.commands_ran_.clear();
state.Reset();
EXPECT_TRUE(builder.AddTarget("out", &err));
EXPECT_EQ("", err);
EXPECT_TRUE(builder.AlreadyUpToDate());

// Test dynamic output are rebuilt if they are deleted
// after having been rebuilt in order to make sure
// when dynamic output information was already exist they
// keep being valid for the next build.
fs_.RemoveFile("out.bis");
command_runner.commands_ran_.clear();
state.Reset();

// Recreate the dynout file because it is not created by the edge
fs_.Create("out.dynout", "out.bis\n");
EXPECT_TRUE(builder.AddTarget("out", &err));
EXPECT_TRUE(builder.Build(&err));
ASSERT_EQ("", err);
ASSERT_EQ(1u, command_runner.commands_ran_.size());

builder.command_runner_.release();
deps_log.Close();
}
}

TEST_F(BuildWithDepsLogTest, RebuildMissingDynamicOutputsWithRestat) {
string err;

FakeCommandRunner command_runner(&fs_);
BuildLog build_log;
State state;
DepsLog deps_log;
ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err));
ASSERT_EQ("", err);
Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0);
builder.command_runner_.reset(&command_runner);

ASSERT_NO_FATAL_FAILURE(
AssertParse(&state,
"rule cp-plus-bis\n"
" command = cp $in $out && cp $in $out.bis\n"
" dynout = $out.dynout\n"
" restat = 1\n"
"build out: cp-plus-bis in\n"));

fs_.Tick();
fs_.Create("in", "");
fs_.Tick();
fs_.Create("out.dynout", "out.bis\n");
fs_.Tick();

EXPECT_TRUE(builder.AddTarget("out", &err));
EXPECT_TRUE(builder.Build(&err));
ASSERT_EQ("", err);
ASSERT_EQ(1u, command_runner.commands_ran_.size());
ASSERT_GT(fs_.Stat("out", &err), 0);
ASSERT_GT(fs_.Stat("out.bis", &err), 0);
// Make sure the dynout file has been removed after its
// information has been extracted in the deps log.
ASSERT_EQ(fs_.Stat("out.dynout", &err), 0);

// all clean, no rebuild.
command_runner.commands_ran_.clear();
state.Reset();
EXPECT_TRUE(builder.AddTarget("out", &err));
EXPECT_EQ("", err);
EXPECT_TRUE(builder.AlreadyUpToDate());

fs_.RemoveFile("out.bis");
command_runner.commands_ran_.clear();
state.Reset();

// Recreate the dynout file because it is not created by the edge
fs_.Create("out.dynout", "out.bis\n");
EXPECT_TRUE(builder.AddTarget("out", &err));
EXPECT_TRUE(builder.Build(&err));
ASSERT_EQ("", err);
ASSERT_EQ(1u, command_runner.commands_ran_.size());

// Make sure the dynout file has been removed after its
// information has been extracted in the deps log.
ASSERT_EQ(fs_.Stat("out.dynout", &err), 0);

builder.command_runner_.release();

deps_log.Close();
}
Loading

0 comments on commit 10f6f0f

Please sign in to comment.