Introduce new tool: -t compdb-targets

Fixes ninja-build#1544

Co-authored-by: Linkun Chen <[email protected]>
Co-authored-by: csmoe <[email protected]>
Co-authored-by: James Widman <[email protected]>

doc/manual.asciidoc

@@ -289,6 +289,11 @@ http://clang.llvm.org/docs/JSONCompilationDatabase.html[JSON format] expected
 by the Clang tooling interface.
 _Available since Ninja 1.2._
 
+`compdb-targets`:: like `compdb`, but takes a list of targets instead of rules,
+and expects at least one target. The resulting compilation database contains
+all commands required to build the indicated targets, and _only_ those
+commands.
+
 `deps`:: show all dependencies stored in the `.ninja_deps` file. When given a
 target, show just the target's dependencies. _Available since Ninja 1.4._
 

doc/style.css

@@ -53,3 +53,7 @@ div.chapter {
 p {
     margin-top: 0;
 }
+
+code.literal {
+    white-space: nowrap;
+}

misc/output_test.py

@@ -12,14 +12,49 @@
 import tempfile
 import unittest
 from textwrap import dedent
-from typing import Dict
+import typing as T
 
 default_env = dict(os.environ)
 default_env.pop('NINJA_STATUS', None)
 default_env.pop('CLICOLOR_FORCE', None)
 default_env['TERM'] = ''
 NINJA_PATH = os.path.abspath('./ninja')
 
+def remove_non_visible_lines(raw_output: bytes) -> str:
+  # When running in a smart terminal, Ninja uses CR (\r) to
+  # return the cursor to the start of the current line, prints
+  # something, then uses `\x1b[K` to clear everything until
+  # the end of the line.
+  #
+  # Thus printing 'FOO', 'BAR', 'ZOO' on the same line, then
+  # jumping to the next one results in the following output
+  # on Posix:
+  #
+  # '\rFOO\x1b[K\rBAR\x1b[K\rZOO\x1b[K\r\n'
+  #
+  # The following splits the output at both \r, \n and \r\n
+  # boundaries, which gives:
+  #
+  #  [ '\r', 'FOO\x1b[K\r', 'BAR\x1b[K\r', 'ZOO\x1b[K\r\n' ]
+  #
+  decoded_lines = raw_output.decode('utf-8').splitlines(True)
+
+  # Remove any item that ends with a '\r' as this means its
+  # content will be overwritten by the next item in the list.
+  # For the previous example, this gives:
+  #
+  #  [ 'ZOO\x1b[K\r\n' ]
+  #
+  final_lines = [ l for l in decoded_lines if not l.endswith('\r') ]
+
+  # Return a single string that concatenates all filtered lines
+  # while removing any remaining \r in it. Needed to transform
+  # \r\n into \n.
+  #
+  #  "ZOO\x1b[K\n'
+  #
+  return ''.join(final_lines).replace('\r', '')
+
 class BuildDir:
     def __init__(self, build_ninja: str):
         self.build_ninja = dedent(build_ninja)
@@ -35,12 +70,18 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.d.cleanup()
 
+    @property
+    def path(self) -> str:
+        return os.path.realpath(self.d.name)
+
+
     def run(
         self,
-        flags: str = '',
+        flags: T.Optional[str] = None,
         pipe: bool = False,
         raw_output: bool = False,
-        env: Dict[str, str] = default_env,
+        env: T.Dict[str, str] = default_env,
+        print_err_output = True,
     ) -> str:
         """Run Ninja command, and get filtered output.
 
@@ -56,13 +97,17 @@ def run(
 
           env: Optional environment dictionary to run the command in.
 
+          print_err_output: set to False if the test expects ninja to print
+            something to stderr. (Otherwise, an error message from Ninja
+            probably represents a failed test.)
+
         Returns:
           A UTF-8 string corresponding to the output (stdout only) of the
           Ninja command. By default, partial lines that were overwritten
           are removed according to the rules described in the comments
           below.
         """
-        ninja_cmd = '{} {}'.format(NINJA_PATH, flags)
+        ninja_cmd = '{} {}'.format(NINJA_PATH, flags if flags else '')
         try:
             if pipe:
                 output = subprocess.check_output(
@@ -74,57 +119,27 @@ def run(
                 output = subprocess.check_output(['script', '-qfec', ninja_cmd, '/dev/null'],
                                                  cwd=self.d.name, env=env)
         except subprocess.CalledProcessError as err:
-            sys.stdout.buffer.write(err.output)
+            if print_err_output:
+              sys.stdout.buffer.write(err.output)
+            err.cooked_output = remove_non_visible_lines(err.output)
             raise err
 
         if raw_output:
             return output.decode('utf-8')
-
-        # When running in a smart terminal, Ninja uses CR (\r) to
-        # return the cursor to the start of the current line, prints
-        # something, then uses `\x1b[K` to clear everything until
-        # the end of the line.
-        #
-        # Thus printing 'FOO', 'BAR', 'ZOO' on the same line, then
-        # jumping to the next one results in the following output
-        # on Posix:
-        #
-        # '\rFOO\x1b[K\rBAR\x1b[K\rZOO\x1b[K\r\n'
-        #
-        # The following splits the output at both \r, \n and \r\n
-        # boundaries, which gives:
-        #
-        #  [ '\r', 'FOO\x1b[K\r', 'BAR\x1b[K\r', 'ZOO\x1b[K\r\n' ]
-        #
-        decoded_lines = output.decode('utf-8').splitlines(True)
-
-        # Remove any item that ends with a '\r' as this means its
-        # content will be overwritten by the next item in the list.
-        # For the previous example, this gives:
-        #
-        #  [ 'ZOO\x1b[K\r\n' ]
-        #
-        final_lines = [ l for l in decoded_lines if not l.endswith('\r') ]
-
-        # Return a single string that concatenates all filtered lines
-        # while removing any remaining \r in it. Needed to transform
-        # \r\n into \n.
-        #
-        #  "ZOO\x1b[K\n'
-        #
-        return ''.join(final_lines).replace('\r', '')
+        return remove_non_visible_lines(output)
 
 def run(
     build_ninja: str,
-    flags: str = '',
+    flags: T.Optional[str] = None,
     pipe: bool = False,
     raw_output: bool = False,
-    env: Dict[str, str] = default_env,
+    env: T.Dict[str, str] = default_env,
+    print_err_output = True,
 ) -> str:
     """Run Ninja with a given build plan in a temporary directory.
     """
     with BuildDir(build_ninja) as b:
-        return b.run(flags, pipe, raw_output, env)
+        return b.run(flags, pipe, raw_output, env, print_err_output)
 
 @unittest.skipIf(platform.system() == 'Windows', 'These test methods do not work on Windows')
 class Output(unittest.TestCase):
@@ -137,6 +152,16 @@ class Output(unittest.TestCase):
         '',
     ))
 
+    def _test_expected_error(self, plan: str, flags: T.Optional[str], expected: str):
+        """Run Ninja with a given plan and flags, and verify its cooked output against an expected content.
+        """
+        actual = ''
+        try:
+          actual = run(plan, flags, print_err_output=False)
+        except subprocess.CalledProcessError as err:
+          actual = err.cooked_output
+        self.assertEqual(expected, actual)
+
     def test_issue_1418(self) -> None:
         self.assertEqual(run(
 '''rule echo
@@ -371,6 +396,59 @@ def test_tool_inputs(self) -> None:
         )
 
 
+    def test_tool_compdb_targets(self) -> None:
+        plan = '''
+rule cat
+  command = cat $in $out
+build out1 : cat in1
+build out2 : cat in2 out1
+build out3 : cat out2 out1
+build out4 : cat in4
+'''
+
+
+        self._test_expected_error(plan, '-t compdb-targets',
+'''ninja: error: compdb-targets expects the name of at least one target
+usage: ninja -t compdb [-hx] target [targets]
+
+options:
+  -h     display this help message
+  -x     expand @rspfile style response file invocations
+''')
+
+        self._test_expected_error(plan, '-t compdb-targets in1',
+            "ninja: fatal: 'in1' is not a target (i.e. it is not an output of any `build` statement)\n")
+
+        self._test_expected_error(plan, '-t compdb-targets nonexistent_target',
+            "ninja: fatal: unknown target 'nonexistent_target'\n")
+
+
+        with BuildDir(plan) as b:
+            actual = b.run(flags='-t compdb-targets out3')
+            expected = f'''[
+  {{
+    "directory": "{b.path}",
+    "command": "cat in1 out1",
+    "file": "in1",
+    "output": "out1"
+  }},
+  {{
+    "directory": "{b.path}",
+    "command": "cat in2 out1 out2",
+    "file": "in2",
+    "output": "out2"
+  }},
+  {{
+    "directory": "{b.path}",
+    "command": "cat out2 out1 out3",
+    "file": "out2",
+    "output": "out3"
+  }}
+]
+'''
+            self.assertEqual(expected, actual)
+
+
     def test_explain_output(self):
         b = BuildDir('''\
             build .FORCE: phony

src/command_collector.h

@@ -0,0 +1,65 @@
+// Copyright 2024 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef NINJA_COMMAND_COLLECTOR_H_
+#define NINJA_COMMAND_COLLECTOR_H_
+
+#include <cassert>
+#include <unordered_set>
+#include <vector>
+
+#include "graph.h"
+
+/// Collects the transitive set of edges that lead into a given set
+/// of starting nodes. Used to implement the `compdb-targets` tool.
+///
+/// When collecting inputs, the outputs of phony edges are always ignored
+/// from the result, but are followed by the dependency walk.
+///
+/// Usage is:
+/// - Create instance.
+/// - Call CollectFrom() for each root node to collect edges from.
+/// - Call TakeResult() to retrieve the list of edges.
+///
+struct CommandCollector {
+  void CollectFrom(const Node* node) {
+    assert(node);
+
+    if (!visited_nodes_.insert(node).second)
+      return;
+
+    Edge* edge = node->in_edge();
+    if (!edge || !visited_edges_.insert(edge).second)
+      return;
+
+    for (Node* input_node : edge->inputs_)
+      CollectFrom(input_node);
+
+    if (!edge->is_phony())
+      in_edges.push_back(edge);
+  }
+
+ private:
+  std::unordered_set<const Node*> visited_nodes_;
+  std::unordered_set<Edge*> visited_edges_;
+
+  /// we use a vector to preserve order from requisites to their dependents.
+  /// This may help LSP server performance in languages that support modules,
+  /// but it also ensures that the output of `-t compdb-targets foo` is
+  /// consistent, which is useful in regression tests.
+ public:
+  std::vector<Edge*> in_edges;
+};
+
+#endif  //  NINJA_COMMAND_COLLECTOR_H_

src/graph_test.cc

@@ -15,6 +15,7 @@
 #include "graph.h"
 
 #include "build.h"
+#include "command_collector.h"
 #include "test.h"
 
 using namespace std;
@@ -310,6 +311,54 @@ TEST_F(GraphTest, InputsCollectorWithEscapes) {
   EXPECT_EQ("order_only", inputs[4]);
 }
 
+TEST_F(GraphTest, CommandCollector) {
+  ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
+                                      "build out1: cat in1\n"
+                                      "build mid1: cat in1\n"
+                                      "build out2: cat mid1\n"
+                                      "build out3 out4: cat mid1\n"
+                                      "build all: phony out1 out2 out3\n"));
+  {
+    CommandCollector collector;
+    auto& edges = collector.in_edges;
+
+    // Start visit from out2; this should add `build mid1` and `build out2` to
+    // the edge list.
+    collector.CollectFrom(GetNode("out2"));
+    ASSERT_EQ(2u, edges.size());
+    EXPECT_EQ("cat in1 > mid1", edges[0]->EvaluateCommand());
+    EXPECT_EQ("cat mid1 > out2", edges[1]->EvaluateCommand());
+
+    // Add a visit from out1, this should append `build out1`
+    collector.CollectFrom(GetNode("out1"));
+    ASSERT_EQ(3u, edges.size());
+    EXPECT_EQ("cat in1 > out1", edges[2]->EvaluateCommand());
+
+    // Another visit from all; this should add edges for out1, out2 and out3,
+    // but not all (because it's phony).
+    collector.CollectFrom(GetNode("all"));
+    ASSERT_EQ(4u, edges.size());
+    EXPECT_EQ("cat in1 > mid1", edges[0]->EvaluateCommand());
+    EXPECT_EQ("cat mid1 > out2", edges[1]->EvaluateCommand());
+    EXPECT_EQ("cat in1 > out1", edges[2]->EvaluateCommand());
+    EXPECT_EQ("cat mid1 > out3 out4", edges[3]->EvaluateCommand());
+  }
+
+  {
+    CommandCollector collector;
+    auto& edges = collector.in_edges;
+
+    // Starting directly from all, will add `build out1` before `build mid1`
+    // compared to the previous example above.
+    collector.CollectFrom(GetNode("all"));
+    ASSERT_EQ(4u, edges.size());
+    EXPECT_EQ("cat in1 > out1", edges[0]->EvaluateCommand());
+    EXPECT_EQ("cat in1 > mid1", edges[1]->EvaluateCommand());
+    EXPECT_EQ("cat mid1 > out2", edges[2]->EvaluateCommand());
+    EXPECT_EQ("cat mid1 > out3 out4", edges[3]->EvaluateCommand());
+  }
+}
+
 TEST_F(GraphTest, VarInOutPathEscaping) {
   ASSERT_NO_FATAL_FAILURE(AssertParse(&state_,
 "build a$ b: cat no'space with$ space$$ no\"space2\n"));