fix performance of large graphs for skip binaries computation (#17436)

* fix performance of large graphs for skip binaries computation

* some minor low hanging fruit

conans/client/graph/graph.py

@@ -116,7 +116,7 @@ def propagate_downstream(self, require, node, src_node=None):
             return
 
         if src_node is not None:  # This happens when closing a loop, and we need to know the edge
-            d = [d for d in self.dependants if d.src is src_node][0]  # TODO: improve ugly
+            d = next(d for d in self.dependants if d.src is src_node)
         else:
             assert len(self.dependants) == 1
             d = self.dependants[0]
@@ -131,7 +131,7 @@ def propagate_downstream(self, require, node, src_node=None):
         # But if the files are not needed in this graph branch, can be marked "Skip"
         if down_require.files:
             down_require.required_nodes = require.required_nodes.copy()
-        down_require.required_nodes.append(self)
+        down_require.required_nodes.add(self)
         return d.src.propagate_downstream(down_require, node)
 
     def check_downstream_exists(self, require):
@@ -216,7 +216,6 @@ def pref(self):
 
     def add_edge(self, edge):
         if edge.src == self:
-            assert edge not in self.dependencies
             self.dependencies.append(edge)
         else:
             self.dependants.append(edge)
@@ -360,8 +359,8 @@ def root(self):
     def add_node(self, node):
         self.nodes.append(node)
 
-    def add_edge(self, src, dst, require):
-        assert src in self.nodes and dst in self.nodes
+    @staticmethod
+    def add_edge(src, dst, require):
         edge = Edge(src, dst, require)
         src.add_edge(edge)
         dst.add_edge(edge)

conans/model/requires.py

@@ -36,7 +36,7 @@ def __init__(self, ref, *, headers=None, libs=None, build=False, run=None, visib
         self.override_ref = None  # to store if the requirement has been overriden (store new ref)
         self.is_test = test  # to store that it was a test, even if used as regular requires too
         self.skip = False
-        self.required_nodes = []  # store which intermediate nodes are required, to compute "Skip"
+        self.required_nodes = set()  # store which intermediate nodes are required, to compute "Skip"
 
     @property
     def files(self):  # require needs some files in dependency package
@@ -260,7 +260,7 @@ def aggregate(self, other):
         # current require already defined it or not
         if self.package_id_mode is None:
             self.package_id_mode = other.package_id_mode
-        self.required_nodes.extend(other.required_nodes)
+        self.required_nodes.update(other.required_nodes)
 
     def transform_downstream(self, pkg_type, require, dep_pkg_type):
         """

conans/model/settings.py

@@ -282,7 +282,7 @@ def __delattr__(self, field):
         del self._data[field]
 
     def __setattr__(self, field, value):
-        if field[0] == "_" or field.startswith("values"):
+        if field[0] == "_":
             return super(Settings, self).__setattr__(field, value)
 
         self._check_field(field)

test/performance/test_large_graph.py

@@ -0,0 +1,85 @@
+import cProfile
+import json
+import pstats
+import time
+from pstats import SortKey
+import pytest
+
+from conan.test.assets.genconanfile import GenConanfile
+from conan.test.utils.tools import TestClient
+
+
+@pytest.mark.skip(reason="This is a performance test, skip for normal runs")
+def test_large_graph():
+    c = TestClient(cache_folder="T:/mycache")
+    num_test = 40
+    num_pkgs = 40
+
+    """for i in range(num_test):
+        conanfile = GenConanfile(f"test{i}", "0.1")
+        if i > 0:
+            conanfile.with_requires(f"test{i-1}/0.1")
+        c.save({"conanfile.py": conanfile})
+        c.run("create .")
+
+    for i in range(num_pkgs):
+        conanfile = GenConanfile(f"pkg{i}", "0.1").with_test_requires(f"test{num_test-1}/0.1")
+        if i > 0:
+            conanfile.with_requires(f"pkg{i-1}/0.1")
+        c.save({"conanfile.py": conanfile})
+        c.run("create .")
+
+    """
+    t = time.time()
+    pr = cProfile.Profile()
+    pr.enable()
+    c.run(f"install --requires=pkg{num_pkgs - 1}/0.1")
+    pr.disable()
+    print(time.time()-t)
+
+    sortby = SortKey.CUMULATIVE
+    ps = pstats.Stats(pr).sort_stats(sortby)
+    ps.print_stats()
+
+    #graph = json.loads(c.stdout)
+    #assert len(graph["graph"]["nodes"]) == 1 + num_pkgs + num_test * num_pkgs
+
+
+@pytest.mark.skip(reason="This is a performance test, skip for normal runs")
+def test_large_graph2():
+    c = TestClient(cache_folder="T:/mycache")
+    num_test = 20
+    num_pkgs = 20
+    branches = ["a", "b", "c"]
+
+    c.save({"conanfile.py": GenConanfile("testbase", "0.1")})
+    c.run("export .")
+    for i in range(num_test):
+        for branch in branches:
+            conanfile = GenConanfile(f"test{branch}{i}", "0.1")
+            if i > 0:
+                conanfile.with_requires(f"test{branch}{i-1}/0.1", "testbase/0.1")
+            else:
+                conanfile.with_requires("testbase/0.1")
+            c.save({"conanfile.py": conanfile})
+            c.run("export .")
+
+    for i in range(num_pkgs):
+        conanfile = GenConanfile(f"pkg{i}", "0.1")
+        for branch in branches:
+            conanfile.with_test_requires(f"test{branch}{num_test-1}/0.1")
+        if i > 0:
+            conanfile.with_requires(f"pkg{i-1}/0.1")
+        c.save({"conanfile.py": conanfile})
+        c.run("export .")
+
+    t = time.time()
+    pr = cProfile.Profile()
+    pr.enable()
+    c.run(f"graph info --requires=pkg{num_pkgs - 1}/0.1")
+    pr.disable()
+    print(time.time()-t)
+
+    sortby = SortKey.CUMULATIVE
+    ps = pstats.Stats(pr).sort_stats(sortby)
+    ps.print_stats()