Improve multithreaded performance

tum-gis · Feb 18, 2024 · c207fe9 · c207fe9
1 parent 134a670
commit c207fe9
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 22 deletions.
diff --git a/src/main/java/jgraf/citygml/CityGMLNeo4jDB.java b/src/main/java/jgraf/citygml/CityGMLNeo4jDB.java
@@ -540,29 +540,32 @@ public boolean diff(int leftPartitionIndex, int rightPartitionIndex) {
 
         // Remaining multi-relationships in right
         logger.info("Checking for potential inserted top-level features");
+        ExecutorService esInsert = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors());
         AtomicInteger insertedRightCount = new AtomicInteger();
         if (!rightCOMIDs.isEmpty()) {
-            BatchUtils.toBatches(rightCOMIDs, 100).forEach(batch -> { // TODO Define a config variable for this
-                try (Transaction tx = graphDb.beginTx()) {
-                    Node leftCOMListNode = tx.getNodeByElementId(tmpLeftCOMListID);
-                    Node rightCOMListNode = tx.getNodeByElementId(tmpRightCOMListID);
-                    batch.forEach(rightCOMID -> {
-                        Node rightCOMNode = tx.getNodeByElementId(rightCOMID);
-                        Relationship rightRel = rightCOMNode.getRelationships(Direction.INCOMING).stream()
-                                .filter(r -> r.getStartNode().getElementId().equals(tmpRightCOMListID))
-                                .collect(Collectors.toSet()).iterator().next();
-                        new InsertNodeChange(tx, leftCOMListNode, rightCOMListNode, rightRel);
-                        insertedRightCount.getAndIncrement();
-                    });
-                    tx.commit();
-                } catch (Exception e) {
-                    logger.error(e.getMessage() + " (F)\n" + Arrays.toString(e.getStackTrace()));
-                }
-                logger.info("-> {} inserted top-level features", insertedRightCount);
-            });
+            BatchUtils.toBatches(rightCOMIDs, 10 * config.MATCHER_TOPLEVEL_BATCH_SIZE)
+                    .forEach(batch -> esInsert.submit(() -> {
+                        try (Transaction tx = graphDb.beginTx()) {
+                            Node leftCOMListNode = tx.getNodeByElementId(tmpLeftCOMListID);
+                            Node rightCOMListNode = tx.getNodeByElementId(tmpRightCOMListID);
+                            batch.forEach(rightCOMID -> {
+                                Node rightCOMNode = tx.getNodeByElementId(rightCOMID);
+                                Relationship rightRel = rightCOMNode.getRelationships(Direction.INCOMING).stream()
+                                        .filter(r -> r.getStartNode().getElementId().equals(tmpRightCOMListID))
+                                        .collect(Collectors.toSet()).iterator().next();
+                                new InsertNodeChange(tx, leftCOMListNode, rightCOMListNode, rightRel);
+                                insertedRightCount.getAndIncrement();
+                            });
+                            tx.commit();
+                        } catch (Exception e) {
+                            logger.error(e.getMessage() + " (F)\n" + Arrays.toString(e.getStackTrace()));
+                        }
+                        logger.info("-> {} inserted top-level features", insertedRightCount);
+                    }));
 
             diffFound.set(true);
         }
+        Neo4jDB.finishThreads(esInsert, config.MATCHER_CONCURRENT_TIMEOUT);
         logger.info("Found {} inserted top-level features", insertedRightCount);
 
         dbStats.stopTimer("Match city models indexed at " + leftPartitionIndex + " and " + rightPartitionIndex);

diff --git a/src/main/java/jgraf/citygml/Patterns.java b/src/main/java/jgraf/citygml/Patterns.java
@@ -180,7 +180,7 @@ public static void interpret(
         logger.info("|--> Applying pattern rules");
 
         // Init list of all top-level features // TODO Memory check?
-        List<String> nonDelToplevelIds = Collections.synchronizedList(new LinkedList<>());
+        List<String> nonDelToplevelIds = new ArrayList<>();
         try (Transaction tx = graphDb.beginTx()) {
             // Find all top-level features and their changes
             tx.findNodes(Label.label(Building.class.getName())).forEachRemaining(toplevel -> {

diff --git a/src/main/java/jgraf/neo4j/Neo4jDB.java b/src/main/java/jgraf/neo4j/Neo4jDB.java
@@ -488,7 +488,6 @@ public Neo4jGraphRef getRootMatcherRef() {
     }
 
     public static void finishThreads(ExecutorService executorService, long seconds) {
-        logger.info("Waiting for all threads to finish");
         executorService.shutdown();
         try {
             if (!executorService.awaitTermination(seconds, TimeUnit.SECONDS)) {

diff --git a/src/main/java/jgraf/utils/ChangeUtils.java b/src/main/java/jgraf/utils/ChangeUtils.java
@@ -64,9 +64,9 @@ public static void addChange(Transaction tx, Class<? extends Change> changeType,
         if (nodes != null) {
             for (Map.Entry<AuxEdgeTypes, Node> entry : nodes.entrySet()) {
                 Node target = entry.getValue();
-                Lock lock = tx.acquireWriteLock(target);
+                // Lock lock = tx.acquireWriteLock(target);
                 changeNode.createRelationshipTo(target, entry.getKey());
-                lock.release();
+                // lock.release();
             }
         }
         // logger.debug("Detected {}", changeType.getSimpleName());