From 9b8f4764299e66e160fea31e2d5443d3a574dcde Mon Sep 17 00:00:00 2001 From: Gang Liao Date: Sat, 21 Sep 2019 11:12:42 -0400 Subject: [PATCH 1/2] Optimize delete recursively --- .../java/org/apache/hadoop/hdfs/db/DatabaseINode.java | 11 +++++++++-- .../hadoop/hdfs/server/namenode/FSDirDeleteOp.java | 9 ++++++--- .../apache/hadoop/hdfs/server/namenode/INodeFile.java | 6 ++++-- voltdb/RemoveChild.java | 4 ++-- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-db/src/main/java/org/apache/hadoop/hdfs/db/DatabaseINode.java b/hadoop-hdfs-project/hadoop-hdfs-db/src/main/java/org/apache/hadoop/hdfs/db/DatabaseINode.java index 6db30c75a3e..58fe70c34c6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-db/src/main/java/org/apache/hadoop/hdfs/db/DatabaseINode.java +++ b/hadoop-hdfs-project/hadoop-hdfs-db/src/main/java/org/apache/hadoop/hdfs/db/DatabaseINode.java @@ -719,14 +719,20 @@ public static List getChildIdsByPath(final long id, final String[] compone return res; } - public static void removeChild(final long id) { + public static List removeChild(final long id) { + List childIds = new ArrayList<>(); try { DatabaseConnection obj = Database.getInstance().getConnection(); String env = System.getenv("DATABASE"); if (env.equals("VOLT")) { // call a stored procedure try { - obj.getVoltClient().callProcedure(new NullCallback(), "RemoveChild", id); + VoltTable[] results = obj.getVoltClient().callProcedure(new NullCallback(), "RemoveChild", id).getResults(); + VoltTable result = results[0]; + result.resetRowPosition(); + while (result.advanceRow()) { + childIds.add(result.getLong(0)); + } } catch (Exception e) { e.printStackTrace(); } @@ -755,6 +761,7 @@ public static void removeChild(final long id) { if (LOG.isInfoEnabled()) { LOG.info("removeChild: " + id); } + return childIds; } public static List getPathComponents(final long childId) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java index baefb15cff7..a2d1500366f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java @@ -265,9 +265,12 @@ private static boolean unprotectedDelete(FSDirectory fsd, INodesInPath iip, targetNode.cleanSubtree(reclaimContext, CURRENT_STATE_ID, latestSnapshot); } - // ADD(gangliao): Remove inode from Postgres - DatabaseINode.removeChild(targetNode.getId()); - INodeKeyedObjects.getCache().invalidateAllWithIndex(Long.class, targetNode.getId()); + // ADD(gangliao): Remove inode from Database + List ids = DatabaseINode.removeChild(targetNode.getId()); + INodeKeyedObjects.getCache().invalidateAllWithIndex(Long.class, targetNode.getId()); + for (Long id : ids) { + INodeKeyedObjects.getCache().invalidateAllWithIndex(Long.class, id); + } if (NameNode.stateChangeLog.isDebugEnabled()) { NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: " diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java index ae5c9962b7f..90fa5f292b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java @@ -832,8 +832,10 @@ private void setBlocks(INodeFile that) { /** Clear all blocks of the file. */ public void clearBlocks() { - blockNum.getAndSet(0); - DatabaseINode2Block.deleteViaBcId(this.getId()); + if (numBlocks() != 0) { + blockNum.getAndSet(0); + DatabaseINode2Block.deleteViaBcId(this.getId()); + } } private void updateRemovedUnderConstructionFiles( diff --git a/voltdb/RemoveChild.java b/voltdb/RemoveChild.java index 88d229a7d6b..0057b3cca26 100644 --- a/voltdb/RemoveChild.java +++ b/voltdb/RemoveChild.java @@ -14,7 +14,7 @@ public class RemoveChild extends VoltProcedure { + " SELECT id FROM cte;"); public final SQLStmt sql2 = new SQLStmt("DELETE FROM inodes WHERE id = ?;"); - public long run(long id) throws VoltAbortException { + public VoltTable[] run(long id) throws VoltAbortException { voltQueueSQL(sql1, id); VoltTable[] results = voltExecuteSQL(); @@ -25,6 +25,6 @@ public long run(long id) throws VoltAbortException { voltQueueSQL(sql2, results[0].fetchRow(i).getLong(0)); } voltExecuteSQL(); - return 1; + return results; } } From 64084bd0c74d0d03a0e50b95e3a77cfe3438a79e Mon Sep 17 00:00:00 2001 From: Gang Liao Date: Sat, 21 Sep 2019 11:12:57 -0400 Subject: [PATCH 2/2] ignore bench result for now --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8ac97b113a4..5ca05b132de 100644 --- a/.gitignore +++ b/.gitignore @@ -79,4 +79,5 @@ benchmark/hopfs/hops benchmark/hopfs/META-INF storedprocs.jar -bench/hdfs \ No newline at end of file +bench/hdfs +bench/voltfs