Merge pull request #178 from DSL-UMD/distributed

Optimize delete recursively

.gitignore

@@ -79,4 +79,5 @@ benchmark/hopfs/hops
 benchmark/hopfs/META-INF
 
 storedprocs.jar
-bench/hdfs
+bench/hdfs
+bench/voltfs

hadoop-hdfs-project/hadoop-hdfs-db/src/main/java/org/apache/hadoop/hdfs/db/DatabaseINode.java

@@ -719,14 +719,20 @@ public static List<Long> getChildIdsByPath(final long id, final String[] compone
     return res;
   }
 
-  public static void removeChild(final long id) {
+  public static List<Long> removeChild(final long id) {
+    List<Long> childIds = new ArrayList<>();
     try {
       DatabaseConnection obj = Database.getInstance().getConnection();
       String env = System.getenv("DATABASE");
       if (env.equals("VOLT")) {
         // call a stored procedure
         try {
-          obj.getVoltClient().callProcedure(new NullCallback(), "RemoveChild", id);
+          VoltTable[] results = obj.getVoltClient().callProcedure(new NullCallback(), "RemoveChild", id).getResults();
+          VoltTable result = results[0];
+          result.resetRowPosition();
+          while (result.advanceRow()) {
+            childIds.add(result.getLong(0));
+          }
         } catch (Exception e) {
           e.printStackTrace();
         }
@@ -755,6 +761,7 @@ public static void removeChild(final long id) {
     if (LOG.isInfoEnabled()) {
       LOG.info("removeChild: " + id);
     }
+    return childIds; 
   }
 
   public static List<String> getPathComponents(final long childId) {

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java

@@ -265,9 +265,12 @@ private static boolean unprotectedDelete(FSDirectory fsd, INodesInPath iip,
       targetNode.cleanSubtree(reclaimContext, CURRENT_STATE_ID, latestSnapshot);
     }
 
-    // ADD(gangliao): Remove inode from Postgres
-    DatabaseINode.removeChild(targetNode.getId());
-    INodeKeyedObjects.getCache().invalidateAllWithIndex(Long.class, targetNode.getId()); 
+    // ADD(gangliao): Remove inode from Database
+    List<Long> ids = DatabaseINode.removeChild(targetNode.getId());
+    INodeKeyedObjects.getCache().invalidateAllWithIndex(Long.class, targetNode.getId());
+    for (Long id : ids) {
+      INodeKeyedObjects.getCache().invalidateAllWithIndex(Long.class, id); 
+    }
 
     if (NameNode.stateChangeLog.isDebugEnabled()) {
       NameNode.stateChangeLog.debug("DIR* FSDirectory.unprotectedDelete: "

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java

@@ -832,8 +832,10 @@ private void setBlocks(INodeFile that) {
 
   /** Clear all blocks of the file. */
   public void clearBlocks() {
-    blockNum.getAndSet(0);
-    DatabaseINode2Block.deleteViaBcId(this.getId());
+    if (numBlocks() != 0) { 
+      blockNum.getAndSet(0);
+      DatabaseINode2Block.deleteViaBcId(this.getId());
+    }
   }
 
   private void updateRemovedUnderConstructionFiles(

voltdb/RemoveChild.java

@@ -14,7 +14,7 @@ public class RemoveChild extends VoltProcedure {
               + " SELECT id FROM cte;");
   public final SQLStmt sql2 = new SQLStmt("DELETE FROM inodes WHERE id = ?;");
 
-  public long run(long id) throws VoltAbortException {
+  public VoltTable[] run(long id) throws VoltAbortException {
     voltQueueSQL(sql1, id);
     VoltTable[] results = voltExecuteSQL();
 
@@ -25,6 +25,6 @@ public long run(long id) throws VoltAbortException {
       voltQueueSQL(sql2, results[0].fetchRow(i).getLong(0));
     }
     voltExecuteSQL();
-    return 1;
+    return results;
   }
 }