bench: intra- and inter-data center rename ops (#263)

* bench: intra- and inter-data center rename ops * optimize remote rename * sync code * fix typo * fix compile error
umd-dslam · Jun 9, 2020 · 4d559f2 · 4d559f2
1 parent b4fb8be
commit 4d559f2
Show file tree

Hide file tree

Showing 5 changed files with 121 additions and 3 deletions.
diff --git a/...op-hdfs-project/hadoop-hdfs-db/src/main/java/org/apache/hadoop/hdfs/db/DatabaseINode.java b/...op-hdfs-project/hadoop-hdfs-db/src/main/java/org/apache/hadoop/hdfs/db/DatabaseINode.java
@@ -555,6 +555,38 @@ public static void setParent(final long id, final long parent) {
     }
   }
 
+  public static void setParents(final Long[] ids, final long parent) {
+    try {
+      DatabaseConnection obj = Database.getInstance().getConnection();
+      String env = System.getenv("DATABASE");
+      if (env.equals("VOLT")) {
+        try {
+          obj.getVoltClient().callProcedure(new NullCallback(), "SetParents", ids, parent);
+        } catch (Exception e) {
+          e.printStackTrace();
+        }
+      } else {
+        // Connection conn = obj.getConnection();
+        // String sql = "UPDATE inodes SET parent = ? WHERE id = ?;";
+        // PreparedStatement pst = conn.prepareStatement(sql);
+        // if (int i = 0; i < ids.length; i++) {
+        //   pst.setLong(1, parent);
+        //   pst.setLong(2, ids[i]);
+        //   pst.addBatch();
+        // }
+        // pst.executeBatch();
+        // pst.close();
+        throw new SQLException("[UNSUPPORT] Invalid operation ...");
+      }
+      Database.getInstance().retConnection(obj);
+    } catch (SQLException ex) {
+      System.err.println(ex.getMessage());
+    }
+    if (LOG.isInfoEnabled()) {
+      LOG.info("parent [UPDATE]: (childs," + parent + ")");
+    }
+  }
+
   public static void setName(final long id, final String name) {
     try {
       DatabaseConnection obj = Database.getInstance().getConnection();

diff --git a/...oject/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/...oject/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -658,7 +658,7 @@ boolean removeSrc4OldRename() {
         return false;
       } else {
         // update the quota count if necessary
-        fsd.updateCountForDelete(srcChild, srcIIP);
+        // fsd.updateCountForDelete(srcChild, srcIIP);
         srcIIP = INodesInPath.replace(srcIIP, srcIIP.length() - 1, null);
         return true;
       }

diff --git a/...ject/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/...ject/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
@@ -688,23 +688,26 @@ public void remoteRename(INode node, String address) {
       INodeDirectory inode = node.asDirectory().copyINodeDirectory();
       inode.setId(node.getId() + NameNode.getId());
 
+      // TODO: using stored procedure to optimize and update the immediated childs
       // update immediate childs's parent id
       HashSet<Long> childs = ((INodeDirectory)node).getCurrentChildrenList2();
       for (long id : childs) {
         INode child = FSDirectory.getInstance().getInode(id);
         if (child != null) {
           child.setParent(inode.getId());
+          // write ahead log
           if (child.isDirectory()) {
-            child.asDirectory().updateINodeDirectory();
             INodeKeyedObjects.getCache().invalidateAllWithIndex(Long.class, (Long) child.getId());
             FSDirectory.getInstance().getEditLog().logMkDir(null, (INodeDirectory)child);
           } else {
-            child.asFile().updateINodeFile();
             INodeKeyedObjects.getCache().invalidateAllWithIndex(Long.class, (Long) child.getId());
             FSDirectory.getInstance().getEditLog().logOpenFile(null, (INodeFile)child, true, true);
           }
         }
       }
+      // using a stored procedure to update childs' parent
+      Long[] kids = childs.toArray(new Long[childs.size()]);
+      DatabaseINode.setParents(kids, getId());
 
       // invalidate old inode
       INodeKeyedObjects.getCache().invalidateAllWithIndex(Long.class, (Long) node.getId());

diff --git a/...doop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/...doop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -911,6 +911,69 @@ long executeOp(int daemonId, int inputIdx, String ignore)
     }
   }
 
+  /**
+   * Rename entire directory: /nnThroughputBenchmark/create.
+   */
+  class RenameDirStats extends OperationStatsBase {
+    // Operation types
+    static final String OP_RENAME_NAME = "renameDir";
+    static final String OP_RENAME_USAGE = "-op renameDir";
+
+    RenameDirStats(List<String> args) {
+      super();
+      parseArguments(args);
+      numOpsRequired = 1;
+      numThreads = 1;
+      keepResults = true;
+    }
+
+    @Override
+    String getOpName() {
+      return OP_RENAME_NAME;
+    }
+
+    @Override
+    void parseArguments(List<String> args) {
+      boolean ignoreUnrelatedOptions = verifyOpArgument(args);
+      if(args.size() > 2 && !ignoreUnrelatedOptions)
+        printUsage();
+    }
+
+    @Override
+    void generateInputs(int[] opsPerThread) throws IOException {
+      // do nothing
+    }
+
+    /**
+     * Does not require the argument
+     */
+    @Override
+    String getExecutionArgument(int daemonId) {
+      return null;
+    }
+
+    /**
+     * Rename entire benchmark directory.
+     */
+    @Override
+    long executeOp(int daemonId, int inputIdx, String ignore) 
+    throws IOException {
+      clientProto.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE,
+          false);
+      long start = Time.now();
+      clientProto.rename(BASE_DIR_NAME + "/create", BASE_DIR_NAME + "/rename");
+      long end = Time.now();
+      return end-start;
+    }
+
+    @Override
+    void printResults() {
+      LOG.info("--- " + getOpName() + " inputs ---");
+      LOG.info("Rename directory " + BASE_DIR_NAME + "/create");
+      printStats();
+    }
+  }
+
   /**
    * Minimal data-node simulator.
    */
@@ -1440,6 +1503,7 @@ static void printUsage() {
         + " | \n\t" + BlockReportStats.OP_BLOCK_REPORT_USAGE
         + " | \n\t" + ReplicationStats.OP_REPLICATION_USAGE
         + " | \n\t" + CleanAllStats.OP_CLEAN_USAGE
+        + " | \n\t" + RenameDirStats.OP_RENAME_USAGE
         + " | \n\t" + GENERAL_OPTIONS_USAGE
     );
     System.err.println();
@@ -1518,6 +1582,10 @@ public int run(String[] aArgs) throws Exception {
           ops.add(opStat);
         }
       }
+      if(runAll || RenameDirStats.OP_RENAME_NAME.equals(type)) {
+        opStat = new RenameDirStats(args);
+        ops.add(opStat);
+      }
       if(runAll || CleanAllStats.OP_CLEAN_NAME.equals(type)) {
         opStat = new CleanAllStats(args);
         ops.add(opStat);

diff --git a/voltdb/SetParents.java b/voltdb/SetParents.java
@@ -0,0 +1,15 @@
+import org.voltdb.*;
+
+// https://docs.voltdb.com/tutorial/Part5.php
+public class SetParents extends VoltProcedure {
+
+  public final SQLStmt sql = new SQLStmt("UPDATE inodes SET parent = ? WHERE id = ?;");
+
+  public long run(final Long ids[], final long parent) throws VoltAbortException {
+    if (int i = 0; i < ids.length; ++i) {
+      voltQueueSQL(sql, parent, ids[i]);
+    }
+    voltExecuteSQL();
+    return 1;
+  }
+}