Bump netty-all from 4.0.52.Final to 4.1.42.Final in /hadoop-project

.gitignore

@@ -61,3 +61,23 @@ output.xml
 report.html
 
 hadoop-hdds/docs/public
+
+hadoop-calvin-util/*
+name/*
+data/*
+tmp/*
+
+bench/*.class
+bench/*.jar
+
+*.class
+.factorypath
+voltdb-ent-9.0.tar.gz
+voltdb-ent
+
+benchmark/hopfs/hops
+benchmark/hopfs/META-INF
+
+storedprocs.jar
+bench/hdfs
+bench/voltfs

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "hadoop-hdfs-project/commons-pool2"]
+	path = hadoop-hdfs-project/commons-pool2
+	url = https://github.com/DSL-UMD/commons-pool2

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+    "java.autobuild.enabled": false,
+    "java.configuration.updateBuildConfiguration": "interactive"
+}

Dockerfile

@@ -0,0 +1,50 @@
+#
+# example Dockerfile for https://docs.docker.com/engine/examples/postgresql_service/
+#
+
+FROM ubuntu:xenial
+
+# Add the PostgreSQL PGP key to verify their Debian packages.
+# It should be the same key as https://www.postgresql.org/media/keys/ACCC4CF8.asc
+RUN apt-get update
+RUN apt-get install -y gnupg vim wget
+RUN wget http://apt.postgresql.org/pub/repos/apt/ACCC4CF8.asc
+RUN apt-key add ACCC4CF8.asc
+# Add PostgreSQL's repository. It contains the most recent stable release
+#     of PostgreSQL, ``9.5``.
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+
+# Install ``python-software-properties``, ``software-properties-common`` and PostgreSQL 9.5
+#  There are some warnings (in red) that show up during the build. You can hide
+#  them by prefixing each apt-get statement with DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y python-software-properties software-properties-common postgresql-9.5 postgresql-client-9.5 postgresql-contrib-9.5
+
+# Note: The official Debian and Ubuntu images automatically ``apt-get clean``
+# after each ``apt-get``
+
+# Run the rest of the commands as the ``postgres`` user created by the ``postgres-9.5`` package when it was ``apt-get installed``
+USER postgres
+
+# Create a PostgreSQL role named ``docker`` with ``docker`` as the password and
+# then create a database `docker` owned by the ``docker`` role.
+# Note: here we use ``&&\`` to run commands one after the other - the ``\``
+#       allows the RUN command to span multiple lines.
+RUN    /etc/init.d/postgresql start &&\
+    psql --command "CREATE USER docker WITH SUPERUSER PASSWORD 'docker';" &&\
+    createdb -O docker docker
+
+# Adjust PostgreSQL configuration so that remote connections to the
+# database are possible.
+RUN echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/9.5/main/pg_hba.conf
+
+# And add ``listen_addresses`` to ``/etc/postgresql/9.5/main/postgresql.conf``
+RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf
+
+# Expose the PostgreSQL port
+EXPOSE 5432
+
+# Add VOLUMEs to allow backup of config, logs and databases
+VOLUME  ["/etc/postgresql", "/var/log/postgresql", "/var/lib/postgresql"]
+
+# Set the default command to run when starting the container
+CMD ["/usr/lib/postgresql/9.5/bin/postgres", "-D", "/var/lib/postgresql/9.5/main", "-c", "config_file=/etc/postgresql/9.5/main/postgresql.conf"]

HdfsMetaInfoSchema.java

@@ -0,0 +1,157 @@
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Properties;
+
+public class HdfsMetaInfoSchema {
+  private static HdfsMetaInfoSchema instance;
+  private Connection connection;
+  private String postgres = "jdbc:postgresql://localhost:5432/docker";
+  private String cockroach = "jdbc:postgresql://localhost:26257/docker";
+  private String volt = "jdbc:voltdb://localhost:21212";
+  private String username = "docker";
+  private String password = "docker";
+
+  private HdfsMetaInfoSchema() throws SQLException {
+    String env = System.getenv("DATABASE");
+    try {
+      String url = null;
+      Properties props = new Properties();
+
+      if (env.equals("VOLT")) {
+        Class.forName("org.voltdb.jdbc.Driver");
+        url = System.getenv("VOLTDB_SERVER");
+        if (url == null) {
+          url = volt;
+        } else {
+          url = "jdbc:voltdb://" + url + ":21212"; 
+        }
+        this.connection = DriverManager.getConnection(url);
+      } else if (env.equals("COCKROACH")) {
+        Class.forName("org.postgresql.Driver");
+        props.setProperty("user", username);
+        props.setProperty("sslmode", "disable");
+        this.connection = DriverManager.getConnection(cockroach, props);
+        url = cockroach;
+      } else {
+        Class.forName("org.postgresql.Driver");
+        props.setProperty("user", username);
+        props.setProperty("password", password);
+        this.connection = DriverManager.getConnection(postgres, props);
+        url = postgres;
+      }
+      System.out.println("HdfsSchemaInDB: [" + env + "] " + url);
+    } catch (Exception ex) {
+      System.err.println("Database Connection Creation Failed : " + ex.getMessage());
+      ex.printStackTrace();
+      System.exit(0);
+    }
+
+    try {
+      // create inode table in Postgres
+      String sql1 = "";
+      String[] tableNames = new String[] {
+        "hdfs", "namespace", "inodes", "namenodes", "mount", "stringtable",
+        "inodexattrs", "inodeuc", "inode2block", "datablocks", "blockstripes",
+        "block2storage", "storage", "delegationkeys", "persisttokens"};
+      for (String tableName : tableNames) {
+        if (env.equals("VOLT")) {
+          sql1 += String.format("DROP TABLE %s IF EXISTS;", tableName);
+        } else {
+          sql1 += String.format("DROP TABLE IF EXISTS %s;", tableName);
+        }
+      }
+
+      String sql2 =
+          "CREATE TABLE hdfs("
+              + "   id int primary key, numEntry int, maskBits int,"
+              + "   currentId int, tokenSequenceNumber int, numKeys int, numTokens int"
+              + ");"
+              + "CREATE TABLE namespace("
+              + "   namespaceId int, genstampV1 bigint, genstampV2 bigint,"
+              + "   genstampV1Limit bigint, lastAllocatedBlockId bigint,"
+              + "   transactionId bigint, rollingUpgradeStartTime bigint,"
+              + "   lastAllocatedStripedBlockId bigint"
+              + ");"
+              + "CREATE TABLE mount("
+              + "   namenode varchar, path varchar, readOnly int,"
+              + "   PRIMARY KEY(namenode, path)"
+              + ");"
+              + "CREATE VIEW namenodes("
+              + "   namenode"
+              + ") AS SELECT DISTINCT namenode FROM mount;"
+              + "CREATE TABLE stringtable("
+              + "   id int primary key, str varchar"
+              + ");"
+              + "CREATE TABLE delegationkeys("
+              + "   id int primary key, expiryDate bigint, key varchar"
+              + ");"
+              + "CREATE TABLE persisttokens("
+              + "   version int, owner varchar, renewer varchar, realuser varchar, issueDate bigint,"
+              + "   maxDate bigint, sequenceNumber int primary key, masterKeyId int, expiryDate bigint"
+              + ");"
+              + "CREATE TABLE inodes("
+              + "   id bigint, parent bigint NOT NULL, name varchar,"
+              + "   accessTime bigint, modificationTime bigint,"
+              + "   header bigint, permission bigint,"
+              + "   PRIMARY KEY (id, parent)"
+              + ");"
+              + "PARTITION TABLE inodes ON COLUMN parent;"
+              + "CREATE TABLE inodexattrs("
+              + "   id bigint, namespace smallint, name varchar, value varchar"
+              + ");"
+              + "CREATE TABLE inodeuc("
+              + "   id bigint primary key, clientName varchar, clientMachine varchar"
+              + ");"
+              + "CREATE TABLE inode2block("
+              + "   blockId bigint primary key, id bigint, idx int"
+              + ");"
+              + "CREATE TABLE datablocks("
+              + "   blockId bigint primary key, numBytes bigint, generationStamp bigint,"
+              + "   replication int, ecPolicyId int"
+              + ");"
+              + "CREATE TABLE blockstripes("
+              + "   blockId bigint, idx int, blockIndex int,"
+              + "   PRIMARY KEY(blockId, idx)"
+              + ");"
+              + "CREATE TABLE block2storage("
+              + "   blockId bigint, idx int, storageId varchar,"
+              + "   PRIMARY KEY(blockId, idx)"
+              + ");"
+              + "CREATE TABLE storage("
+              + "   storageId varchar primary key, storageType int, state int,"
+              + "   capacity bigint, dfsUsed bigint, nonDfsUsed bigint, remaining bigint,"
+              + "   blockPoolUsed bigint, blockReportCount int, heartbeatedSinceFailover smallint,"
+              + "   blockContentsStale smallint, datanodeUuid varchar"
+              + ");";
+      Statement st = connection.createStatement();
+      st.execute(sql1);
+      st.execute(sql2);
+      st.close();
+    } catch (SQLException ex) {
+      System.err.println(ex.getMessage());
+    }
+  }
+
+  public Connection getConnection() {
+    return connection;
+  }
+
+  public static HdfsMetaInfoSchema getInstance() throws SQLException {
+    if (instance == null) {
+      instance = new HdfsMetaInfoSchema();
+    } else if (instance.getConnection().isClosed()) {
+      instance = new HdfsMetaInfoSchema();
+    }
+    return instance;
+  }
+
+  public static void main(String[] args) {
+    try {
+      HdfsMetaInfoSchema.getInstance();
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+}

Readme.md

@@ -0,0 +1 @@
+

bench/InstrumentationAgent.java

@@ -0,0 +1,19 @@
+import java.util.ArrayList;
+import java.util.List;
+import java.lang.instrument.Instrumentation;
+
+class InstrumentationAgent {
+    private static volatile Instrumentation globalInstrumentation;
+    public static void premain(final String agentArgs, final Instrumentation inst) {
+        globalInstrumentation = inst;
+    }
+    public static void agentmain(final String agentArgs, final Instrumentation inst) {
+        globalInstrumentation = inst;
+    }
+    public static long getObjectSize(final Object object) {
+        if (globalInstrumentation == null) {
+            throw new IllegalStateException("Agent not initialized.");
+        }
+        return globalInstrumentation.getObjectSize(object);
+    }
+}

bench/InstrumentationExample.java

@@ -0,0 +1,68 @@
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+public class InstrumentationExample {
+
+    public static void printObjectSize(Object object) {
+        System.out.println("Object type: " + object.getClass() +
+          ", size: " + InstrumentationAgent.getObjectSize(object) + " bytes");
+    }
+
+    public static void main(String[] arguments) {
+
+		class Person {
+			String name;
+			int age;
+			long phone;
+			boolean female;
+			byte[] password = {1, 2, 3, 4};
+		}
+
+		Person p = new Person();
+
+		int[] a0 = {};
+		int[] a1 = {1};
+		int[] a2 = {1, 2};
+		int[] a3 = new int[100];
+
+		String[] b0 = {};
+		String[] b1 = {"1"};
+		String[] b2 = {"1", "2"};
+		String[] b3 = new String[100];
+
+		String s0 = "";
+		String s1 = "hello";
+
+		List<Person> al0 = new ArrayList<>(0);
+		List<Person> al1 = new ArrayList<>(1);
+		al1.add(new Person());
+		List<Person> al2 = new ArrayList<>(2);
+		al2.add(new Person());
+		al2.add(new Person());
+		List<Person> al3 = new ArrayList<>(100);
+		for (int i = 0; i < 100; i++) {
+		    al3.add(new Person());
+		}
+
+		printObjectSize(p);
+
+		printObjectSize(a0);
+		printObjectSize(a1);
+		printObjectSize(a2);
+		printObjectSize(a3);
+
+		printObjectSize(b0);
+		printObjectSize(b1);
+		printObjectSize(b2);
+		printObjectSize(b3);
+
+		printObjectSize(al0);
+		printObjectSize(al1);
+		printObjectSize(al2);
+		printObjectSize(al3);
+
+		printObjectSize(s0);
+		printObjectSize(s1);
+    }
+}

bench/MANIFEST.MF

@@ -0,0 +1,2 @@
+Premain-class: InstrumentationAgent
+Agent-class: InstrumentationAgent

bench/aws_hdfs_create.py

@@ -0,0 +1,15 @@
+import sys
+from fabric import Connection
+from tee import StdoutTee, StderrTee
+
+cnn = "3.89.124.4"
+
+node = {"1":"3.89.124.4"}
+
+connect_kwargs = {"key_filename":['/Users/liaogang/.ssh/voltfs.pem']}
+
+
+with Connection(host=cnn, user="ec2-user", connect_kwargs=connect_kwargs) as c:
+	c.run("cd /home/ec2-user/hdfs/hadoop-3.3.0-SNAPSHOT;"
+		"source etc/hadoop/hadoop-env.sh;"
+		"bash bench.sh")

bench/aws_hdfs_open.py

@@ -0,0 +1,15 @@
+import sys
+from fabric import Connection
+from tee import StdoutTee, StderrTee
+
+cnn = "34.203.247.95"
+
+node = {"1":"34.203.247.95"}
+
+connect_kwargs = {"key_filename":['/Users/liaogang/.ssh/voltfs.pem']}
+
+
+with Connection(host=cnn, user="ec2-user", connect_kwargs=connect_kwargs) as c:
+	c.run("cd /home/ec2-user/hdfs/hadoop-3.3.0-SNAPSHOT;"
+		"source etc/hadoop/hadoop-env.sh;"
+		"bash bench.sh")