Adding support for processing module of Druid

core/add_project.sh

@@ -48,6 +48,14 @@ function setup_alluxio() {
     mvn clean install -DskipTests -Dcheckstyle.skip -Dlicense.skip -Dfindbugs.skip -Dmaven.javadoc.skip=true
 }
 
+function setup_druid() {
+    [ ! -d "app/ctest-druid" ] && git clone https://github.com/b06902047/druid.git app/ctest-druid
+    cd app/ctest-druid
+    git fetch && git checkout ctest-processing-inject-release
+    cd processing
+    mvn install -DskipTests -Dcheckstyle.skip
+}
+
 function usage() {
     echo "Usage: add_project.sh <main project>"
     exit 1
@@ -64,7 +72,8 @@ function main() {
             hbase) setup_hbase ;;
             zookeeper) setup_zookeeper ;;
             alluxio) setup_alluxio ;;
-            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper and alluxio." ;;
+            druid) setup_druid ;;
+            *) echo "Unexpected project: $project - only support hadoop, hbase, zookeeper, alluxio and druid." ;;
         esac
     fi
 }

core/ctest_const.py

@@ -12,18 +12,21 @@
 HBASE = "hbase-server"
 ZOOKEEPER = "zookeeper-server"
 ALLUXIO = "alluxio-core"
+DPROCESSING = "druid-processing"
 
 CTEST_HADOOP_DIR = os.path.join(APP_DIR, "ctest-hadoop")
 CTEST_HBASE_DIR = os.path.join(APP_DIR, "ctest-hbase")
 CTEST_ZK_DIR = os.path.join(APP_DIR, "ctest-zookeeper")
 CTEST_ALLUXIO_DIR = os.path.join(APP_DIR, "ctest-alluxio")
+CTEST_DRUID_DIR = os.path.join(APP_DIR, "ctest-druid")
 
 PROJECT_DIR = {
     HCOMMON: CTEST_HADOOP_DIR,
     HDFS: CTEST_HADOOP_DIR,
     HBASE: CTEST_HBASE_DIR,
     ZOOKEEPER: CTEST_ZK_DIR,
     ALLUXIO: CTEST_ALLUXIO_DIR,
+    DPROCESSING: CTEST_DRUID_DIR
 }
 
 
@@ -34,6 +37,7 @@
     HBASE: "hbase-server",
     ZOOKEEPER: "zookeeper-server",
     ALLUXIO: "core",
+    DPROCESSING: "processing"
 }
 
 
@@ -58,6 +62,7 @@
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/worker", SUREFIRE_SUBDIR),
         os.path.join(CTEST_ALLUXIO_DIR, MODULE_SUBDIR[ALLUXIO], "server/master", SUREFIRE_SUBDIR),
     ],
+    DPROCESSING: [os.path.join(CTEST_DRUID_DIR, MODULE_SUBDIR[DPROCESSING], SUREFIRE_SUBDIR)],
 }
 
 # default or deprecate conf path
@@ -74,7 +79,8 @@
     HDFS: os.path.join(DEFAULT_CONF_DIR, HDFS + "-default.tsv"),
     HBASE: os.path.join(DEFAULT_CONF_DIR, HBASE + "-default.tsv"),
     ALLUXIO: os.path.join(DEFAULT_CONF_DIR, ALLUXIO + "-default.tsv"),
-    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv")
+    ZOOKEEPER: os.path.join(DEFAULT_CONF_DIR, ZOOKEEPER + "-default.tsv"),
+    DPROCESSING: os.path.join(DEFAULT_CONF_DIR, DPROCESSING + "-default.tsv")
 }
 
 
@@ -96,6 +102,9 @@
     ],
     ALLUXIO: [
         os.path.join(CTEST_ALLUXIO_DIR, "core/alluxio-ctest.properties")
+    ],
+    DPROCESSING: [
+        os.path.join(CTEST_DRUID_DIR, "processing/core-ctest.xml")
     ]
 }
 

core/generate_ctest/ctest_mapping/ctests-hadoop-common.json

@@ -0,0 +1 @@
+{}

core/generate_ctest/inject.py

@@ -14,7 +14,7 @@ def inject_config(param_value_pairs):
     for p, v in param_value_pairs.items():
         print(">>>>[ctest_core] injecting {} with value {}".format(p, v))
 
-    if project in [ZOOKEEPER, ALLUXIO]:
+    if project in [ZOOKEEPER, ALLUXIO, DPROCESSING]:
         for inject_path in INJECTION_PATH[project]:
             print(">>>>[ctest_core] injecting into file: {}".format(inject_path))
             file = open(inject_path, "w")
@@ -41,7 +41,7 @@ def inject_config(param_value_pairs):
 
 def clean_conf_file(project):
     print(">>>> cleaning injected configuration from file")
-    if project in [ZOOKEEPER, ALLUXIO]:
+    if project in [ZOOKEEPER, ALLUXIO, DPROCESSING]:
         for inject_path in INJECTION_PATH[project]:
             file = open(inject_path, "w")
             file.write("\n")

core/generate_ctest/main.py

@@ -33,7 +33,7 @@ def test_value_pair(test_input):
         mt_file = open(os.path.join(GENCTEST_TR_DIR, project, MT_FILE.format(id=param)), "w")
 
         associated_tests = mapping[param] if param in mapping else []
-        if len(mapping[param]) != 0:
+        if len(associated_tests) != 0:
             for value in values:
                 tr = run_test_seperate(param, value, associated_tests)
 

core/generate_ctest/program_input.py

@@ -4,17 +4,17 @@
     # run mode
     "run_mode": "generate_ctest", # string
     # name of the project, i.e. hadoop-common, hadoop-hdfs, see constant.py
-    "project": "hadoop-common", # string
+    "project": "druid-processing", # string
     # path to param -> tests json mapping
-    "mapping_path": "../../data/ctest_mapping/opensource-hadoop-common.json", # string
+    "mapping_path": "../../data/ctest_mapping/opensource-druid-processing.json", # string
     # good values of params tests will be run against
-    "param_value_tsv": "sample-hadoop-common.tsv", # string
+    "param_value_tsv": "sample-druid-processing.tsv", # string
     # display the terminal output live, without saving any results
     "display_mode": False, # bool
     # whether to use mvn test or mvn surefire:test
-    "use_surefire": True, # bool
+    "use_surefire": False, # bool
     # additional maven options to pass to `mvn surefire:test -Dtest=...`
-    "maven_args": [], # list of strings, each element is an option
+    "maven_args": ["-Dcheckstyle.skip"], # list of strings, each element is an option
     # timeout on the mvn test command
     "cmd_timeout": None, # int
 }