IMPALA-11183: Fix run-all-tests.sh can't repeat tests more than once

We launch a background process checking whether tests are timeout in
run-all-tests.sh. When NUM_TEST_ITERATIONS is set to larger than 1,
run-all-tests.sh will repeat the tests. However, the timeout process is
killed at the end of each iteration, which fails the script when we want
to repeat tests. This patch moves the killing logic outside the loop.

This patch also adds a new variable, CLUSTER_TEST_FILES, to specify
a particular custom-cluster test to run.

To speedup the test iteration, this patch avoids always restarting the
Impala cluster. E.g. when we just need to run a particular EE test, we
only need to start the Impala cluster once.

Tested with NUM_TEST_ITERATIONS=10 and verified with following
scenarios.

1) custom-cluster test only
export BE_TEST, FE_TEST, JDBC_TEST, EE_TEST to false
export CLUSTER_TEST=true and CLUSTER_TEST_FILES to following values:
custom_cluster/test_local_catalog.py
custom_cluster/test_local_catalog.py::TestLocalCatalogRetries
custom_cluster/test_local_catalog.py::TestLocalCatalogRetries::test_replan_limit
"custom_cluster/test_local_catalog.py -k replan_limit"

2) e2e test only
export BE_TEST, FE_TEST, JDBC_TEST, CLUSTER_TEST to false
export EE_TEST=true and
EE_TEST_FILES=query_test/test_scanners.py::TestParquet::test_multiple_blocks_mt_dop

Change-Id: I2bdd8a9c68ffb0dd1c3ea72c3649b00abcc05a49
Reviewed-on: http://gerrit.cloudera.org:8080/18328
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>

bin/run-all-tests.sh

@@ -51,6 +51,7 @@ fi
 : ${JDBC_TEST:=true}
 # Run Cluster Tests
 : ${CLUSTER_TEST:=true}
+: ${CLUSTER_TEST_FILES:=}
 # Extra arguments passed to start-impala-cluster for tests. These do not apply to custom
 # cluster tests.
 : ${TEST_START_CLUSTER_ARGS:=}
@@ -192,12 +193,22 @@ run_ee_tests() {
 
 for i in $(seq 1 $NUM_TEST_ITERATIONS)
 do
+  echo "Test iteration $i"
   TEST_RET_CODE=0
 
   # Store a list of the files at the beginning of each iteration.
   hdfs dfs -ls -R /test-warehouse > ${IMPALA_LOGS_DIR}/file-list-begin-${i}.log 2>&1
 
-  start_impala_cluster
+  # Try not restarting the cluster to save time. BE, FE, JDBC and EE tests require
+  # running on a cluster with default flags. We just need to restart the cluster when
+  # there are custom-cluster tests which will leave the cluster running with specifit
+  # flags.
+  if [[ "$BE_TEST" == true || "$FE_TEST" == true || "$EE_TEST" == true
+      || "$JDBC_TEST" == true ]]; then
+    if [[ $i == 1 || "$CLUSTER_TEST" == true ]]; then
+      start_impala_cluster
+    fi
+  fi
 
   if [[ "$BE_TEST" == true ]]; then
     if [[ "$TARGET_FILESYSTEM" == "local" ]]; then
@@ -325,12 +336,16 @@ do
   # the list of files is from dataload.
   hdfs dfs -ls -R /test-warehouse > ${IMPALA_LOGS_DIR}/file-list-end-${i}.log 2>&1
 
-  # Finally, kill the spawned timeout process and its child sleep process.
-  # There may not be a sleep process, so ignore failure.
-  pkill -P $TIMEOUT_PID || true
-  kill $TIMEOUT_PID
-
   if [[ $TEST_RET_CODE == 1 ]]; then
-    exit $TEST_RET_CODE
+    break
   fi
 done
+
+# Finally, kill the spawned timeout process and its child sleep process.
+# There may not be a sleep process, so ignore failure.
+pkill -P $TIMEOUT_PID || true
+kill $TIMEOUT_PID
+
+if [[ $TEST_RET_CODE == 1 ]]; then
+  exit $TEST_RET_CODE
+fi

tests/run-custom-cluster-tests.sh

@@ -36,8 +36,13 @@ mkdir -p "${RESULTS_DIR}"
 cd "${IMPALA_HOME}/tests"
 . "${IMPALA_HOME}/bin/set-classpath.sh" &> /dev/null
 
+: ${CLUSTER_TEST_FILES:=}
+if [[ "$CLUSTER_TEST_FILES" != "" ]]; then
+  ARGS=($CLUSTER_TEST_FILES)
+else
+  ARGS=(custom_cluster/ authorization/)
+fi
 AUX_CUSTOM_DIR="${IMPALA_AUX_TEST_HOME}/tests/aux_custom_cluster_tests/"
-ARGS=(custom_cluster/ authorization/)
 if [[ -d "${AUX_CUSTOM_DIR}" ]]
 then
   ARGS+=("${AUX_CUSTOM_DIR}")

tests/run-tests.py

@@ -47,7 +47,7 @@
 TEST_HELPER_DIRS = ['aux_parquet_data_load', 'comparison', 'benchmark',
                      'custom_cluster', 'util', 'experiments', 'verifiers', 'common',
                      'performance', 'beeswax', 'aux_custom_cluster_tests',
-                     'authorization']
+                     'authorization', 'test-hive-udfs']
 
 TEST_DIR = os.path.join(os.environ['IMPALA_HOME'], 'tests')
 RESULT_DIR = os.path.join(os.environ['IMPALA_EE_TEST_LOGS_DIR'], 'results')