STORM-803: Better CI logs

.travis.yml

@@ -17,4 +17,5 @@ before_install:
   - rvm use 2.1.5 --install
   - nvm install 0.12.2
   - nvm use 0.12.2
-script: /bin/bash ./dev-tools/travis/travis-build.sh `pwd`
+install: /bin/bash ./dev-tools/travis/travis-install.sh `pwd`
+script: /bin/bash ./dev-tools/travis/travis-script.sh `pwd`

dev-tools/test-ns.py

@@ -22,26 +22,7 @@
 os.chdir("storm-core")
 
 ns = sys.argv[1]
-pipe = Popen(["mvn", "clojure:repl"], stdin=PIPE)
-
-
-pipe.stdin.write("""
-(do
-  (use 'clojure.test)
-  (require '%s :reload-all)
-  (let [results (run-tests '%s)]
-    (println results)
-    (if (or
-         (> (:fail results) 0)
-         (> (:error results) 0))
-      (System/exit 1)
-      (System/exit 0))))
-""" % (ns, ns))
-
-
-
-pipe.stdin.write("\n")
-pipe.stdin.close()
+pipe = Popen(["mvn", "test", "-DfailIfNoTests=false", "-Dtest=%s"%ns])
 pipe.wait()
 
 os.chdir("..")

dev-tools/travis/print-errors-from-clojure-test-reports.py → dev-tools/travis/print-errors-from-test-reports.py

@@ -18,11 +18,18 @@
 import traceback
 from xml.etree.ElementTree import ElementTree
 
-
 def print_detail_information(testcase, fail_or_error):
     print "-" * 50
     print "classname: %s / testname: %s" % (testcase.get("classname"), testcase.get("name"))
     print fail_or_error.text
+    stdout = testcase.find("system-out")
+    if stdout != None:
+        print "-" * 20, "system-out", "-"*20
+        print stdout.text
+    stderr = testcase.find("system-err")
+    if stderr != None:
+        print "-" * 20, "system-err", "-"*20
+        print stderr.text
     print "-" * 50
 
 
@@ -45,6 +52,7 @@ def main(report_dir_path):
     for test_report in glob.iglob(report_dir_path + '/*.xml'):
         file_path = os.path.abspath(test_report)
         try:
+            print "Checking %s" % test_report
             print_error_reports_from_report_file(file_path)
         except Exception, e:
             print "Error while reading report file, %s" % file_path
@@ -57,4 +65,4 @@ def main(report_dir_path):
         print "Usage: %s [report dir path]" % sys.argv[0]
         sys.exit(1)
 
-    main(sys.argv[1])
+    main(sys.argv[1])

dev-tools/travis/save-logs.py

@@ -0,0 +1,54 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import sys
+import subprocess
+from datetime import datetime, timedelta
+
+def main(file, cmd):
+    print cmd, "writing to", file
+    out = open(file, "w")
+    count = 0
+    process = subprocess.Popen(cmd,
+                           stderr=subprocess.STDOUT,
+                           stdout=subprocess.PIPE)
+
+    start = datetime.now()
+    nextPrint = datetime.now() + timedelta(seconds=1)
+    # wait for the process to terminate
+    pout = process.stdout
+    line = pout.readline()
+    while line:
+        count = count + 1
+        if datetime.now() > nextPrint:
+            diff = datetime.now() - start
+            sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count))
+            sys.stdout.flush()
+            nextPrint = datetime.now() + timedelta(seconds=10)
+        out.write(line)
+        line = pout.readline()
+    out.close()
+    errcode = process.wait()
+    diff = datetime.now() - start
+    sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count))
+    print
+    print cmd, "done", errcode
+    return errcode
+
+if __name__ == "__main__":
+    if sys.argv < 1:
+        print "Usage: %s [file info]" % sys.argv[0]
+        sys.exit(1)
+
+    sys.exit(main(sys.argv[1], sys.argv[2:]))

dev-tools/travis/travis-install.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+PYTHON_VERSION_TO_FILE=`python -V > /tmp/python_version 2>&1`
+PYTHON_VERSION=`cat /tmp/python_version`
+RUBY_VERSION=`ruby -v`
+NODEJS_VERSION=`node -v`
+
+echo "Python version : $PYTHON_VERSION"
+echo "Ruby version : $RUBY_VERSION"
+echo "NodeJs version : $NODEJS_VERSION"
+
+STORM_SRC_ROOT_DIR=$1
+
+TRAVIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+cd ${STORM_SRC_ROOT_DIR}
+
+python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install.txt" mvn clean install -DskipTests -Pnative
+BUILD_RET_VAL=$?
+
+if [[ "$BUILD_RET_VAL" != "0" ]];
+then
+  cat "install.txt"
+fi
+
+exit ${BUILD_RET_VAL}

dev-tools/travis/travis-build.sh → dev-tools/travis/travis-script.sh

@@ -27,24 +27,17 @@ TRAVIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 
 cd ${STORM_SRC_ROOT_DIR}
 
-# Travis CI doesn't allow stdout bigger than 4M, so we have to reduce log while running tests
-export LOG_LEVEL=WARN
 # We should concern that Travis CI could be very slow cause it uses VM
 export STORM_TEST_TIMEOUT_MS=100000
 
 # We now lean on Travis CI's implicit behavior, ```mvn clean install -DskipTests``` before running script
-mvn -fae -pl '!external/storm-kafka' test
-
+python ${TRAVIS_SCRIPT_DIR}/save-logs.py "test.txt" mvn test -fae -Pnative
 BUILD_RET_VAL=$?
 
-if [ ${BUILD_RET_VAL} -ne 0 ]
-then
-    echo "There may be clojure test errors from storm-core, printing error reports..."
-    python ${TRAVIS_SCRIPT_DIR}/print-errors-from-clojure-test-reports.py ${STORM_SRC_ROOT_DIR}/storm-core/target/test-reports
-else
-    echo "Double checking clojure test-report, Errors or Failures:"
-    egrep -il '<fail|<error' */target/test-reports/*.xml | xargs -I '{}' bash -c "echo -n '{}':' '; egrep -ic '<fail|<error' '{}'"
-    egrep -iq '<fail|<error' */target/test-reports/*.xml
-fi
+for dir in `find . -type d -and -wholename \*/target/\*-reports`;
+do
+  echo "Looking for errors in ${dir}"
+  python ${TRAVIS_SCRIPT_DIR}/print-errors-from-test-reports.py "${dir}"
+done
 
 exit ${BUILD_RET_VAL}

pom.xml

@@ -592,6 +592,14 @@
 
         <pluginManagement>
             <plugins>
+                <plugin>
+                    <groupId>org.apache.maven.plugins</groupId>
+                    <artifactId>maven-surefire-plugin</artifactId>
+                    <version>2.18.1</version>
+                    <configuration>
+                      <redirectTestOutputToFile>true</redirectTestOutputToFile>
+                    </configuration>
+                </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-assembly-plugin</artifactId>
@@ -630,7 +638,7 @@
                 <plugin>
                     <groupId>com.theoryinpractise</groupId>
                     <artifactId>clojure-maven-plugin</artifactId>
-                    <version>1.3.18</version>
+                    <version>1.7.1</version>
                     <extensions>true</extensions>
                 </plugin>
                 <plugin>

storm-core/pom.xml

@@ -32,6 +32,7 @@
     <properties>
         <worker-launcher.conf.dir>/etc/storm</worker-launcher.conf.dir>
         <worker-launcher.additional_cflags></worker-launcher.additional_cflags>
+        <argLine></argLine>
     </properties>
 
     <dependencies>
@@ -368,9 +369,11 @@
                         <id>test-clojure</id>
                         <phase>test</phase>
                         <goals>
-                            <goal>test-with-junit</goal>
+                            <goal>test</goal>
                         </goals>
                         <configuration>
+                            <junitOutput>true</junitOutput>
+                            <testScript>test/resources/test_runner.clj</testScript>
                             <!-- argLine is set by JaCoCo for code coverage -->
                             <vmargs>${argLine} ${test.extra.args}</vmargs>
                         </configuration>

storm-core/test/resources/log4j2-test.xml

@@ -18,7 +18,7 @@
 
 <configuration monitorInterval="60">
     <Appenders>
-        <Console name="Console" target="SYSTEM_OUT">
+        <Console name="Console" target="SYSTEM_OUT" follow="true">
             <PatternLayout pattern="%-4r [%t] %-5p %c{1.} - %msg%n"/>
         </Console>
     </Appenders>

storm-core/test/resources/test_runner.clj

@@ -0,0 +1,91 @@
+;; Licensed to the Apache Software Foundation (ASF) under one
+;; or more contributor license agreements.  See the NOTICE file
+;; distributed with this work for additional information
+;; regarding copyright ownership.  The ASF licenses this file
+;; to you under the Apache License, Version 2.0 (the
+;; "License"); you may not use this file except in compliance
+;; with the License.  You may obtain a copy of the License at
+;;
+;; http://www.apache.org/licenses/LICENSE-2.0
+;;
+;; Unless required by applicable law or agreed to in writing, software
+;; distributed under the License is distributed on an "AS IS" BASIS,
+;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+;; See the License for the specific language governing permissions and
+;; limitations under the License.
+
+(ns org.apache.storm.testrunner)
+
+(import `java.util.Properties)
+(import `java.io.ByteArrayOutputStream)
+(import `java.io.FileInputStream)
+(import `java.io.FileOutputStream)
+(import `java.io.FileWriter)
+(import `java.io.File)
+(import `java.io.OutputStream)
+(import `java.io.PrintStream)
+(import `java.io.PrintWriter)
+(use 'clojure.test)
+(use 'clojure.test.junit)
+
+(def props (Properties.))
+(.load props (FileInputStream. (first *command-line-args*)))
+
+(def namespaces  (into [] 
+                       (for [[key val] props
+                             :when (.startsWith key "ns.")]
+                               (symbol val))))
+
+(def output-dir (.get props "outputDir"))
+
+(dorun (for [ns namespaces]
+  (require ns)))
+
+(.mkdirs (File. output-dir))
+
+(let [sys-out System/out
+      sys-err System/err
+      num-bad (atom 0)
+      original-junit-report junit-report
+      cached-out (atom nil)
+      test-error (atom 0)
+      orig-out *out*]
+  (dorun (for [ns namespaces]
+    (with-open [writer (FileWriter. (str output-dir "/" ns ".xml"))
+                out-stream (FileOutputStream. (str output-dir "/" ns "-output.txt"))
+                out-writer (PrintStream. out-stream true)]
+      (.println sys-out (str "Running " ns))
+      (try
+        (System/setOut out-writer)
+        (System/setErr out-writer)
+        (binding [*test-out* writer
+                  *out* (PrintWriter. out-stream true)
+                  junit-report (fn [data]
+                                   (let [type (data :type)]
+                                     (cond
+                                       (= type :begin-test-var) (do
+                                                                   (reset! cached-out (ByteArrayOutputStream.))
+                                                                   (let [writer (PrintStream. @cached-out true)]
+                                                                     (set! *out* (PrintWriter. @cached-out true))
+                                                                     (System/setOut writer)
+                                                                     (System/setErr writer))
+                                                                   (reset! test-error 0))
+                                       (= type :end-test-var) (do
+                                                                  (.write out-writer (.toByteArray @cached-out))
+                                                                  (when (> @test-error 0)
+                                                                    (with-test-out
+                                                                      (start-element 'system-out true)
+                                                                      (element-content (String. (.toByteArray @cached-out)))
+                                                                      (finish-element 'system-out true))))
+                                       (= type :fail) (swap! test-error inc)
+                                       (= type :error) (swap! test-error inc)))
+                                   (original-junit-report data))]
+          (with-junit-output
+            (let [result (run-tests ns)]
+               (.println sys-out (str "Tests run: " (result :test) ", Passed: " (result :pass) ", Failures: " (result :fail) ", Errors: " (result :error)))
+               (reset! num-bad (+ @num-bad (result :error) (result :fail))))))
+        (finally 
+          (System/setOut sys-out)
+          (System/setErr sys-err))))))
+  (shutdown-agents)
+  (System/exit (if (> @num-bad 0) 1 0)))