diff --git a/DEVELOPER.md b/DEVELOPER.md index 02fe135bcf0..7376949ce51 100644 --- a/DEVELOPER.md +++ b/DEVELOPER.md @@ -241,17 +241,16 @@ in order to get started as fast as possible. Users can still install a specific The following commands must be run from the top-level directory. - # Build the code and run the tests (requires nodejs, python and ruby installed) - # `mvn clean package` will fail because storm-core requires storm-maven-plugin. - # This plugin should be installed before compiling storm-core. - $ mvn clean install +The first step is to build/install the plugins and storm-core - # Build the code and run the tests, with specifying default test timeout (in millisecond) - $ export STORM_TEST_TIMEOUT_MS=10000 - $ mvn clean install +`mvn clean install -Pstorm-core` - # Build the code but skip the tests - $ mvn clean install -DskipTests=true +If you wish to skip the unit tests you can do this by adding `-DskipTests` to the command line. Once this +completes successfully you may run + +`mvn clean install -Pstorm-more` + +to build and test all of the external libraries and examples. Again if you want to include `-DskipTests` you can. In case you modified `storm.thrift`, you have to regenerate thrift code as java and python code before compiling whole project. @@ -266,19 +265,7 @@ You can also run tests selectively via the Clojure REPL. The following example [auth_test.clj](storm-core/test/clj/backtype/storm/security/auth/auth_test.clj), which has the namespace `backtype.storm.security.auth.auth-test`. -First, start the REPL from within the relevant sub-project (here: `storm-core`): - - $ cd storm-core/ - $ mvn clojure:repl - -Now we run the tests in `auth_test.clj` in the REPL: - -```clojure -;; You can use both absolute as well as relative paths to the .clj file. -(load-file "test/clj/backtype/storm/security/auth/auth_test.clj") -(ns backtype.storm.security.auth.auth-test) -(run-tests) -``` +You can also run tests selectively with `-Dtest=`. This works for both clojure and junit tests. > Tip: IDEs such as IntelliJ IDEA support a built-in Clojure REPL, which you can also use to run tests selectively. > Sometimes you may find that tests pass/fail depending on which REPL you use, which -- although frustrating -- @@ -295,7 +282,8 @@ You can create a _distribution_ (like what you can download from Apache) as foll do not use the Maven release plugin because creating an official release is the task of our release manager. # First, build the code. - $ mvn clean install # you may skip tests with `-DskipTests=true` to save time + $ mvn clean install -Pstorm-core # you may skip tests with `-DskipTests=true` to save time + $ mvn clean install -Pstorm-more # you may skip tests with `-DskipTests=true` to save time # Create the binary distribution. $ cd storm-dist/binary && mvn package diff --git a/dev-tools/travis/travis-install.sh b/dev-tools/travis/travis-install.sh index 6af2f9062e4..4857a1e2154 100755 --- a/dev-tools/travis/travis-install.sh +++ b/dev-tools/travis/travis-install.sh @@ -26,12 +26,23 @@ TRAVIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) cd ${STORM_SRC_ROOT_DIR} -python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install.txt" mvn clean install --batch-mode -DskipTests -Pnative +python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install-storm-core.txt" mvn clean install -DskipTests -Pnative -Pstorm-core --batch-mode BUILD_RET_VAL=$? if [[ "$BUILD_RET_VAL" != "0" ]]; then - cat "install.txt" + cat "install-storm-core.txt" + exit ${BUILD_RET_VAL} +fi + +exit ${BUILD_RET_VAL} + +python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install-storm-more.txt" mvn clean install -DskipTests -Pstorm-more +BUILD_RET_VAL=$? + +if [[ "$BUILD_RET_VAL" != "0" ]]; +then + cat "install-storm-more.txt" fi exit ${BUILD_RET_VAL} diff --git a/dev-tools/travis/travis-script.sh b/dev-tools/travis/travis-script.sh index 4caff17fc72..c59cb1ad832 100755 --- a/dev-tools/travis/travis-script.sh +++ b/dev-tools/travis/travis-script.sh @@ -31,7 +31,7 @@ cd ${STORM_SRC_ROOT_DIR} export STORM_TEST_TIMEOUT_MS=100000 # We now lean on Travis CI's implicit behavior, ```mvn clean install -DskipTests``` before running script -mvn --batch-mode test -fae -Pnative +mvn --batch-mode test -fae -Pnative -Pstorm-core && mvn test -fae -Pnative -Pstorm-more BUILD_RET_VAL=$? for dir in `find . -type d -and -wholename \*/target/\*-reports`; diff --git a/external/storm-elasticsearch/pom.xml b/external/storm-elasticsearch/pom.xml index 777754959f3..fd263e67472 100644 --- a/external/storm-elasticsearch/pom.xml +++ b/external/storm-elasticsearch/pom.xml @@ -40,6 +40,11 @@ + + commons-io + commons-io + compile + org.apache.storm storm-core diff --git a/external/storm-kafka/pom.xml b/external/storm-kafka/pom.xml index c731a0f7b67..8f2867b0578 100644 --- a/external/storm-kafka/pom.xml +++ b/external/storm-kafka/pom.xml @@ -56,6 +56,11 @@ src/test + + commons-io + commons-io + compile + org.mockito mockito-all diff --git a/external/storm-solr/pom.xml b/external/storm-solr/pom.xml index 2b5c7424c6d..5e7caa44e61 100644 --- a/external/storm-solr/pom.xml +++ b/external/storm-solr/pom.xml @@ -21,7 +21,7 @@ - + org.apache.storm storm-core ${project.version} @@ -95,4 +95,4 @@ - \ No newline at end of file + diff --git a/pom.xml b/pom.xml index 86ce5fd82fc..edb7f2b74c4 100644 --- a/pom.xml +++ b/pom.xml @@ -154,26 +154,6 @@ 3.0.0 - - storm-buildtools/maven-shade-clojure-transformer - storm-buildtools/storm-maven-plugins - storm-multilang/javascript - storm-multilang/python - storm-multilang/ruby - storm-core - external/storm-kafka - external/storm-hdfs - external/storm-hbase - external/storm-hive - external/storm-jdbc - external/storm-redis - external/storm-eventhubs - external/flux - external/storm-elasticsearch - external/storm-solr - examples/storm-starter - - scm:git:https://git-wip-us.apache.org/repos/asf/storm.git scm:git:https://git-wip-us.apache.org/repos/asf/storm.git @@ -197,10 +177,13 @@ 2.4 2.5 1.1 + 1.2.1 + 1.6 0.8.0 2.5.0 1.1 1.3.0 + 0.3.1 7.6.13.v20130916 0.2.3 0.0.1 @@ -214,6 +197,7 @@ 3.9.0.Final 1.6.6 2.1 + 1.7.7 0.2.3 0.2.3 1.9.5 @@ -224,9 +208,47 @@ 0.14.0 2.6.0 2.21 + 2.5 + 2.3 + 2.3.1 + 0.9.2 + 4.11 + + storm-core + + true + + + storm-multilang/javascript + storm-multilang/python + storm-multilang/ruby + storm-buildtools/maven-shade-clojure-transformer + storm-buildtools/storm-maven-plugins + storm-core + + + + storm-more + + true + + + external/storm-kafka + external/storm-hdfs + external/storm-hbase + external/storm-hive + external/storm-jdbc + external/storm-redis + external/storm-eventhubs + external/flux + external/storm-elasticsearch + external/storm-solr + examples/storm-starter + + sign @@ -379,6 +401,41 @@ + + javax.servlet + servlet-api + ${servlet.version} + + + org.slf4j + slf4j-api + ${slf4j.version} + + + joda-time + joda-time + ${joda-time.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + ${jackson.version} + + + commons-fileupload + commons-fileupload + ${commons-fileupload.version} + + + commons-codec + commons-codec + ${commons-codec.version} + org.apache.curator curator-framework @@ -409,6 +466,11 @@ + + org.apache.curator + curator-client + ${curator.version} + com.googlecode.json-simple @@ -435,6 +497,11 @@ ring-jetty-adapter ${ring.version} + + ring + ring-json + ${ring-json.version} + org.eclipse.jetty jetty-servlet @@ -445,6 +512,17 @@ jetty-servlets ${jetty.version} + + org.eclipse.jetty + jetty-server + ${jetty.version} + + + org.eclipse.jetty + jetty-util + ${jetty.version} + + org.clojure tools.logging @@ -570,16 +648,15 @@ org.apache.thrift libthrift - 0.9.2 + ${thrift.version} compile - - - junit - junit - 4.11 - test - + + junit + junit + ${junit.version} + test + @@ -673,7 +750,7 @@ org.apache.maven.plugins maven-shade-plugin - 2.2 + 2.4.1 org.apache.maven.plugins diff --git a/storm-core/pom.xml b/storm-core/pom.xml index 3eb41127ecd..ca40cb873b4 100644 --- a/storm-core/pom.xml +++ b/storm-core/pom.xml @@ -39,6 +39,12 @@ com.esotericsoftware.kryo kryo + + + org.objenesis + objenesis + + @@ -74,7 +80,6 @@ ring ring-json - 0.3.1 org.eclipse.jetty @@ -141,7 +146,6 @@ org.apache.thrift libthrift - 0.9.2 compile @@ -154,8 +158,6 @@ - - org.apache.curator curator-framework @@ -192,8 +194,6 @@ json-simple compile - - com.twitter carbonite @@ -244,10 +244,53 @@ test - junit - junit - 4.11 - test + junit + junit + test + + + org.apache.zookeeper + zookeeper + + + javax.servlet + servlet-api + + + org.slf4j + slf4j-api + + + joda-time + joda-time + + + org.eclipse.jetty + jetty-server + + + org.eclipse.jetty + jetty-util + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.dataformat + jackson-dataformat-smile + + + org.apache.curator + curator-client + + + commons-fileupload + commons-fileupload + + + commons-codec + commons-codec @@ -360,23 +403,22 @@ false + ns-tracker:ns-tracker + hiccup:hiccup + ring:* + compojure:compojure + clj-time:clj-time org.apache.thrift:* io.netty:netty com.google.guava:guava org.apache.httpcomponents:http* org.apache.zookeeper:zookeeper org.apache.curator:* - org.apache.httpcomponents:http* - org.apache.zookeeper:zookeeper - org.apache.curator:* com.twitter:carbonite com.twitter:chill-java - org.objenesis:objenesis org.tukaani:xz org.yaml:snakeyaml org.jgrapht:jgrapht-core - commons-httpclient:commons-httpclient - org.apache.commons:commons-compress org.apache.commons:commons-exec commons-io:commons-io commons-codec:commons-codec @@ -385,168 +427,199 @@ com.googlecode.json-simple:json-simple org.clojure:math.numeric-tower org.clojure:tools.cli + org.clojure:tools.logging org.clojure:tools.macro + org.clojure:java.jmx joda-time:joda-time org.eclipse.jetty:* com.fasterxml.jackson.core:* com.fasterxml.jackson.dataformat:* + clout:clout + org.clojure:tools.namespace + cheshire:cheshire + org.clojure:core.incubator - - com.fasterxml.jackson.core - com.fasterxml.jackson.storm.core + cheshire + org.apache.storm.shade.cheshire - com.fasterxml.jackson.dataformat - com.fasterxml.jackson.storm.dataformat + clojure.tools.logging + org.apache.storm.shade.clojure.tools.logging - org.apache.thrift - org.apache.thrift7 + clojure.core.incubator + org.apache.storm.shade.clojure.core.incubator - org.jboss.netty - org.apache.storm.netty + clojure.tools.namespace + org.apache.storm.shade.clojure.tools.namespace - com.google.common - org.apache.storm.guava + clout + org.apache.storm.shade.clout - com.google.thirdparty - org.apache.storm.guava.thirdparty + compojure + org.apache.storm.shade.compojure - org.apache.http - org.apache.storm.http + ns_tracker + org.apache.storm.shade.ns_tracker - org.apache.zookeeper - org.apache.storm.zookeeper + ns-tracker + org.apache.storm.shade.ns-tracker - org.apache.curator - org.apache.storm.curator + hiccup + org.apache.storm.shade.hiccup + + + ring + org.apache.storm.shade.ring + + + clj_time + org.apache.storm.shade.clj_time + + + clj-time + org.apache.storm.shade.clj-time + + + com.fasterxml + org.apache.storm.shade.com.fasterxml + + + org.apache.thrift + + org.apache.thrift7 org.jboss.netty - org.apache.storm.netty + org.apache.storm.shade.org.jboss.netty com.google.common - org.apache.storm.guava + org.apache.storm.shade.com.google.common com.google.thirdparty - org.apache.storm.guava.thirdparty + org.apache.storm.shade.com.google.thirdparty org.apache.http - org.apache.storm.http + org.apache.storm.shade.org.apache.http - org.apache.zookeeper - org.apache.storm.zookeeper + org.apache.curator + org.apache.storm.shade.org.apache.curator - org.apache.jute - org.apache.storm.jute + org.apache.zookeeper + org.apache.storm.shade.org.apache.zookeeper - org.apache.curator - org.apache.storm.curator + org.apache.jute + org.apache.storm.shade.org.apache.jute carbonite - org.apache.storm.carbonite + org.apache.storm.shade.carbonite com.twitter.chill - org.apache.storm.chill - - - org.objenesis - org.apache.storm.objenesis + org.apache.storm.shade.com.twitter.chill org.tukaani.xz - org.apache.storm.xz + org.apache.storm.shade.org.tukaani.xz org.yaml.snakeyaml - org.apache.storm.snakeyaml + org.apache.storm.shade.org.yaml.snakeyaml org.jgrapht - org.apache.storm.jgrapht + org.apache.storm.shade.org.jgrapht org.fusesource - org.apache.storm.fusesource + org.apache.storm.shade.org.fusesource com.metamx.http.client - org.apache.storm.metamx.http.client + org.apache.storm.shade.com.metamx.http.client org.apache.commons.io - org.apache.storm.commons.io + org.apache.storm.shade.org.apache.commons.io org.apache.commons.codec - org.apache.storm.commons.codec + org.apache.storm.shade.org.apache.commons.codec org.apache.commons.fileupload - org.apache.storm.commons.fileupload - - - org.apache.commons.compress - org.apache.storm.commons.compress + org.apache.storm.shade.org.apache.commons.fileupload org.apache.commons.exec - org.apache.storm.commons.exec + org.apache.storm.shade.org.apache.commons.exec org.apache.commons.lang - org.apache.storm.commons.lang + org.apache.storm.shade.org.apache.commons.lang org.json.simple - org.apache.storm.json.simple + org.apache.storm.shade.org.json.simple clojure.math - org.apache.storm.clojure.math + org.apache.storm.shade.clojure.math clojure.tools.cli - org.apache.storm.clojure.tools.cli + org.apache.storm.shade.clojure.tools.cli cljs.tools.cli - org.apache.storm.cljs.tools.cli + org.apache.storm.shade.cljs.tools.cli clojure.tools.macro - org.apache.storm.clojure.tools.macro + org.apache.storm.shade.clojure.tools.macro org.joda.time - org.apache.storm.joda.time + org.apache.storm.shade.org.joda.time org.eclipse.jetty - org.apache.storm.jetty + org.apache.storm.shade.org.eclipse.jetty + org.clojure:core.incubator**/*.clj + cheshire:cheshire**/*.clj + org.clojure:tools.logging**/*.clj + org.clojure:tools.namespace**/*.clj + org.clojure:math.numeric-tower**/*.clj + org.clojure:tools.macro**/*.clj + org.clojure:tools.cli**/*.clj + ns-tracker:ns-tracker**/*.clj + clout:clout**/*.clj + hiccup:hiccup**/*.clj + clj-time:clj-time**/*.clj + ring:***/*.clj + compojure:compojure**/*.clj org.apache.thrift:* @@ -561,48 +634,12 @@ META-INF/NOTICE.txt - - commons-httpclient:commons-httpclient - - META-INF/LICENSE.txt - META-INF/NOTICE.txt - META-INF/README.txt - - org.apache.zookeeper:zookeeper LICENSE.txt - - commons-httpclient:commons-httpclient - - META-INF/LICENSE.txt - META-INF/NOTICE.txt - META-INF/README.txt - - - - org.apache.zookeeper:zookeeper - - LICENSE.txt - - - - org.objenesis:objenesis - - META-INF/LICENSE.txt - META-INF/NOTICE.txt - - - - org.apache.commons:commons-compress - - META-INF/LICENSE.txt - META-INF/NOTICE.txt - - org.apache.commons:commons-exec diff --git a/storm-core/src/clj/backtype/storm/log.clj b/storm-core/src/clj/backtype/storm/log.clj index 0fcf8228ac6..55b44fec23a 100644 --- a/storm-core/src/clj/backtype/storm/log.clj +++ b/storm-core/src/clj/backtype/storm/log.clj @@ -15,7 +15,7 @@ ;; limitations under the License. (ns backtype.storm.log - (:require [clojure.tools [logging :as log]])) + (:require [clojure.tools.logging :as log])) (defmacro log-message [& args] diff --git a/storm-dist/binary/src/main/assembly/binary.xml b/storm-dist/binary/src/main/assembly/binary.xml index 9f6c8f7c838..4cd6911e9c6 100644 --- a/storm-dist/binary/src/main/assembly/binary.xml +++ b/storm-dist/binary/src/main/assembly/binary.xml @@ -30,38 +30,6 @@ false lib false - - org.apache.thrift:* - io.netty:netty - com.google.guava:guava - org.apache.httpcomponents:http* - org.apache.zookeeper:zookeeper - org.apache.curator:* - org.apache.httpcomponents:http* - org.apache.zookeeper:zookeeper - org.apache.curator:* - com.twitter:carbonite - com.twitter:chill-java - org.objenesis:objenesis - org.tukaani:xz - org.yaml:snakeyaml - org.jgrapht:jgrapht-core - commons-httpclient:commons-httpclient - org.apache.commons:commons-compress - org.apache.commons:commons-exec - commons-io:commons-io - commons-codec:commons-codec - commons-fileupload:commons-fileupload - commons-lang:commons-lang - com.googlecode.json-simple:json-simple - org.clojure:math.numeric-tower - org.clojure:tools.cli - org.clojure:tools.macro - joda-time:joda-time - org.eclipse.jetty:* - com.fasterxml.jackson.core:* - com.fasterxml.jackson.dataformat:* -