Shaded everything that was not already shaded, and cleaned up build s…

…o the dependency-reduced pom.xml is used correctly.
aiping312 · Oct 2, 2015 · 12fdaa0 · 12fdaa0
1 parent 768a859
commit 12fdaa0
Show file tree

Hide file tree

Showing 10 changed files with 289 additions and 198 deletions.
diff --git a/DEVELOPER.md b/DEVELOPER.md
@@ -241,17 +241,16 @@ in order to get started as fast as possible. Users can still install a specific
 
 The following commands must be run from the top-level directory.
 
-    # Build the code and run the tests (requires nodejs, python and ruby installed) 
-    # `mvn clean package` will fail because storm-core requires storm-maven-plugin.
-    # This plugin should be installed before compiling storm-core.
-    $ mvn clean install
+The first step is to build/install the plugins and storm-core
 
-    # Build the code and run the tests, with specifying default test timeout (in millisecond)
-    $ export STORM_TEST_TIMEOUT_MS=10000
-    $ mvn clean install
+`mvn clean install -Pstorm-core`
 
-    # Build the code but skip the tests
-    $ mvn clean install -DskipTests=true
+If you wish to skip the unit tests you can do this by adding `-DskipTests` to the command line. Once this
+completes successfully you may run
+
+`mvn clean install -Pstorm-more`
+
+to build and test all of the external libraries and examples.  Again if you want to include `-DskipTests` you can.
 
 In case you modified `storm.thrift`, you have to regenerate thrift code as java and python code before compiling whole project.
 
@@ -266,19 +265,7 @@ You can also run tests selectively via the Clojure REPL.  The following example
 [auth_test.clj](storm-core/test/clj/backtype/storm/security/auth/auth_test.clj), which has the namespace
 `backtype.storm.security.auth.auth-test`.
 
-First, start the REPL from within the relevant sub-project (here: `storm-core`):
-
-    $ cd storm-core/
-    $ mvn clojure:repl
-
-Now we run the tests in `auth_test.clj` in the REPL:
-
-```clojure
-;; You can use both absolute as well as relative paths to the .clj file.
-(load-file "test/clj/backtype/storm/security/auth/auth_test.clj")
-(ns backtype.storm.security.auth.auth-test)
-(run-tests)
-```
+You can also run tests selectively with `-Dtest=<test_name>`.  This works for both clojure and junit tests.
 
 > Tip: IDEs such as IntelliJ IDEA support a built-in Clojure REPL, which you can also use to run tests selectively.
 > Sometimes you may find that tests pass/fail depending on which REPL you use, which -- although frustrating --
@@ -295,7 +282,8 @@ You can create a _distribution_ (like what you can download from Apache) as foll
 do not use the Maven release plugin because creating an official release is the task of our release manager.
 
     # First, build the code.
-    $ mvn clean install  # you may skip tests with `-DskipTests=true` to save time
+    $ mvn clean install -Pstorm-core # you may skip tests with `-DskipTests=true` to save time
+    $ mvn clean install -Pstorm-more # you may skip tests with `-DskipTests=true` to save time
 
     # Create the binary distribution.
     $ cd storm-dist/binary && mvn package

diff --git a/dev-tools/travis/travis-install.sh b/dev-tools/travis/travis-install.sh
@@ -26,12 +26,23 @@ TRAVIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 
 cd ${STORM_SRC_ROOT_DIR}
 
-python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install.txt" mvn clean install --batch-mode -DskipTests -Pnative
+python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install-storm-core.txt" mvn clean install -DskipTests -Pnative -Pstorm-core --batch-mode
 BUILD_RET_VAL=$?
 
 if [[ "$BUILD_RET_VAL" != "0" ]];
 then
-  cat "install.txt"
+  cat "install-storm-core.txt"
+  exit ${BUILD_RET_VAL}
+fi
+
+exit ${BUILD_RET_VAL}
+
+python ${TRAVIS_SCRIPT_DIR}/save-logs.py "install-storm-more.txt" mvn clean install -DskipTests -Pstorm-more
+BUILD_RET_VAL=$?
+
+if [[ "$BUILD_RET_VAL" != "0" ]];
+then
+  cat "install-storm-more.txt"
 fi
 
 exit ${BUILD_RET_VAL}
diff --git a/dev-tools/travis/travis-script.sh b/dev-tools/travis/travis-script.sh
@@ -31,7 +31,7 @@ cd ${STORM_SRC_ROOT_DIR}
 export STORM_TEST_TIMEOUT_MS=100000
 
 # We now lean on Travis CI's implicit behavior, ```mvn clean install -DskipTests``` before running script
-mvn --batch-mode test -fae -Pnative
+mvn --batch-mode test -fae -Pnative -Pstorm-core && mvn test -fae -Pnative -Pstorm-more
 BUILD_RET_VAL=$?
 
 for dir in `find . -type d -and -wholename \*/target/\*-reports`;

diff --git a/external/storm-elasticsearch/pom.xml b/external/storm-elasticsearch/pom.xml
@@ -40,6 +40,11 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <scope>compile</scope>
+        </dependency>
         <dependency>
             <groupId>org.apache.storm</groupId>
             <artifactId>storm-core</artifactId>

diff --git a/external/storm-kafka/pom.xml b/external/storm-kafka/pom.xml
@@ -56,6 +56,11 @@
         <testSourceDirectory>src/test</testSourceDirectory>
     </build>
     <dependencies>
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <scope>compile</scope>
+        </dependency>
         <dependency>
             <groupId>org.mockito</groupId>
             <artifactId>mockito-all</artifactId>

diff --git a/external/storm-solr/pom.xml b/external/storm-solr/pom.xml
@@ -21,7 +21,7 @@
     </developers>
 
     <dependencies>
-        <dependency>
+       <dependency>
             <groupId>org.apache.storm</groupId>
             <artifactId>storm-core</artifactId>
             <version>${project.version}</version>
@@ -95,4 +95,4 @@
             </plugin>
         </plugins>
     </build>
-</project>
+</project>
diff --git a/pom.xml b/pom.xml
@@ -154,26 +154,6 @@
         <maven>3.0.0</maven>
     </prerequisites>
 
-    <modules>
-        <module>storm-buildtools/maven-shade-clojure-transformer</module>
-        <module>storm-buildtools/storm-maven-plugins</module>
-        <module>storm-multilang/javascript</module>
-        <module>storm-multilang/python</module>
-        <module>storm-multilang/ruby</module>
-        <module>storm-core</module>
-        <module>external/storm-kafka</module>
-        <module>external/storm-hdfs</module>
-        <module>external/storm-hbase</module>
-        <module>external/storm-hive</module>
-        <module>external/storm-jdbc</module>
-        <module>external/storm-redis</module>
-        <module>external/storm-eventhubs</module>
-        <module>external/flux</module>
-        <module>external/storm-elasticsearch</module>
-        <module>external/storm-solr</module>
-        <module>examples/storm-starter</module>
-    </modules>
-
     <scm>
         <connection>scm:git:https://git-wip-us.apache.org/repos/asf/storm.git</connection>
         <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/storm.git</developerConnection>
@@ -197,10 +177,13 @@
         <commons-io.version>2.4</commons-io.version>
         <commons-lang.version>2.5</commons-lang.version>
         <commons-exec.version>1.1</commons-exec.version>
+        <commons-fileupload.version>1.2.1</commons-fileupload.version>
+        <commons-codec.version>1.6</commons-codec.version>
         <clj-time.version>0.8.0</clj-time.version>
         <curator.version>2.5.0</curator.version>
         <json-simple.version>1.1</json-simple.version>
         <ring.version>1.3.0</ring.version>
+        <ring-json.version>0.3.1</ring-json.version>
         <jetty.version>7.6.13.v20130916</jetty.version>
         <clojure.tools.logging.version>0.2.3</clojure.tools.logging.version>
         <clojure.math.numeric-tower.version>0.0.1</clojure.math.numeric-tower.version>
@@ -214,6 +197,7 @@
         <netty.version>3.9.0.Final</netty.version>
         <log4j-over-slf4j.version>1.6.6</log4j-over-slf4j.version>
         <log4j.version>2.1</log4j.version>
+        <slf4j.version>1.7.7</slf4j.version>
         <clojure.tools.nrepl.version>0.2.3</clojure.tools.nrepl.version>
         <clojure-complete.version>0.2.3</clojure-complete.version>
         <mockito.version>1.9.5</mockito.version>
@@ -224,9 +208,47 @@
         <hive.version>0.14.0</hive.version>
         <hadoop.version>2.6.0</hadoop.version>
         <kryo.version>2.21</kryo.version>
+        <servlet.version>2.5</servlet.version>
+        <joda-time.version>2.3</joda-time.version>
+        <jackson.version>2.3.1</jackson.version>
+        <thrift.version>0.9.2</thrift.version>
+        <junit.version>4.11</junit.version>
     </properties>
 
     <profiles>
+        <profile>
+            <id>storm-core</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <modules>
+                <module>storm-multilang/javascript</module>
+                <module>storm-multilang/python</module>
+                <module>storm-multilang/ruby</module>
+                <module>storm-buildtools/maven-shade-clojure-transformer</module>
+                <module>storm-buildtools/storm-maven-plugins</module>
+                <module>storm-core</module>
+            </modules>
+        </profile>
+        <profile>
+            <id>storm-more</id>
+            <activation>
+                <activeByDefault>true</activeByDefault>
+            </activation>
+            <modules>
+                <module>external/storm-kafka</module>
+                <module>external/storm-hdfs</module>
+                <module>external/storm-hbase</module>
+                <module>external/storm-hive</module>
+                <module>external/storm-jdbc</module>
+                <module>external/storm-redis</module>
+                <module>external/storm-eventhubs</module>
+                <module>external/flux</module>
+                <module>external/storm-elasticsearch</module>
+                <module>external/storm-solr</module>
+                <module>examples/storm-starter</module>
+            </modules>
+        </profile>
         <profile>
             <id>sign</id>
             <build>
@@ -379,6 +401,41 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+            <dependency>
+                <groupId>javax.servlet</groupId>
+                <artifactId>servlet-api</artifactId>
+                <version>${servlet.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-api</artifactId>
+                <version>${slf4j.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>joda-time</groupId>
+                <artifactId>joda-time</artifactId>
+                <version>${joda-time.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-core</artifactId>
+                <version>${jackson.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.dataformat</groupId>
+                <artifactId>jackson-dataformat-smile</artifactId>
+                <version>${jackson.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>commons-fileupload</groupId>
+                <artifactId>commons-fileupload</artifactId>
+                <version>${commons-fileupload.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>commons-codec</groupId>
+                <artifactId>commons-codec</artifactId>
+                <version>${commons-codec.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.apache.curator</groupId>
                 <artifactId>curator-framework</artifactId>
@@ -409,6 +466,11 @@
                     </exclusion>
                 </exclusions>
             </dependency>
+            <dependency>
+                <groupId>org.apache.curator</groupId>
+                <artifactId>curator-client</artifactId>
+                <version>${curator.version}</version>
+            </dependency>
 
             <dependency>
                 <groupId>com.googlecode.json-simple</groupId>
@@ -435,6 +497,11 @@
                 <artifactId>ring-jetty-adapter</artifactId>
                 <version>${ring.version}</version>
             </dependency>
+            <dependency>
+                <groupId>ring</groupId>
+                <artifactId>ring-json</artifactId>
+                <version>${ring-json.version}</version>
+            </dependency>
             <dependency>
 	      <groupId>org.eclipse.jetty</groupId>
 	      <artifactId>jetty-servlet</artifactId>
@@ -445,6 +512,17 @@
 	      <artifactId>jetty-servlets</artifactId>
 	      <version>${jetty.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.eclipse.jetty</groupId>
+                <artifactId>jetty-server</artifactId>
+                <version>${jetty.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.eclipse.jetty</groupId>
+                <artifactId>jetty-util</artifactId>
+                <version>${jetty.version}</version>
+            </dependency>
+
             <dependency>
                 <groupId>org.clojure</groupId>
                 <artifactId>tools.logging</artifactId>
@@ -570,16 +648,15 @@
             <dependency>
                 <groupId>org.apache.thrift</groupId>
                 <artifactId>libthrift</artifactId>
-                <version>0.9.2</version>
+                <version>${thrift.version}</version>
                 <scope>compile</scope>
             </dependency>
-			<!-- used by examples/storm-starter -->
-		    <dependency>
-		      <groupId>junit</groupId>
-		      <artifactId>junit</artifactId>
-		      <version>4.11</version>
-		      <scope>test</scope>
-		    </dependency>
+            <dependency>
+                <groupId>junit</groupId>
+                <artifactId>junit</artifactId>
+                <version>${junit.version}</version>
+                <scope>test</scope>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -673,7 +750,7 @@
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-shade-plugin</artifactId>
-                    <version>2.2</version>
+                    <version>2.4.1</version>
                 </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>