Develop and build Kubernetes modules in isolation without the other Spark modules

pom.xml

@@ -2615,7 +2615,7 @@
     <profile>
       <id>kubernetes</id>
       <modules>
-        <module>resource-managers/kubernetes/core</module>
+        <module>resource-managers/kubernetes</module>
       </modules>
     </profile>
 

resource-managers/kubernetes/core/pom.xml

@@ -19,9 +19,9 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.11</artifactId>
+    <artifactId>spark-kubernetes-parent_2.11</artifactId>
     <version>2.2.0-k8s-0.5.0</version>
-    <relativePath>../../../pom.xml</relativePath>
+    <relativePath>../pom.xml</relativePath>
   </parent>
 
   <artifactId>spark-kubernetes_2.11</artifactId>
@@ -39,6 +39,64 @@
       <version>${project.version}</version>
     </dependency>
 
+    <!-- Jetty dependencies promoted to compile here so they are shaded
+     and inlined into spark-core jar -->
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-plus</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-security</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-util</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-server</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-http</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-continuation</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlet</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-proxy</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-client</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlets</artifactId>
+      <scope>compile</scope>
+    </dependency>
+    <dependency>
+      <groupId>javax.servlet</groupId>
+      <artifactId>javax.servlet-api</artifactId>
+      <version>${javaxservlet.version}</version>
+    </dependency>
+
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-core_${scala.binary.version}</artifactId>

resource-managers/kubernetes/docker-minimal-bundle/pom.xml

@@ -20,9 +20,9 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.11</artifactId>
+    <artifactId>spark-kubernetes-parent_2.11</artifactId>
     <version>2.2.0-k8s-0.5.0</version>
-    <relativePath>../../../pom.xml</relativePath>
+    <relativePath>../pom.xml</relativePath>
   </parent>
 
   <artifactId>spark-docker-minimal-bundle_2.11</artifactId>

resource-managers/kubernetes/integration-tests-spark-jobs/pom.xml

@@ -19,9 +19,9 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.11</artifactId>
+    <artifactId>spark-kubernetes-parent_2.11</artifactId>
     <version>2.2.0-k8s-0.5.0</version>
-    <relativePath>../../../pom.xml</relativePath>
+    <relativePath>../pom.xml</relativePath>
   </parent>
 
   <artifactId>spark-kubernetes-integration-tests-spark-jobs_2.11</artifactId>

resource-managers/kubernetes/integration-tests/pom.xml

@@ -19,13 +19,14 @@
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>org.apache.spark</groupId>
-    <artifactId>spark-parent_2.11</artifactId>
+    <artifactId>spark-kubernetes-parent_2.11</artifactId>
     <version>2.2.0-k8s-0.5.0</version>
-    <relativePath>../../../pom.xml</relativePath>
+    <relativePath>../pom.xml</relativePath>
   </parent>
 
   <artifactId>spark-kubernetes-integration-tests_2.11</artifactId>
   <properties>
+    <root.src.folder>${project.basedir}/../../..</root.src.folder>
     <sbt.project.name>kubernetes-integration-tests</sbt.project.name>
   </properties>
   <packaging>jar</packaging>
@@ -56,6 +57,14 @@
       <type>test-jar</type>
       <scope>test</scope>
     </dependency>
+    <!--
+      jetty scope is defined as `provided` in parent pom.
+    -->
+    <dependency>
+      <groupId>org.eclipse.jetty</groupId>
+      <artifactId>jetty-servlet</artifactId>
+      <version>${jetty.version}</version>
+    </dependency>
     <dependency>
       <groupId>org.apache.spark</groupId>
       <artifactId>spark-kubernetes-integration-tests-spark-jobs_${scala.binary.version}</artifactId>
@@ -220,6 +229,51 @@
               </resources>
             </configuration>
           </execution>
+          <execution>
+            <id>copy-integration-bin</id>
+            <phase>pre-integration-test</phase>
+            <goals>
+              <goal>copy-resources</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/docker/bin</outputDirectory>
+              <resources>
+                <resource>
+                  <directory>${root.src.folder}/bin</directory>
+                </resource>
+              </resources>
+            </configuration>
+          </execution>
+          <execution>
+            <id>copy-integration-sbin</id>
+            <phase>pre-integration-test</phase>
+            <goals>
+              <goal>copy-resources</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/docker/sbin</outputDirectory>
+              <resources>
+                <resource>
+                  <directory>${root.src.folder}/sbin</directory>
+                </resource>
+              </resources>
+            </configuration>
+          </execution>
+          <execution>
+            <id>copy-integration-conf</id>
+            <phase>pre-integration-test</phase>
+            <goals>
+              <goal>copy-resources</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/docker/conf</outputDirectory>
+              <resources>
+                <resource>
+                  <directory>${root.src.folder}/conf</directory>
+                </resource>
+              </resources>
+            </configuration>
+          </execution>
           <execution>
               <id>copy-integration-python</id>
               <phase>pre-integration-test</phase>
@@ -230,10 +284,10 @@
                   <outputDirectory>${project.build.directory}/docker/python</outputDirectory>
                   <resources>
                       <resource>
-                          <directory>${project.parent.basedir}/python</directory>
+                          <directory>${root.src.folder}/python</directory>
                           <excludes>
-                            <exclude>${project.parent.basedir}/python/.egg</exclude>
-                            <exclude>${project.parent.basedir}/python/dist</exclude>
+                            <exclude>${root.src.folder}/python/.egg</exclude>
+                            <exclude>${root.src.folder}/python/dist</exclude>
                           </excludes>
                       </resource>
                   </resources>
@@ -249,7 +303,7 @@
               <outputDirectory>${project.build.directory}/docker/R</outputDirectory>
               <resources>
                 <resource>
-                  <directory>${project.parent.basedir}/R</directory>
+                  <directory>${root.src.folder}/R</directory>
                 </resource>
               </resources>
             </configuration>
@@ -264,7 +318,7 @@
               <outputDirectory>${project.build.directory}/docker/data</outputDirectory>
               <resources>
                 <resource>
-                  <directory>${project.parent.basedir}/data</directory>
+                  <directory>${root.src.folder}/data</directory>
                   <filtering>true</filtering>
                 </resource>
               </resources>
@@ -280,7 +334,7 @@
               <outputDirectory>${project.build.directory}/docker/licenses</outputDirectory>
               <resources>
                 <resource>
-                  <directory>${project.parent.basedir}/licenses</directory>
+                  <directory>${root.src.folder}/licenses</directory>
                   <filtering>true</filtering>
                 </resource>
               </resources>
@@ -296,7 +350,7 @@
               <outputDirectory>${project.build.directory}/docker/examples/jars</outputDirectory>
               <resources>
                 <resource>
-                  <directory>${project.parent.basedir}/examples/target/scala-2.11/jars</directory>
+                  <directory>${root.src.folder}/examples/target/scala-2.11/jars</directory>
                   <filtering>true</filtering>
                 </resource>
               </resources>
@@ -312,7 +366,7 @@
               <outputDirectory>${project.build.directory}/docker/examples/src/main</outputDirectory>
               <resources>
                 <resource>
-                  <directory>${project.parent.basedir}/examples/src/main</directory>
+                  <directory>${root.src.folder}/examples/src/main</directory>
                   <filtering>true</filtering>
                 </resource>
               </resources>
@@ -337,6 +391,30 @@
               </target>
             </configuration>
           </execution>
+          <execution>
+            <id>bin-chmod</id>
+            <phase>pre-integration-test</phase>
+            <configuration>
+              <target>
+                <chmod file="${project.build.directory}/docker/bin/*" perm="755"/>
+              </target>
+            </configuration>
+            <goals>
+              <goal>run</goal>
+            </goals>
+          </execution>
+          <execution>
+            <id>sbin-chmod</id>
+            <phase>pre-integration-test</phase>
+            <configuration>
+              <target>
+                <chmod file="${project.build.directory}/docker/sbin/*" perm="755"/>
+              </target>
+            </configuration>
+            <goals>
+              <goal>run</goal>
+            </goals>
+          </execution>
         </executions>
       </plugin>
       <plugin>

resource-managers/kubernetes/pom.xml

@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.spark</groupId>
+    <artifactId>spark-parent_2.11</artifactId>
+    <version>2.2.0-k8s-0.5.0</version>
+    <relativePath>../../pom.xml</relativePath>
+  </parent>
+
+  <artifactId>spark-kubernetes-parent_2.11</artifactId>
+  <packaging>pom</packaging>
+  <name>Spark Project Kubernetes Parent</name>
+
+  <modules>
+    <module>core</module>
+    <module>docker-minimal-bundle</module>
+    <module>integration-tests</module>
+    <module>integration-tests-spark-jobs</module>
+    <module>integration-tests-spark-jobs-helpers</module>
+  </modules>
+
+  <build>
+    <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
+    <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+  </build>
+
+</project>