copy reference classes from doris fe common and remove fe common depe…

…ndency
apache · Jul 4, 2024 · 6280415 · 6280415
1 parent e9fdd44
commit 6280415
Show file tree

Hide file tree

Showing 38 changed files with 163 additions and 131 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,11 +8,6 @@ spark-doris-connector/output/
 spark-doris-connector/target/
 spark-doris-connector/.idea/
 
-spark-load/spark-dpp/dependency-reduced-pom.xml
-spark-load/spark-dpp/output/
-spark-load/spark-dpp/target/
-spark-load/spark-dpp/.idea/
-
 spark-load/target
 spark-load/spark-load-core/dependency-reduced-pom.xml
 spark-load/spark-load-core/output/

diff --git a/spark-load/pom.xml b/spark-load/pom.xml
@@ -30,14 +30,14 @@
         <module>spark-load-core</module>
         <module>spark-load-dpp</module>
         <module>spark-load-dist</module>
+        <module>spark-load-common</module>
     </modules>
 
     <properties>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <revision>1.0-SNAPSHOT</revision>
-        <doris.fe.version>1.2-SNAPSHOT</doris.fe.version>
         <commons-codec.version>1.13</commons-codec.version>
         <commons-lang3.version>3.9</commons-lang3.version>
         <hadoop.version>3.3.6</hadoop.version>
@@ -56,49 +56,11 @@
         <slf4j.version>2.0.7</slf4j.version>
         <commons-logging.version>1.2</commons-logging.version>
         <aws-java-sdk.version>1.12.669</aws-java-sdk.version>
+        <RoaringBitmap.version>0.8.13</RoaringBitmap.version>
     </properties>
 
     <dependencyManagement>
         <dependencies>
-            <dependency>
-                <groupId>org.apache.doris</groupId>
-                <artifactId>fe-common</artifactId>
-                <version>${doris.fe.version}</version>
-                <exclusions>
-                    <exclusion>
-                        <groupId>org.apache.logging.log4j</groupId>
-                        <artifactId>log4j-1.2-api</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>org.apache.logging.log4j</groupId>
-                        <artifactId>log4j-api</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>org.apache.logging.log4j</groupId>
-                        <artifactId>log4j-core</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>commons-logging</groupId>
-                        <artifactId>commons-logging</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>org.slf4j</groupId>
-                        <artifactId>slf4j-api</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>com.fasterxml.jackson.core</groupId>
-                        <artifactId>jackson-core</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>com.fasterxml.jackson.core</groupId>
-                        <artifactId>jackson-databind</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>com.fasterxml.jackson.core</groupId>
-                        <artifactId>jackson-annotations</artifactId>
-                    </exclusion>
-                </exclusions>
-            </dependency>
             <!-- https://mvnrepository.com/artifact/commons-codec/commons-codec -->
             <dependency>
                 <groupId>commons-codec</groupId>
@@ -372,7 +334,11 @@
                 <artifactId>commons-logging</artifactId>
                 <version>${commons-logging.version}</version>
             </dependency>
-
+            <dependency>
+                <groupId>org.roaringbitmap</groupId>
+                <artifactId>RoaringBitmap</artifactId>
+                <version>${RoaringBitmap.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 

diff --git a/spark-load/spark-load-common/pom.xml b/spark-load/spark-load-common/pom.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.doris</groupId>
+        <artifactId>spark-load</artifactId>
+        <version>${revision}</version>
+    </parent>
+
+    <artifactId>spark-load-common</artifactId>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.roaringbitmap</groupId>
+            <artifactId>RoaringBitmap</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+        </dependency>
+    </dependencies>
+
+</project>
diff --git a/...va/org/apache/doris/common/DppResult.java → ...va/org/apache/doris/common/DppResult.java b/...va/org/apache/doris/common/DppResult.java → ...va/org/apache/doris/common/DppResult.java
@@ -21,6 +21,9 @@
 
 import java.io.Serializable;
 
+/**
+ * Copied from Apache Doris org.apache.doris.sparkdpp.DppResult
+ */
 public class DppResult implements Serializable {
 
     @JsonProperty(value = "is_success", required = true)

diff --git a/spark-load/spark-load-common/src/main/java/org/apache/doris/common/io/BitmapValue.java b/spark-load/spark-load-common/src/main/java/org/apache/doris/common/io/BitmapValue.java
@@ -6,6 +6,9 @@
 import java.io.DataOutput;
 import java.io.IOException;
 
+/**
+ * Copied from Apache Doris
+ */
 public class BitmapValue {
 
     public static final int EMPTY = 0;

diff --git a/spark-load/spark-load-common/src/main/java/org/apache/doris/common/io/Codec.java b/spark-load/spark-load-common/src/main/java/org/apache/doris/common/io/Codec.java
@@ -21,6 +21,9 @@
 import java.io.DataOutput;
 import java.io.IOException;
 
+/**
+ * Copied from Apache Doris
+ */
 public class Codec {
 
     // not support encode negative value now

diff --git a/spark-load/spark-load-common/src/main/java/org/apache/doris/common/io/Hll.java b/spark-load/spark-load-common/src/main/java/org/apache/doris/common/io/Hll.java
@@ -9,6 +9,9 @@
 import java.util.HashSet;
 import java.util.Set;
 
+/**
+ * Copied from Apache Doris
+ */
 public class Hll {
 
     public static final byte HLL_DATA_EMPTY = 0;

diff --git a/spark-load/spark-load-common/src/main/java/org/apache/doris/common/io/Roaring64Map.java b/spark-load/spark-load-common/src/main/java/org/apache/doris/common/io/Roaring64Map.java
@@ -27,6 +27,9 @@
 import java.util.SortedMap;
 import java.util.TreeMap;
 
+/**
+ * Copied from Apache Doris
+ */
 public class Roaring64Map {
 
     private static final boolean DEFAULT_ORDER_IS_SIGNED = false;

diff --git a/spark-load/spark-load-common/src/main/java/org/apache/doris/common/jmockit/AutoType.java b/spark-load/spark-load-common/src/main/java/org/apache/doris/common/jmockit/AutoType.java
@@ -19,25 +19,15 @@
  * |Short                |short
  * |Integer              |int
  * |Float                |float
- * |Long                 |long
+ * |Long                 |longFieldReflection
  * |Double               |double
+ * <p>
+ * Copied from Apache Doris
  */
 public class AutoType {
     private static final Map<Class<?>, Class<?>> PRIMITIVE_TO_WRAPPER = new HashMap();
     private static final Map<Class<?>, Class<?>> WRAPPER_TO_PRIMITIVE = new HashMap();
 
-    public static boolean isWrapperOfPrimitiveType(Class<?> type) {
-        return WRAPPER_TO_PRIMITIVE.containsKey(type);
-    }
-
-    public static Class<?> getPrimitiveType(Class<?> wrapperType) {
-        return WRAPPER_TO_PRIMITIVE.get(wrapperType);
-    }
-
-    public static Class<?> getWrapperType(Class<?> primitiveType) {
-        return PRIMITIVE_TO_WRAPPER.get(primitiveType);
-    }
-
     static {
         WRAPPER_TO_PRIMITIVE.put(Boolean.class, Boolean.TYPE);
         WRAPPER_TO_PRIMITIVE.put(Character.class, Character.TYPE);
@@ -57,4 +47,16 @@ public static Class<?> getWrapperType(Class<?> primitiveType) {
         PRIMITIVE_TO_WRAPPER.put(Long.TYPE, Long.class);
         PRIMITIVE_TO_WRAPPER.put(Double.TYPE, Double.class);
     }
+
+    public static boolean isWrapperOfPrimitiveType(Class<?> type) {
+        return WRAPPER_TO_PRIMITIVE.containsKey(type);
+    }
+
+    public static Class<?> getPrimitiveType(Class<?> wrapperType) {
+        return WRAPPER_TO_PRIMITIVE.get(wrapperType);
+    }
+
+    public static Class<?> getWrapperType(Class<?> primitiveType) {
+        return PRIMITIVE_TO_WRAPPER.get(primitiveType);
+    }
 }
diff --git a/...park-load-common/src/main/java/org/apache/doris/common/jmockit/ConstructorReflection.java b/...park-load-common/src/main/java/org/apache/doris/common/jmockit/ConstructorReflection.java
@@ -12,6 +12,8 @@
 /**
  * Modify from mockit.internal.util.ConstructorReflection JMockit v1.13
  * Util class to invoke constructor of specified class.
+ * <p>
+ * Copied from Apache Doris
  */
 public final class ConstructorReflection {
 
@@ -129,7 +131,8 @@ private static <T> Constructor<T> findCompatibleConstructor(Class<?> theClass, C
             int gap = declaredParamTypes.length - argTypes.length;
             if (gap == 0 && (ParameterReflection.matchesParameterTypes(declaredParamTypes, argTypes)
                     || ParameterReflection.acceptsArgumentTypes(declaredParamTypes, argTypes))
-                    && (found == null || ParameterReflection.hasMoreSpecificTypes(declaredParamTypes, foundParameters))) {
+                    &&
+                    (found == null || ParameterReflection.hasMoreSpecificTypes(declaredParamTypes, foundParameters))) {
                 found = (Constructor<T>) declaredConstructor;
                 foundParameters = declaredParamTypes;
             }
@@ -143,10 +146,12 @@ private static <T> Constructor<T> findCompatibleConstructor(Class<?> theClass, C
             // check if this constructor is belong to a inner class
             // the parameter[0] of inner class's constructor is a instance of outer class
             if (paramTypes[0] == declaringClass && paramTypes.length > argTypes.length) {
-                throw new IllegalArgumentException("Invalid instantiation of inner class; use newInnerInstance instead");
+                throw new IllegalArgumentException(
+                        "Invalid instantiation of inner class; use newInnerInstance instead");
             } else {
                 String argTypesDesc = ParameterReflection.getParameterTypesDescription(argTypes);
-                throw new IllegalArgumentException("No compatible constructor found: " + theClass.getSimpleName() + argTypesDesc);
+                throw new IllegalArgumentException(
+                        "No compatible constructor found: " + theClass.getSimpleName() + argTypesDesc);
             }
         }
     }

diff --git a/...load/spark-load-common/src/main/java/org/apache/doris/common/jmockit/Deencapsulation.java b/...load/spark-load-common/src/main/java/org/apache/doris/common/jmockit/Deencapsulation.java
@@ -7,6 +7,8 @@
 
 /**
  * Modify from mockit.internal.util.Deencapsulation JMockit ver1.13
+ * <p>
+ * Copied from Apache Doris
  */
 public final class Deencapsulation {
     private Deencapsulation() {

diff --git a/...load/spark-load-common/src/main/java/org/apache/doris/common/jmockit/FieldReflection.java b/...load/spark-load-common/src/main/java/org/apache/doris/common/jmockit/FieldReflection.java
@@ -12,10 +12,11 @@
 import java.lang.reflect.Type;
 import java.lang.reflect.TypeVariable;
 
-
 /**
  * Modify from mockit.internal.util.FieldReflection JMockit v1.13
  * Util class to set and get the value of specified field.
+ * <p>
+ * Copied from Apache Doris
  */
 public final class FieldReflection {
     private FieldReflection() {

diff --git a/...oad/spark-load-common/src/main/java/org/apache/doris/common/jmockit/GeneratedClasses.java b/...oad/spark-load-common/src/main/java/org/apache/doris/common/jmockit/GeneratedClasses.java
@@ -10,6 +10,8 @@
 /**
  * Modify from mockit.internal.util.GeneratedClasses JMockit v1.13
  * Helper class to return type of mocked-object
+ * <p>
+ * Copied from Apache Doris
  */
 public final class GeneratedClasses {
     private static final String IMPLCLASS_PREFIX = "$Impl_";

diff --git a/...oad/spark-load-common/src/main/java/org/apache/doris/common/jmockit/MethodReflection.java b/...oad/spark-load-common/src/main/java/org/apache/doris/common/jmockit/MethodReflection.java
@@ -13,6 +13,8 @@
 /**
  * Modify from mockit.internal.util.MethodReflection JMockit v1.13
  * Util class to get and invoke method from specified class.
+ * <p>
+ * Copied from Apache Doris
  */
 public final class MethodReflection {
     private MethodReflection() {
@@ -24,9 +26,11 @@ public static <T> T invoke(Class<?> theClass, Object targetInstance, String meth
         }
         boolean staticMethod = targetInstance == null;
         Class<?>[] argTypes = ParameterReflection.getArgumentTypesFromArgumentValues(methodArgs);
-        Method method = staticMethod ? findCompatibleStaticMethod(theClass, methodName, argTypes) : findCompatibleMethod(theClass, methodName, argTypes);
+        Method method = staticMethod ? findCompatibleStaticMethod(theClass, methodName, argTypes) :
+                findCompatibleMethod(theClass, methodName, argTypes);
         if (staticMethod && !Modifier.isStatic(method.getModifiers())) {
-            throw new IllegalArgumentException("Attempted to invoke non-static method without an instance to invoke it on");
+            throw new IllegalArgumentException(
+                    "Attempted to invoke non-static method without an instance to invoke it on");
         } else {
             T result = invoke(targetInstance, method, methodArgs);
             return result;
@@ -110,7 +114,7 @@ private static Method findCompatibleMethodInClass(Class<?> theClass, String meth
                 if (gap == 0 && (ParameterReflection.matchesParameterTypes(declaredParamTypes, argTypes)
                         || ParameterReflection.acceptsArgumentTypes(declaredParamTypes, argTypes))
                         && (foundParamTypes == null
-                            || ParameterReflection.hasMoreSpecificTypes(declaredParamTypes, foundParamTypes))) {
+                        || ParameterReflection.hasMoreSpecificTypes(declaredParamTypes, foundParamTypes))) {
                     found = declaredMethod;
                     foundParamTypes = declaredParamTypes;
                 }
@@ -132,7 +136,9 @@ private static Method findCompatibleMethodIfAvailable(Class<?> theClass, String
 
         while (true) {
             Method compatibleMethod = findCompatibleMethodInClass(theClass, methodName, argTypes);
-            if (compatibleMethod != null && (methodFound == null || ParameterReflection.hasMoreSpecificTypes(compatibleMethod.getParameterTypes(), methodFound.getParameterTypes()))) {
+            if (compatibleMethod != null && (methodFound == null ||
+                    ParameterReflection.hasMoreSpecificTypes(compatibleMethod.getParameterTypes(),
+                            methodFound.getParameterTypes()))) {
                 methodFound = compatibleMethod;
             }
 
@@ -146,7 +152,6 @@ private static Method findCompatibleMethodIfAvailable(Class<?> theClass, String
     }
 
 
-
     // ensure that field is accessible
     public static void makeAccessible(AccessibleObject classMember) {
         if (!classMember.isAccessible()) {

diff --git a/.../spark-load-common/src/main/java/org/apache/doris/common/jmockit/ParameterReflection.java b/.../spark-load-common/src/main/java/org/apache/doris/common/jmockit/ParameterReflection.java
@@ -10,6 +10,8 @@
 /**
  * Modify from mockit.internal.util.ParameterReflection JMockit v1.13
  * Util class to verify parameter of methods.
+ * <p>
+ * Copied from Apache Doris
  */
 public final class ParameterReflection {
     public static final Class<?>[] NO_PARAMETERS = new Class[0];

diff --git a/spark-load/spark-load-common/src/main/java/org/apache/doris/config/EtlJobConfig.java b/spark-load/spark-load-common/src/main/java/org/apache/doris/config/EtlJobConfig.java
@@ -14,6 +14,9 @@
 import java.util.List;
 import java.util.Map;
 
+/**
+ * Copied from Apache Doris org.apache.doris.sparkdpp.EtlJobConfig
+ */
 public class EtlJobConfig implements Serializable {
     // global dict
     public static final String GLOBAL_DICT_TABLE_NAME = "doris_global_dict_table_%d";

diff --git a/spark-load/spark-load-core/pom.xml b/spark-load/spark-load-core/pom.xml
@@ -36,6 +36,11 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.doris</groupId>
+            <artifactId>spark-load-common</artifactId>
+            <version>${revision}</version>
+        </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
@@ -79,10 +84,6 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.apache.doris</groupId>
-            <artifactId>fe-common</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>
             <artifactId>log4j-core</artifactId>