Merge branch 'master' into better-kernel-log

oaplatform · Dec 27, 2023 · 10caf5f · 10caf5f
2 parents a8246a3 + c6a80b5
commit 10caf5f
Show file tree

Hide file tree

Showing 136 changed files with 5,196 additions and 3,418 deletions.
diff --git a/maven/pom.xml b/maven/pom.xml
@@ -57,7 +57,7 @@
                     <dependency>
                         <groupId>org.ow2.asm</groupId>
                         <artifactId>asm</artifactId>
-                        <version>9.2</version>
+                        <version>9.6</version>
                     </dependency>
                 </dependencies>
                 <executions>

diff --git a/oap-hadoop/pom.xml b/oap-hadoop/pom.xml
@@ -1,24 +1,63 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
 
     <parent>
         <groupId>oap</groupId>
         <artifactId>oap</artifactId>
         <version>${oap.project.version}</version>
     </parent>
 
+    <modelVersion>4.0.0</modelVersion>
     <artifactId>oap-hadoop</artifactId>
 
     <dependencies>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-bundle</artifactId>
+            <version>${oap.deps.aws-java-sdk-bundle.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.docker-java</groupId>
+            <artifactId>docker-java-core</artifactId>
+            <version>${oap.deps.docker-java.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.github.docker-java/docker-java-transport-zerodep -->
+        <dependency>
+            <groupId>com.github.docker-java</groupId>
+            <artifactId>docker-java-transport-zerodep</artifactId>
+            <version>${oap.deps.docker-java.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.hierynomus</groupId>
+            <artifactId>sshj</artifactId>
+            <version>${oap.deps.sshj.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.jcraft</groupId>
+            <artifactId>jsch</artifactId>
+            <version>${oap.deps.jsch.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>net.i2p.crypto</groupId>
+            <artifactId>eddsa</artifactId>
+            <version>${oap.deps.eddsa.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>oap</groupId>
             <artifactId>oap-stdlib</artifactId>
             <version>${project.parent.version}</version>
         </dependency>
-
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-aws</artifactId>
+            <version>${oap.deps.hadoop.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
@@ -28,6 +67,10 @@
                     <groupId>com.fasterxml.jackson.core</groupId>
                     <artifactId>jackson-databind</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-log4j12</artifactId>
@@ -38,48 +81,24 @@
                 </exclusion>
             </exclusions>
         </dependency>
-
         <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-aws</artifactId>
-            <version>${oap.deps.hadoop.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.amazonaws</groupId>
-            <artifactId>aws-java-sdk-bundle</artifactId>
-            <version>${oap.deps.aws-java-sdk-bundle.version}</version>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>${oap.deps.lombok.version}</version>
+            <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers</artifactId>
             <version>${oap.deps.testcontainers.version}</version>
-        </dependency>
-
-
-        <dependency>
-            <groupId>com.hierynomus</groupId>
-            <artifactId>sshj</artifactId>
-            <version>0.24.0</version>
             <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.jcraft</groupId>
-            <artifactId>jsch</artifactId>
-            <version>0.1.54</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>net.i2p.crypto</groupId>
-            <artifactId>eddsa</artifactId>
-            <version>0.2.0</version>
-            <scope>test</scope>
-        </dependency>
-
-        <dependency>
-            <groupId>org.projectlombok</groupId>
-            <artifactId>lombok</artifactId>
-            <version>${oap.deps.lombok.version}</version>
-            <scope>provided</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
     </dependencies>
+
 </project>
diff --git a/oap-hadoop/src/main/java/oap/hadoop/OapHadoopConfiguration.java b/oap-hadoop/src/main/java/oap/hadoop/OapHadoopConfiguration.java
@@ -1,5 +1,7 @@
 package oap.hadoop;
 
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Preconditions;
 import lombok.ToString;
 import lombok.extern.slf4j.Slf4j;
@@ -12,9 +14,11 @@
 @Slf4j
 @ToString
 public class OapHadoopConfiguration extends Configuration {
-    private final OapFileSystemType fileSystemType;
+    public OapFileSystemType fileSystemType;
 
-    public OapHadoopConfiguration( OapFileSystemType fileSystemType, Map<String, Object> configuration ) {
+    @JsonCreator
+    public OapHadoopConfiguration( @JsonProperty OapFileSystemType fileSystemType,
+                                   @JsonProperty Map<String, Object> configuration ) {
         super( false );
         this.fileSystemType = fileSystemType;
 
@@ -67,4 +71,23 @@ private static String concat( String left, String right ) {
     public Path getPath( String name ) {
         return fileSystemType.getPath( name, this );
     }
+
+    @JsonProperty
+    public Map<String, String> getConfiguration() {
+        var ret = new LinkedHashMap<String, String>();
+        getProps().forEach( ( k, v ) -> ret.put( ( String ) k, ( String ) v ) );
+        return ret;
+    }
+
+    public void set( OapHadoopConfiguration oapHadoopConfiguration ) {
+        reset();
+
+        this.fileSystemType = oapHadoopConfiguration.fileSystemType;
+
+        oapHadoopConfiguration.getConfiguration().forEach( this::set );
+    }
+
+    public void reset() {
+        reloadConfiguration();
+    }
 }
diff --git a/oap-hadoop/src/test/java/oap/hadoop/FileSystemTest.java b/oap-hadoop/src/test/java/oap/hadoop/FileSystemTest.java
@@ -9,6 +9,7 @@
 import java.nio.file.Path;
 import java.util.Map;
 
+import static oap.io.content.ContentReader.ofString;
 import static org.assertj.core.api.Assertions.assertThat;
 
 public class FileSystemTest extends Fixtures {
@@ -43,7 +44,7 @@ public void testCopy() {
         assertThat( fileSystem.exists( s3File ) ).isFalse();
         fileSystem.copy( inFile, s3File, false );
         assertThat( fileSystem.exists( s3File ) ).isTrue();
-        assertThat( s3MockFixture.readFile( "test-bucket", "folder/file.txt" ) ).isEqualTo( "txt" );
+        assertThat( s3MockFixture.readFile( "test-bucket", "folder/file.txt", ofString() ) ).isEqualTo( "txt" );
 
         fileSystem.copy( s3File, outFile, false );
 

diff --git a/...c/main/java/oap/hadoop/S3MockFixture.java → ...c/test/java/oap/hadoop/S3MockFixture.java b/...c/main/java/oap/hadoop/S3MockFixture.java → ...c/test/java/oap/hadoop/S3MockFixture.java
@@ -6,13 +6,13 @@
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.util.IOUtils;
 import com.github.dockerjava.api.model.ExposedPort;
 import com.github.dockerjava.api.model.PortBinding;
 import com.github.dockerjava.api.model.Ports;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import oap.io.IoStreams;
+import oap.io.content.ContentReader;
 import oap.testng.AbstractEnvFixture;
 import org.testcontainers.containers.GenericContainer;
 import org.testcontainers.containers.output.Slf4jLogConsumer;
@@ -82,9 +82,9 @@ public void deleteBucket( String bucketName ) {
         s3().deleteBucket( bucketName );
     }
 
-    public String readFile( String bucketName, String key ) {
+    public <T> T readFile( String bucketName, String key, ContentReader<T> reader ) {
         try( S3Object object = s3().getObject( bucketName, key ) ) {
-            return IOUtils.toString( IoStreams.in( object.getObjectContent(), IoStreams.Encoding.from( key ) ) );
+            return reader.read( IoStreams.in( object.getObjectContent(), IoStreams.Encoding.from( key ) ) );
         } catch( IOException e ) {
             throw new SdkClientException( "Error streaming content from S3 during download", e );
         }

diff --git a/oap-pnio/src/test/java/oap/http/pnio/PnioHttpHandlerTest.java b/oap-pnio/src/test/java/oap/http/pnio/PnioHttpHandlerTest.java
@@ -176,7 +176,7 @@ private void runWithWorkflow( int requestSize, int responseSize, int ioThreads,
                 .cpuAffinityFirstCpu( -1 )
                 .build();
         try( PnioHttpHandler<TestState> httpHandler = new PnioHttpHandler<>( settings, workflow, this::errorResponse );
-             NioHttpServer httpServer = new NioHttpServer( port ) ) {
+             NioHttpServer httpServer = new NioHttpServer( new NioHttpServer.DefaultPort( port ) ) ) {
             httpServer.ioThreads = ioThreads;
             httpServer.start();
 

diff --git a/oap-stdlib/pom.xml b/oap-stdlib/pom.xml
@@ -53,11 +53,6 @@
             <artifactId>commons-text</artifactId>
             <version>${oap.deps.apache.commons.text.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-compress</artifactId>
-            <version>${oap.deps.apache.compress.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-lang3</artifactId>
@@ -68,6 +63,11 @@
             <artifactId>commons-io</artifactId>
             <version>${oap.deps.apache.io.version}</version>
         </dependency>
+        <dependency>
+            <groupId>commons-codec</groupId>
+            <artifactId>commons-codec</artifactId>
+            <version>${oap.deps.apache.commons.codec.version}</version>
+        </dependency>
         <dependency>
             <groupId>joda-time</groupId>
             <artifactId>joda-time</artifactId>
@@ -178,9 +178,14 @@
             <version>${oap.deps.assertj.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.lz4</groupId>
-            <artifactId>lz4-java</artifactId>
-            <version>${oap.deps.lz4.version}</version>
+            <groupId>io.airlift</groupId>
+            <artifactId>aircompressor</artifactId>
+            <version>${oap.deps.aircompressor.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+            <version>${oap.deps.luben-zstd-jni.version}</version>
         </dependency>
         <dependency>
             <groupId>com.github.rholder</groupId>
@@ -219,7 +224,7 @@
         <dependency>
             <groupId>oap</groupId>
             <artifactId>config</artifactId>
-            <version>${oap.deps.typesafe-config.version}</version>
+            <version>${oap.deps.config.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
@@ -401,7 +406,7 @@
         <dependency>
             <groupId>net.openhft</groupId>
             <artifactId>affinity</artifactId>
-            <version>${oap.deps.affinity}</version>
+            <version>${oap.deps.affinity.version}</version>
         </dependency>
         <dependency>
             <groupId>org.mock-server</groupId>
@@ -447,11 +452,6 @@
                 </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
-            <groupId>com.github.luben</groupId>
-            <artifactId>zstd-jni</artifactId>
-            <version>${oap.deps.zstd-jni.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.openjdk.jol</groupId>
             <artifactId>jol-core</artifactId>

diff --git a/oap-stdlib/src/main/java/oap/application/Kernel.java b/oap-stdlib/src/main/java/oap/application/Kernel.java
@@ -156,7 +156,8 @@ public void start( Path appConfigPath, Map<Object, Object> properties ) throws A
     }
 
     public void start( ApplicationConfiguration config ) throws ApplicationException {
-        log.debug( "initializing application kernel: {} with config boot: {}", name, config.boot );
+
+        log.debug( "initializing application kernel {} with config boot {}, services {}", name, config.boot, config.services.keySet() );
 
         this.profiles.addAll( config.getProfiles() );
 
@@ -170,6 +171,7 @@ public void start( ApplicationConfiguration config ) throws ApplicationException
             }
             this.modules.add( new ModuleWithLocation( module, moduleConfiguration ) );
         }
+
         log.debug( "init modules: {} with profiles: {}, from main: {}", Sets.map( this.modules, m -> m.module.name ), profiles, config.boot.main );
 
         checkForUnknownServices( config.services );