FLUME-3415 - Provide Docker image for Flume

.github/workflows/build.yml

@@ -53,11 +53,23 @@ jobs:
         if: runner.os == 'macOS'
         run: env | grep '^JAVA'
 
-      - name: Build with Maven
-        timeout-minutes: 120
+      - name: Build with Maven (Linux)
+        if: runner.os == 'Linux'
+        timeout-minutes: 90
         shell: bash
         run: |
           ./mvnw clean verify -DredirectTestOutput=true \
           --show-version --batch-mode --errors --no-transfer-progress \
           -DtrimStackTrace=false \
           -Dsurefire.rerunFailingTestsCount=2
+
+      - name: Build with Maven (MacOS)
+        if: runner.os == 'macOS'
+        timeout-minutes: 120
+        shell: bash
+        run: |
+          ./mvnw clean verify -DredirectTestOutput=true \
+          --show-version --batch-mode --errors --no-transfer-progress \
+          -DtrimStackTrace=false \
+          -Dsurefire.rerunFailingTestsCount=2 \
+          -P skipDocker

conf/log4j2.xml

@@ -18,10 +18,10 @@
 -->
 <Configuration status="ERROR">
   <Properties>
-    <Property name="LOG_DIR">./logs</Property>
+    <Property name="LOG_DIR">.</Property>
   </Properties>
   <Appenders>
-    <Console name="Console" target="SYSTEM_OUT">
+    <Console name="Console" target="SYSTEM_ERR">
       <PatternLayout pattern="%d (%t) [%p - %l] %m%n" />
     </Console>
     <RollingFile name="LogFile" fileName="${LOG_DIR}/flume.log" filePattern="${LOG_DIR}/archive/flume.log.%d{yyyyMMdd}-%i">
@@ -50,7 +50,8 @@
     <Logger name="org.apache.hadoop" level="INFO"/>
     <Logger name="org.apache.hadoop.hive" level="ERROR"/>
     <Root level="INFO">
+      <AppenderRef ref="Console" />
       <AppenderRef ref="LogFile" />
     </Root>
   </Loggers>
-</Configuration>
+</Configuration>

flume-ng-dist/Dockerfile

@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+ARG IMAGE_NAME=adoptopenjdk/openjdk8
+ARG IMAGE_TAG=centos-jre
+
+FROM ${IMAGE_NAME}:${IMAGE_TAG}
+
+ARG ASSEMBLY_VERSION
+ARG MAINTAINER="Apache Flume <[email protected]>"
+
+LABEL maintainer="${MAINTAINER}"
+LABEL site="https://flume.apache.org"
+
+COPY target/apache-flume-${ASSEMBLY_VERSION}-bin/apache-flume-${ASSEMBLY_VERSION}-bin/ /etc/flume/
+
+RUN echo $ASSEMBLY_VERSION
+
+RUN groupadd --system flume && useradd --system --shell /bin/false -g flume flume
+
+WORKDIR /etc/flume
+RUN chown -R flume:flume /etc/flume
+USER flume
+ENTRYPOINT ["/etc/flume/bin/flume-ng"]

flume-ng-dist/pom.xml

@@ -83,6 +83,29 @@
           <failIfNoFiles>false</failIfNoFiles><!-- usually, no file to do checksum: don't consider error -->
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>com.spotify</groupId>
+        <artifactId>dockerfile-maven-plugin</artifactId>
+        <version>${dockerfile.version}</version>
+        <!-- Wire up to the default build phases -->
+        <executions>
+          <execution>
+            <id>default</id>
+            <goals>
+              <goal>build</goal>
+              <goal>push</goal>
+            </goals>
+          </execution>
+        </executions>
+        <configuration>
+          <repository>apache/flume</repository>
+          <tag>${project.version}</tag>
+          <buildArgs>
+            <ASSEMBLY_VERSION>${project.version}</ASSEMBLY_VERSION>
+          </buildArgs>
+          <skip>${docker.skip}</skip>
+        </configuration>
+      </plugin>
     </plugins>
   </build>
 

flume-ng-node/src/main/java/org/apache/flume/node/Application.java

@@ -274,7 +274,7 @@ public static void main(String[] args) {
       option.setRequired(false);
       options.addOption(option);
 
-      option = new Option("u", "conf-uri", true,
+      option = new Option("cl", "conf-uri", true,
               "specify a config uri (required if -c, -f and -z are missing)");
       option.setRequired(false);
       options.addOption(option);
@@ -284,16 +284,16 @@ public static void main(String[] args) {
       option.setRequired(false);
       options.addOption(option);
 
-      option = new Option("c", "conf-provider", true,
+      option = new Option("cd", "conf-provider", true,
               "specify a configuration provider class (required if -f, -u, and -z are missing)");
       option.setRequired(false);
       options.addOption(option);
 
-      option = new Option("n", "conf-user", true, "user name to access configuration uri");
+      option = new Option("cu", "conf-user", true, "user name to access configuration uri");
       option.setRequired(false);
       options.addOption(option);
 
-      option = new Option("p", "conf-password", true, "password to access configuration uri");
+      option = new Option("cp", "conf-password", true, "password to access configuration uri");
       option.setRequired(false);
       options.addOption(option);
 

flume-ng-tests/pom.xml

@@ -40,6 +40,17 @@
       <groupId>org.apache.flume</groupId>
       <artifactId>flume-ng-configuration</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <!-- only used for tests -->
+      <optional>true</optional>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.logging.log4j</groupId>
       <artifactId>log4j-1.2-api</artifactId>

flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestFileChannelDocker.java

@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.flume.test.agent;
+
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import org.apache.flume.test.util.DockerInstall;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.ArrayList;
+
+import java.util.List;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+public class TestFileChannelDocker {
+
+  private static final Logger LOGGER = LoggerFactory.getLogger(TestFileChannelDocker.class);
+
+  private static final List<File> tempResources = new ArrayList<File>();
+
+  private Properties agentProps;
+  private File sinkOutputDir;
+
+  @Before
+  public void setUp() throws Exception {
+    /* Create 3 temp dirs, each used as value within agentProps */
+
+    final File sinkOutputDir = Files.createTempDir();
+    sinkOutputDir.setWritable(true, false);
+    sinkOutputDir.setReadable(true, false);
+    tempResources.add(sinkOutputDir.getCanonicalFile());
+    final String sinkOutputDirPath = sinkOutputDir.getCanonicalPath();
+    LOGGER.info("Created rolling file sink's output dir: " + sinkOutputDirPath);
+
+    final File channelCheckpointDir = Files.createTempDir();
+    channelCheckpointDir.setWritable(true, false);
+    channelCheckpointDir.setReadable(true, false);
+    tempResources.add(channelCheckpointDir.getCanonicalFile());
+    final String channelCheckpointDirPath = channelCheckpointDir.getCanonicalPath();
+    LOGGER.info("Created file channel's checkpoint dir: " + channelCheckpointDirPath);
+
+    final File channelDataDir = Files.createTempDir();
+    channelDataDir.setWritable(true, false);
+    channelDataDir.setReadable(true, false);
+    tempResources.add(channelDataDir.getCanonicalFile());
+    final String channelDataDirPath = channelDataDir.getCanonicalPath();
+    LOGGER.info("Created file channel's data dir: " + channelDataDirPath);
+
+    /* Build props to pass to flume agent */
+
+    Properties agentProps = new Properties();
+
+    // Active sets
+    agentProps.put("a1.channels", "c1");
+    agentProps.put("a1.sources", "r1");
+    agentProps.put("a1.sinks", "k1");
+
+    // c1
+    agentProps.put("a1.channels.c1.type", "FILE");
+    agentProps.put("a1.channels.c1.checkpointDir", channelCheckpointDirPath);
+    agentProps.put("a1.channels.c1.dataDirs", channelDataDirPath);
+
+    // r1
+    agentProps.put("a1.sources.r1.channels", "c1");
+    agentProps.put("a1.sources.r1.type", "EXEC");
+    agentProps.put("a1.sources.r1.command", "seq 1 100");
+
+    // k1
+    agentProps.put("a1.sinks.k1.channel", "c1");
+    agentProps.put("a1.sinks.k1.type", "FILE_ROLL");
+    agentProps.put("a1.sinks.k1.sink.directory", sinkOutputDirPath);
+    agentProps.put("a1.sinks.k1.sink.rollInterval", "0");
+
+    this.agentProps = agentProps;
+    this.sinkOutputDir = sinkOutputDir;
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    DockerInstall.getInstance().stopAgent();
+    for (File tempResource : tempResources) {
+      tempResource.delete();
+    }
+    agentProps = null;
+  }
+
+  /**
+   * File channel in/out test. Verifies that all events inserted into the
+   * file channel are received by the sink in order.
+   * <p>
+   * The EXEC source creates 100 events where the event bodies have
+   * sequential numbers. The source puts those events into the file channel,
+   * and the FILE_ROLL The sink is expected to take all 100 events in FIFO
+   * order.
+   *
+   * @throws Exception
+   */
+  @Test
+  public void testInOut() throws Exception {
+    LOGGER.debug("testInOut() started.");
+
+
+    DockerInstall.getInstance().startAgent("a1", agentProps, tempResources);
+    TimeUnit.SECONDS.sleep(10); // Wait for source and sink to finish
+    // TODO make this more deterministic
+
+    /* Create expected output */
+    StringBuffer sb = new StringBuffer();
+    for (int i = 1; i <= 100; i++) {
+      sb.append(i).append("\n");
+    }
+    String expectedOutput = sb.toString();
+    LOGGER.info("Created expected output: " + expectedOutput);
+
+    /* Create actual output file */
+
+    File[] sinkOutputDirChildren = sinkOutputDir.listFiles();
+    // Only 1 file should be in FILE_ROLL sink's dir (rolling is disabled)
+    Assert.assertEquals("Expected FILE_ROLL sink's dir to have only 1 child," +
+                        " but found " + sinkOutputDirChildren.length + " children.",
+                        1, sinkOutputDirChildren.length);
+    File actualOutput = sinkOutputDirChildren[0];
+
+    if (!Files.toString(actualOutput, Charsets.UTF_8).equals(expectedOutput)) {
+      LOGGER.error("Actual output doesn't match expected output.\n");
+      throw new AssertionError("FILE_ROLL sink's actual output doesn't " +
+                               "match expected output.");
+    }
+
+    LOGGER.debug("testInOut() ended.");
+  }
+}