From b61c4c5040648ccebf2765ba2a81b417ccac96fd Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Sat, 11 Dec 2021 19:07:59 +0000 Subject: [PATCH 01/20] Draft docker --- Dockerfile | 33 ++++++++++++++++++++ flume-ng-sources/flume-scribe-source/pom.xml | 4 +-- 2 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000..7d7390f734 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +ARG IMAGE_NAME=adoptopenjdk/openjdk8 +ARG IMAGE_TAG=centos-jre +FROM ${IMAGE_NAME}:${IMAGE_TAG} +ARG MAINTAINER="Apache Flume " +LABEL maintainer="${MAINTAINER}" +LABEL site="http://flume.apache.org" + +COPY flume-ng-dist/target/apache-flume-1.10.0-SNAPSHOT-bin/apache-flume-1.10.0-SNAPSHOT-bin/ /etc/flume/ + +RUN groupadd --system flume && useradd --system --shell /bin/false -g flume flume + +WORKDIR /etc/flume +RUN chown -R flume:flume /etc/flume +USER flume +ENTRYPOINT ["/etc/flume/bin/flume-ng"] diff --git a/flume-ng-sources/flume-scribe-source/pom.xml b/flume-ng-sources/flume-scribe-source/pom.xml index 5e9f94705d..e19ce818ee 100644 --- a/flume-ng-sources/flume-scribe-source/pom.xml +++ b/flume-ng-sources/flume-scribe-source/pom.xml @@ -114,8 +114,8 @@ limitations under the License. maven-compiler-plugin ${mvn-compiler-plugin.version} - 1.6 - 1.6 + 1.7 + 1.7 **/generated-sources/** From 31d99d3cd1a8ad531c2a16cb4a63d1a25d5a01db Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Wed, 2 Feb 2022 17:33:28 +0000 Subject: [PATCH 02/20] Add docker build to flume-ng-dist as part of package phase --- conf/log4j2.xml | 7 ++++--- Dockerfile => flume-ng-dist/Dockerfile | 10 ++++++++-- flume-ng-dist/pom.xml | 22 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 5 deletions(-) rename Dockerfile => flume-ng-dist/Dockerfile (85%) diff --git a/conf/log4j2.xml b/conf/log4j2.xml index d3efac4d23..7a45f044f2 100644 --- a/conf/log4j2.xml +++ b/conf/log4j2.xml @@ -18,10 +18,10 @@ --> - ./logs + . - + @@ -50,7 +50,8 @@ + - \ No newline at end of file + diff --git a/Dockerfile b/flume-ng-dist/Dockerfile similarity index 85% rename from Dockerfile rename to flume-ng-dist/Dockerfile index 7d7390f734..62d41fe340 100644 --- a/Dockerfile +++ b/flume-ng-dist/Dockerfile @@ -18,12 +18,18 @@ ARG IMAGE_NAME=adoptopenjdk/openjdk8 ARG IMAGE_TAG=centos-jre + FROM ${IMAGE_NAME}:${IMAGE_TAG} + +ARG ASSEMBLY_VERSION ARG MAINTAINER="Apache Flume " + LABEL maintainer="${MAINTAINER}" -LABEL site="http://flume.apache.org" +LABEL site="https://flume.apache.org" + +COPY target/apache-flume-${ASSEMBLY_VERSION}-bin/apache-flume-${ASSEMBLY_VERSION}-bin/ /etc/flume/ -COPY flume-ng-dist/target/apache-flume-1.10.0-SNAPSHOT-bin/apache-flume-1.10.0-SNAPSHOT-bin/ /etc/flume/ +RUN echo $ASSEMBLY_VERSION RUN groupadd --system flume && useradd --system --shell /bin/false -g flume flume diff --git a/flume-ng-dist/pom.xml b/flume-ng-dist/pom.xml index 1c808dd971..865d96dfdc 100644 --- a/flume-ng-dist/pom.xml +++ b/flume-ng-dist/pom.xml @@ -83,6 +83,28 @@ false + + com.spotify + dockerfile-maven-plugin + ${dockerfile.version} + + + + default + + build + push + + + + + apache/flume + ${project.version} + + ${project.version} + + + From 39042bd48047ad10038c0c13fb130c4455be8e8f Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Mon, 7 Feb 2022 13:18:54 +0000 Subject: [PATCH 03/20] De-dupe command line arguments --- .../src/main/java/org/apache/flume/node/Application.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flume-ng-node/src/main/java/org/apache/flume/node/Application.java b/flume-ng-node/src/main/java/org/apache/flume/node/Application.java index 1f4df59aac..0ae0316bbb 100644 --- a/flume-ng-node/src/main/java/org/apache/flume/node/Application.java +++ b/flume-ng-node/src/main/java/org/apache/flume/node/Application.java @@ -274,7 +274,7 @@ public static void main(String[] args) { option.setRequired(false); options.addOption(option); - option = new Option("u", "conf-uri", true, + option = new Option("cl", "conf-uri", true, "specify a config uri (required if -c, -f and -z are missing)"); option.setRequired(false); options.addOption(option); @@ -284,16 +284,16 @@ public static void main(String[] args) { option.setRequired(false); options.addOption(option); - option = new Option("c", "conf-provider", true, + option = new Option("cd", "conf-provider", true, "specify a configuration provider class (required if -f, -u, and -z are missing)"); option.setRequired(false); options.addOption(option); - option = new Option("n", "conf-user", true, "user name to access configuration uri"); + option = new Option("cu", "conf-user", true, "user name to access configuration uri"); option.setRequired(false); options.addOption(option); - option = new Option("p", "conf-password", true, "password to access configuration uri"); + option = new Option("cp", "conf-password", true, "password to access configuration uri"); option.setRequired(false); options.addOption(option); From fdc31cfddd8c01894bc2127ab8d3c41ee24ce60e Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Mon, 7 Feb 2022 13:19:36 +0000 Subject: [PATCH 04/20] Re-add in dockerfile.version property --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index d0089c5df7..0ec3375232 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,7 @@ limitations under the License. 1.9 5.1.0 10.14.1.0 + 1.4.13 4.1.18 0.90.1 2.13.1 From d74905002d107497d63c94f38e080a7bbd8da1ef Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Tue, 8 Feb 2022 17:12:30 +0000 Subject: [PATCH 05/20] Re-add in dockerfile.version property --- .../test/agent/TestFileChannelDocker.java | 149 ++++++++++++ .../apache/flume/test/util/DockerInstall.java | 219 ++++++++++++++++++ .../apache/flume/test/util/StagedInstall.java | 116 +++++----- 3 files changed, 428 insertions(+), 56 deletions(-) create mode 100644 flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestFileChannelDocker.java create mode 100644 flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java diff --git a/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestFileChannelDocker.java b/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestFileChannelDocker.java new file mode 100644 index 0000000000..3c370ac2cd --- /dev/null +++ b/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestFileChannelDocker.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.flume.test.agent; + +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import org.apache.flume.test.util.DockerInstall; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; + +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +public class TestFileChannelDocker { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestFileChannelDocker.class); + + private static final List tempResources = new ArrayList(); + + private Properties agentProps; + private File sinkOutputDir; + + @Before + public void setUp() throws Exception { + /* Create 3 temp dirs, each used as value within agentProps */ + + final File sinkOutputDir = Files.createTempDir(); + tempResources.add(sinkOutputDir.getCanonicalFile()); + final String sinkOutputDirPath = sinkOutputDir.getCanonicalPath(); + LOGGER.info("Created rolling file sink's output dir: " + sinkOutputDirPath); + + final File channelCheckpointDir = Files.createTempDir(); + tempResources.add(channelCheckpointDir.getCanonicalFile()); + final String channelCheckpointDirPath = channelCheckpointDir.getCanonicalPath(); + LOGGER.info("Created file channel's checkpoint dir: " + channelCheckpointDirPath); + + final File channelDataDir = Files.createTempDir(); + tempResources.add(channelDataDir.getCanonicalFile()); + final String channelDataDirPath = channelDataDir.getCanonicalPath(); + LOGGER.info("Created file channel's data dir: " + channelDataDirPath); + + /* Build props to pass to flume agent */ + + Properties agentProps = new Properties(); + + // Active sets + agentProps.put("a1.channels", "c1"); + agentProps.put("a1.sources", "r1"); + agentProps.put("a1.sinks", "k1"); + + // c1 + agentProps.put("a1.channels.c1.type", "FILE"); + agentProps.put("a1.channels.c1.checkpointDir", channelCheckpointDirPath); + agentProps.put("a1.channels.c1.dataDirs", channelDataDirPath); + + // r1 + agentProps.put("a1.sources.r1.channels", "c1"); + agentProps.put("a1.sources.r1.type", "EXEC"); + agentProps.put("a1.sources.r1.command", "seq 1 100"); + + // k1 + agentProps.put("a1.sinks.k1.channel", "c1"); + agentProps.put("a1.sinks.k1.type", "FILE_ROLL"); + agentProps.put("a1.sinks.k1.sink.directory", sinkOutputDirPath); + agentProps.put("a1.sinks.k1.sink.rollInterval", "0"); + + this.agentProps = agentProps; + this.sinkOutputDir = sinkOutputDir; + } + + @After + public void tearDown() throws Exception { + DockerInstall.getInstance().stopAgent(); + for (File tempResource : tempResources) { + tempResource.delete(); + } + agentProps = null; + } + + /** + * File channel in/out test. Verifies that all events inserted into the + * file channel are received by the sink in order. + *

+ * The EXEC source creates 100 events where the event bodies have + * sequential numbers. The source puts those events into the file channel, + * and the FILE_ROLL The sink is expected to take all 100 events in FIFO + * order. + * + * @throws Exception + */ + @Test + public void testInOut() throws Exception { + LOGGER.debug("testInOut() started."); + + + DockerInstall.getInstance().startAgent("a1", agentProps, tempResources); + TimeUnit.SECONDS.sleep(10); // Wait for source and sink to finish + // TODO make this more deterministic + + /* Create expected output */ + StringBuffer sb = new StringBuffer(); + for (int i = 1; i <= 100; i++) { + sb.append(i).append("\n"); + } + String expectedOutput = sb.toString(); + LOGGER.info("Created expected output: " + expectedOutput); + + /* Create actual output file */ + + File[] sinkOutputDirChildren = sinkOutputDir.listFiles(); + // Only 1 file should be in FILE_ROLL sink's dir (rolling is disabled) + Assert.assertEquals("Expected FILE_ROLL sink's dir to have only 1 child," + + " but found " + sinkOutputDirChildren.length + " children.", + 1, sinkOutputDirChildren.length); + File actualOutput = sinkOutputDirChildren[0]; + + if (!Files.toString(actualOutput, Charsets.UTF_8).equals(expectedOutput)) { + LOGGER.error("Actual output doesn't match expected output.\n"); + throw new AssertionError("FILE_ROLL sink's actual output doesn't " + + "match expected output."); + } + + LOGGER.debug("testInOut() ended."); + } +} diff --git a/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java b/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java new file mode 100644 index 0000000000..874c206024 --- /dev/null +++ b/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.flume.test.util; + +import com.google.common.base.Joiner; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; + +import java.nio.charset.StandardCharsets; +import java.util.*; + +/** + * Attempts to setup a staged install using explicitly specified tar-ball + * distribution or by using relative path into the flume-ng-dist module. + */ +public class DockerInstall extends StagedInstall { + + private static final Logger LOGGER = LoggerFactory.getLogger(DockerInstall.class); + + private String configFilePath; + + private ProcessShutdownHook shutdownHook; + private ProcessInputStreamConsumer consumer; + private String containerId; + + private static DockerInstall INSTANCE; + private String dockerImageId; + + public static synchronized DockerInstall getInstance() throws Exception { + if (INSTANCE == null) { + INSTANCE = new DockerInstall(); + } + return INSTANCE; + } + + public synchronized boolean isRunning() { + return containerId != null; + } + + public synchronized void stopAgent() throws Exception { + if (containerId == null) { + throw new Exception("Process not found"); + } + + LOGGER.info("Shutting down agent process"); + + ImmutableList.Builder builder = new ImmutableList.Builder<>(); + builder.add("/bin/sh"); + builder.add("-c"); + builder.add("'docker kill " + containerId + "'"); + + List cmdArgs = builder.build(); + + File tempShellFileKiller = File.createTempFile("docker", ".sh"); + tempShellFileKiller.setExecutable(true); + tempShellFileKiller.deleteOnExit(); + Files.write(Joiner.on(" ").join(cmdArgs).getBytes(StandardCharsets.UTF_8), tempShellFileKiller); + + ProcessBuilder processKiller = new ProcessBuilder(tempShellFileKiller.getAbsolutePath()); + + Process killer = processKiller.start(); + killer.waitFor(); + + containerId = null; + consumer.interrupt(); + consumer = null; + configFilePath = null; + Runtime.getRuntime().removeShutdownHook(shutdownHook); + shutdownHook = null; + + Thread.sleep(3000); // sleep for 3s to let system shutdown + } + + public synchronized void startAgent(String name, Properties properties) throws Exception { + startAgent(name, properties, new HashMap<>(), new HashMap<>(), new ArrayList<>()); + } + + public synchronized void startAgent(String name, Properties properties, List mountPoints) throws Exception { + startAgent(name, properties, new HashMap<>(), new HashMap<>(), mountPoints); + } + + public synchronized void startAgent( + String name, Properties properties, Map environmentVariables, + Map commandOptions, List mountPoints) + throws Exception { + Preconditions.checkArgument(!name.isEmpty(), "agent name must not be empty"); + Preconditions.checkNotNull(properties, "properties object must not be null"); + + // State per invocation - config file, process, shutdown hook + String agentName = name; + + if (containerId != null) { + throw new Exception("A process is already running"); + } + LOGGER.info("Starting process for agent: " + agentName + " using config: " + properties); + + File configFile = createConfigurationFile(agentName, properties); + configFilePath = configFile.getCanonicalPath(); + + String configFileName = configFile.getName(); + String logFileName = "flume-" + agentName + "-" + + configFileName.substring(0, configFileName.indexOf('.')) + ".log"; + + LOGGER.info("Created configuration file: " + configFilePath); + + ImmutableList.Builder builder = new ImmutableList.Builder<>(); + builder.add("/bin/sh"); + builder.add("-c"); + + + StringBuilder sb = new StringBuilder("'"); + sb.append("docker run "); + sb.append(" --detach"); + sb.append(" -v ").append(confDirPath).append(":").append(confDirPath); + sb.append(" -v ").append(getStageDir().toString()).append(":").append(getStageDir().toString()); + sb.append(" -v ").append(logDirPath).append(":").append(logDirPath); + + mountPoints.forEach(file -> sb.append(" -v ").append(file.toString()).append(":").append(file.toString())); + + sb.append(" -t ").append(dockerImageId).append(" agent"); + sb.append(" --conf ").append(confDirPath); + sb.append(" --conf-file ").append(configFilePath); + sb.append(" --name ").append(agentName); + sb.append(" -D").append(ENV_FLUME_LOG_DIR).append( "=").append(logDirPath); + sb.append(" -D" ).append( ENV_FLUME_ROOT_LOGGER ).append( "=" ).append( ENV_FLUME_ROOT_LOGGER_VALUE); + sb.append(" -D" ).append( ENV_FLUME_LOG_FILE ).append( "=").append(logFileName); + sb.append("'"); + + builder.add(sb.toString()); + + commandOptions.forEach((key, value) -> builder.add(key, value)); + + List cmdArgs = builder.build(); + + File tempShellFile = File.createTempFile("docker", ".sh"); + tempShellFile.setExecutable(true); + tempShellFile.deleteOnExit(); + Files.write(Joiner.on(" ").join(cmdArgs).getBytes(StandardCharsets.UTF_8), tempShellFile); + + ProcessBuilder pb = new ProcessBuilder(tempShellFile.getAbsolutePath()); + + Map env = pb.environment(); + env.putAll(environmentVariables); + + pb.directory(baseDir); + + Process process = pb.start(); + + BufferedReader reader = + new BufferedReader(new InputStreamReader(process.getInputStream())); + StringBuilder containerIdSb = new StringBuilder(); + String line; + while ( (line = reader.readLine()) != null) { + containerIdSb.append(line); + } + + containerId = containerIdSb.toString(); + + ImmutableList.Builder logBuilder = new ImmutableList.Builder(); + logBuilder.add("/bin/sh"); + logBuilder.add("-c"); + logBuilder.add("'docker logs --follow " + containerId + "'"); + + List logCmdArgs = logBuilder.build(); + + File tempLogShellFile = File.createTempFile("docker", ".sh"); + tempLogShellFile.setExecutable(true); + tempLogShellFile.deleteOnExit(); + Files.write(Joiner.on(" ").join(logCmdArgs).getBytes(StandardCharsets.UTF_8), tempLogShellFile); + + + ProcessBuilder logReaderPb = new ProcessBuilder(tempLogShellFile.getAbsolutePath()); + Process logReaderProc = logReaderPb.start(); + + consumer = new ProcessInputStreamConsumer(logReaderProc.getInputStream()); + consumer.start(); + + shutdownHook = new ProcessShutdownHook(); + Runtime.getRuntime().addShutdownHook(shutdownHook); + + Thread.sleep(3000); // sleep for 3s to let system initialize + } + + + + private DockerInstall() throws Exception { + + super(); + dockerImageId = getDockerImageId(); + } + + + private static String getDockerImageId() throws Exception { + File dockerImageIdFile = new File("../flume-ng-dist/target/docker/image-id"); + return Files.readFirstLine(dockerImageIdFile, StandardCharsets.UTF_8); + } + +} diff --git a/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java b/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java index b75f7a809c..01ff253b85 100644 --- a/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java +++ b/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java @@ -58,11 +58,11 @@ public class StagedInstall { public static final String ENV_FLUME_ROOT_LOGGER_VALUE = "DEBUG,LOGFILE"; public static final String ENV_FLUME_LOG_FILE = "flume.log.file"; - private final File stageDir; - private final File baseDir; - private final String launchScriptPath; - private final String confDirPath; - private final String logDirPath; + final File stageDir; + final File baseDir; + final String launchScriptPath; + final String confDirPath; + final String logDirPath; // State per invocation - config file, process, shutdown hook private String agentName; @@ -196,7 +196,7 @@ public synchronized File getStageDir() { return stageDir; } - private File createConfigurationFile(String agentName, Properties properties) + File createConfigurationFile(String agentName, Properties properties) throws Exception { Preconditions.checkNotNull(properties, "properties object must not be null"); @@ -222,70 +222,74 @@ private File createConfigurationFile(String agentName, Properties properties) return file; } - private StagedInstall() throws Exception { + StagedInstall() { + try { + String tarballPath = System.getProperty(PROP_PATH_TO_DIST_TARBALL); + if (tarballPath == null || tarballPath.trim().length() == 0) { + LOGGER.info("No value specified for system property: " + + PROP_PATH_TO_DIST_TARBALL + + ". Will attempt to use relative path to locate dist tarball."); - String tarballPath = System.getProperty(PROP_PATH_TO_DIST_TARBALL); - if (tarballPath == null || tarballPath.trim().length() == 0) { - LOGGER.info("No value specified for system property: " - + PROP_PATH_TO_DIST_TARBALL - + ". Will attempt to use relative path to locate dist tarball."); + tarballPath = getRelativeTarballPath(); + } - tarballPath = getRelativeTarballPath(); - } + if (tarballPath == null || tarballPath.trim().length() == 0) { + throw new RuntimeException("Failed to locate tar-ball distribution. " + + "Please specify explicitly via system property: " + + PROP_PATH_TO_DIST_TARBALL); + } - if (tarballPath == null || tarballPath.trim().length() == 0) { - throw new Exception("Failed to locate tar-ball distribution. " - + "Please specify explicitly via system property: " - + PROP_PATH_TO_DIST_TARBALL); - } + // Validate + File tarballFile = new File(tarballPath); + if (!tarballFile.isFile() || !tarballFile.canRead()) { + throw new RuntimeException("The tarball distribution file is invalid: " + + tarballPath + ". You can override this by explicitly setting the " + + "system property: " + PROP_PATH_TO_DIST_TARBALL); + } - // Validate - File tarballFile = new File(tarballPath); - if (!tarballFile.isFile() || !tarballFile.canRead()) { - throw new Exception("The tarball distribution file is invalid: " - + tarballPath + ". You can override this by explicitly setting the " - + "system property: " + PROP_PATH_TO_DIST_TARBALL); - } + LOGGER.info("Dist tarball to use: " + tarballPath); - LOGGER.info("Dist tarball to use: " + tarballPath); - // Now set up a staging directory for this distribution - stageDir = getStagingDirectory(); + // Now set up a staging directory for this distribution + stageDir = getStagingDirectory(); - // Deflate the gzip compressed archive - File tarFile = gunzipDistTarball(tarballFile, stageDir); + // Deflate the gzip compressed archive + File tarFile = gunzipDistTarball(tarballFile, stageDir); - // Untar the deflated file - untarTarFile(tarFile, stageDir); + // Untar the deflated file + untarTarFile(tarFile, stageDir); - // Delete the tarfile - tarFile.delete(); + // Delete the tarfile + tarFile.delete(); - LOGGER.info("Dist tarball staged to: " + stageDir); + LOGGER.info("Dist tarball staged to: " + stageDir); - File rootDir = stageDir; - File[] listBaseDirs = stageDir.listFiles(); - if (listBaseDirs != null && listBaseDirs.length == 1 - && listBaseDirs[0].isDirectory()) { - rootDir = listBaseDirs[0]; - } - baseDir = rootDir; + File rootDir = stageDir; + File[] listBaseDirs = stageDir.listFiles(); + if (listBaseDirs != null && listBaseDirs.length == 1 + && listBaseDirs[0].isDirectory()) { + rootDir = listBaseDirs[0]; + } + baseDir = rootDir; - // Give execute permissions to the bin/flume-ng script - File launchScript = new File(baseDir, "bin/flume-ng"); - giveExecutePermissions(launchScript); + // Give execute permissions to the bin/flume-ng script + File launchScript = new File(baseDir, "bin/flume-ng"); + giveExecutePermissions(launchScript); - launchScriptPath = launchScript.getCanonicalPath(); + launchScriptPath = launchScript.getCanonicalPath(); - File confDir = new File(baseDir, "conf"); - confDirPath = confDir.getCanonicalPath(); + File confDir = new File(baseDir, "conf"); + confDirPath = confDir.getCanonicalPath(); - File logDir = new File(baseDir, "logs"); - logDir.mkdirs(); + File logDir = new File(baseDir, "logs"); + logDir.mkdirs(); - logDirPath = logDir.getCanonicalPath(); + logDirPath = logDir.getCanonicalPath(); - LOGGER.info("Staged install root directory: " + rootDir.getCanonicalPath()); + LOGGER.info("Staged install root directory: " + rootDir.getCanonicalPath()); + } catch (Exception e) { + throw new RuntimeException(e); + } } private void giveExecutePermissions(File file) throws Exception { @@ -501,7 +505,7 @@ public static void waitUntilPortOpens(String host, int port, long timeout) } } - private class ProcessShutdownHook extends Thread { + class ProcessShutdownHook extends Thread { public void run() { synchronized (StagedInstall.this) { if (StagedInstall.this.process != null) { @@ -511,10 +515,10 @@ public void run() { } } - private static class ProcessInputStreamConsumer extends Thread { + static class ProcessInputStreamConsumer extends Thread { private final InputStream is; - private ProcessInputStreamConsumer(InputStream is) { + ProcessInputStreamConsumer(InputStream is) { this.is = is; this.setDaemon(true); } From 1f19ec5dc35611adf11033f02db304d03cb307a3 Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Tue, 8 Feb 2022 18:52:40 +0000 Subject: [PATCH 06/20] Revert "Draft docker" This reverts commit b61c4c5040648ccebf2765ba2a81b417ccac96fd. Conflicts: flume-ng-dist/Dockerfile --- flume-ng-sources/flume-scribe-source/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flume-ng-sources/flume-scribe-source/pom.xml b/flume-ng-sources/flume-scribe-source/pom.xml index e19ce818ee..5e9f94705d 100644 --- a/flume-ng-sources/flume-scribe-source/pom.xml +++ b/flume-ng-sources/flume-scribe-source/pom.xml @@ -114,8 +114,8 @@ limitations under the License. maven-compiler-plugin ${mvn-compiler-plugin.version} - 1.7 - 1.7 + 1.6 + 1.6 **/generated-sources/** From 81773b813dbcb2da810ce3289e51d56467f7314a Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Tue, 8 Feb 2022 22:00:14 +0000 Subject: [PATCH 07/20] Add check to make sure container starter okay. --- .../test/java/org/apache/flume/test/util/DockerInstall.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java b/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java index 874c206024..b0c5cbcdc6 100644 --- a/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java +++ b/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java @@ -177,6 +177,11 @@ public synchronized void startAgent( containerId = containerIdSb.toString(); + if (process.exitValue() != 0) { + throw new RuntimeException("Docker container did not start: " + process.exitValue() + " " + containerId); + } + + ImmutableList.Builder logBuilder = new ImmutableList.Builder(); logBuilder.add("/bin/sh"); logBuilder.add("-c"); From 5e8fc3badf4e5171c294b54f27995cc37b76fef9 Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Fri, 11 Feb 2022 10:31:41 +0000 Subject: [PATCH 08/20] Checkstyle fixes --- .../apache/flume/test/util/DockerInstall.java | 19 +++++++++---------- .../apache/flume/test/util/StagedInstall.java | 1 - 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java b/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java index b0c5cbcdc6..1a4ac54361 100644 --- a/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java +++ b/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java @@ -25,10 +25,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; - +import java.io.BufferedReader; +import java.io.File; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; /** * Attempts to setup a staged install using explicitly specified tar-ball @@ -173,7 +178,7 @@ public synchronized void startAgent( String line; while ( (line = reader.readLine()) != null) { containerIdSb.append(line); - } + } containerId = containerIdSb.toString(); @@ -181,7 +186,6 @@ public synchronized void startAgent( throw new RuntimeException("Docker container did not start: " + process.exitValue() + " " + containerId); } - ImmutableList.Builder logBuilder = new ImmutableList.Builder(); logBuilder.add("/bin/sh"); logBuilder.add("-c"); @@ -194,7 +198,6 @@ public synchronized void startAgent( tempLogShellFile.deleteOnExit(); Files.write(Joiner.on(" ").join(logCmdArgs).getBytes(StandardCharsets.UTF_8), tempLogShellFile); - ProcessBuilder logReaderPb = new ProcessBuilder(tempLogShellFile.getAbsolutePath()); Process logReaderProc = logReaderPb.start(); @@ -207,15 +210,11 @@ public synchronized void startAgent( Thread.sleep(3000); // sleep for 3s to let system initialize } - - private DockerInstall() throws Exception { - super(); dockerImageId = getDockerImageId(); } - private static String getDockerImageId() throws Exception { File dockerImageIdFile = new File("../flume-ng-dist/target/docker/image-id"); return Files.readFirstLine(dockerImageIdFile, StandardCharsets.UTF_8); diff --git a/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java b/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java index 01ff253b85..737ef25615 100644 --- a/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java +++ b/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java @@ -249,7 +249,6 @@ File createConfigurationFile(String agentName, Properties properties) LOGGER.info("Dist tarball to use: " + tarballPath); - // Now set up a staging directory for this distribution stageDir = getStagingDirectory(); From ac262b23db72be418dc18c74c9d754ceb45dfe5d Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Fri, 11 Feb 2022 10:32:05 +0000 Subject: [PATCH 09/20] Docker fixes for Ubuntu --- flume-ng-tests/pom.xml | 11 +++++++++++ .../flume/test/agent/TestFileChannelDocker.java | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/flume-ng-tests/pom.xml b/flume-ng-tests/pom.xml index cd37e002c2..2aed61fafa 100644 --- a/flume-ng-tests/pom.xml +++ b/flume-ng-tests/pom.xml @@ -40,6 +40,17 @@ org.apache.flume flume-ng-configuration + + org.slf4j + slf4j-api + + true + + + org.apache.logging.log4j + log4j-slf4j-impl + test + org.apache.logging.log4j log4j-1.2-api diff --git a/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestFileChannelDocker.java b/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestFileChannelDocker.java index 3c370ac2cd..29a382d85b 100644 --- a/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestFileChannelDocker.java +++ b/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestFileChannelDocker.java @@ -49,16 +49,22 @@ public void setUp() throws Exception { /* Create 3 temp dirs, each used as value within agentProps */ final File sinkOutputDir = Files.createTempDir(); + sinkOutputDir.setWritable(true, false); + sinkOutputDir.setReadable(true, false); tempResources.add(sinkOutputDir.getCanonicalFile()); final String sinkOutputDirPath = sinkOutputDir.getCanonicalPath(); LOGGER.info("Created rolling file sink's output dir: " + sinkOutputDirPath); final File channelCheckpointDir = Files.createTempDir(); + channelCheckpointDir.setWritable(true, false); + channelCheckpointDir.setReadable(true, false); tempResources.add(channelCheckpointDir.getCanonicalFile()); final String channelCheckpointDirPath = channelCheckpointDir.getCanonicalPath(); LOGGER.info("Created file channel's checkpoint dir: " + channelCheckpointDirPath); final File channelDataDir = Files.createTempDir(); + channelDataDir.setWritable(true, false); + channelDataDir.setReadable(true, false); tempResources.add(channelDataDir.getCanonicalFile()); final String channelDataDirPath = channelDataDir.getCanonicalPath(); LOGGER.info("Created file channel's data dir: " + channelDataDirPath); From 87dc92a229cdf2147da58e7acba407cce281f057 Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Fri, 11 Feb 2022 18:40:53 +0000 Subject: [PATCH 10/20] Setup docker on Macos --- .github/workflows/build.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fec17a151b..ca142b6984 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,6 +29,11 @@ jobs: strategy: matrix: os: [ ubuntu-latest, macos-latest ] + include: + # macOS + - os: macos-latest + docker_channel: stable + docker_version: "20.10" steps: @@ -45,6 +50,13 @@ jobs: architecture: x64 cache: maven + - name: Setup Docker + uses: docker-practice/actions-setup-docker@master + run: | + set -x + + docker version + - name: Inspect environment (Linux) if: runner.os == 'Linux' run: env | grep '^JAVA' From 5f8f98120ef35950ce31f270343ce1ec9f17e757 Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Fri, 11 Feb 2022 18:42:44 +0000 Subject: [PATCH 11/20] Update build --- .github/workflows/build.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca142b6984..59db1cfba5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,10 +52,7 @@ jobs: - name: Setup Docker uses: docker-practice/actions-setup-docker@master - run: | - set -x - - docker version + run: docker version - name: Inspect environment (Linux) if: runner.os == 'Linux' From 877337ad8849872f24b5696c308f0a94ac5598fc Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Fri, 11 Feb 2022 18:43:28 +0000 Subject: [PATCH 12/20] Update build --- .github/workflows/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59db1cfba5..b3be185a25 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,6 @@ jobs: - name: Setup Docker uses: docker-practice/actions-setup-docker@master - run: docker version - name: Inspect environment (Linux) if: runner.os == 'Linux' From caa27459dbec1e560a92c8f0a99f1e963178d582 Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Fri, 11 Feb 2022 18:46:40 +0000 Subject: [PATCH 13/20] Remove macos --- .github/workflows/build.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b3be185a25..9d0cf1434c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,12 +28,7 @@ jobs: strategy: matrix: - os: [ ubuntu-latest, macos-latest ] - include: - # macOS - - os: macos-latest - docker_channel: stable - docker_version: "20.10" + os: [ ubuntu-latest ] steps: @@ -50,9 +45,6 @@ jobs: architecture: x64 cache: maven - - name: Setup Docker - uses: docker-practice/actions-setup-docker@master - - name: Inspect environment (Linux) if: runner.os == 'Linux' run: env | grep '^JAVA' From ccbbda756f0dd4755583990dfc109f7880dec59c Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Fri, 11 Feb 2022 19:37:34 +0000 Subject: [PATCH 14/20] Add profile to skip docker on Macos builds if required --- .github/workflows/build.yml | 14 +++++++++++++- flume-ng-dist/pom.xml | 1 + pom.xml | 9 +++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d0cf1434c..b406fb9e26 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,7 @@ jobs: strategy: matrix: - os: [ ubuntu-latest ] + os: [ ubuntu-latest, macos-latest ] steps: @@ -54,6 +54,17 @@ jobs: run: env | grep '^JAVA' - name: Build with Maven + if: runner.os == 'Linux' + timeout-minutes: 90 + shell: bash + run: | + ./mvnw clean verify -DredirectTestOutput=true \ + --show-version --batch-mode --errors --no-transfer-progress \ + -DtrimStackTrace=false \ + -Dsurefire.rerunFailingTestsCount=2 + + - name: Build with Maven + if: runner.os == 'macOS' timeout-minutes: 90 shell: bash run: | @@ -61,3 +72,4 @@ jobs: --show-version --batch-mode --errors --no-transfer-progress \ -DtrimStackTrace=false \ -Dsurefire.rerunFailingTestsCount=2 + -P skipDocker diff --git a/flume-ng-dist/pom.xml b/flume-ng-dist/pom.xml index 865d96dfdc..cccb815fff 100644 --- a/flume-ng-dist/pom.xml +++ b/flume-ng-dist/pom.xml @@ -103,6 +103,7 @@ ${project.version} + ${docker.skip} diff --git a/pom.xml b/pom.xml index 0ec3375232..b48ee07371 100644 --- a/pom.xml +++ b/pom.xml @@ -62,6 +62,7 @@ limitations under the License. 5.1.0 10.14.1.0 1.4.13 + false 4.1.18 0.90.1 2.13.1 @@ -244,6 +245,14 @@ limitations under the License. + + + skipDocker + + true + **/*Docker.java + + 2009 From d733b3f817716f9f536ad23b0c9c97e8c58b0a16 Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Fri, 11 Feb 2022 22:07:26 +0000 Subject: [PATCH 15/20] Add profile to skip docker on Macos builds if required --- .github/workflows/build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6746deb9e3..2ec67f1081 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,8 +53,7 @@ jobs: if: runner.os == 'macOS' run: env | grep '^JAVA' - - name: Build with Maven - + - name: Build with Maven (Linux) if: runner.os == 'Linux' timeout-minutes: 90 shell: bash @@ -64,7 +63,7 @@ jobs: -DtrimStackTrace=false \ -Dsurefire.rerunFailingTestsCount=2 - - name: Build with Maven + - name: Build with Maven (MacOS) if: runner.os == 'macOS' timeout-minutes: 120 shell: bash @@ -72,5 +71,5 @@ jobs: ./mvnw clean verify -DredirectTestOutput=true \ --show-version --batch-mode --errors --no-transfer-progress \ -DtrimStackTrace=false \ - -Dsurefire.rerunFailingTestsCount=2 + -Dsurefire.rerunFailingTestsCount=2 \ -P skipDocker From d2bd7812dbacd86459726c0fd3dc774272ce0222 Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Sat, 12 Feb 2022 09:29:46 +0000 Subject: [PATCH 16/20] Check if process is still running --- .../src/test/java/org/apache/flume/test/util/DockerInstall.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java b/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java index 1a4ac54361..715bf8170a 100644 --- a/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java +++ b/flume-ng-tests/src/test/java/org/apache/flume/test/util/DockerInstall.java @@ -182,7 +182,7 @@ public synchronized void startAgent( containerId = containerIdSb.toString(); - if (process.exitValue() != 0) { + if (!process.isAlive()) { throw new RuntimeException("Docker container did not start: " + process.exitValue() + " " + containerId); } From fdb3a76c5b186377805dc71ea5612c32c246424b Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Tue, 8 Mar 2022 16:10:16 +0000 Subject: [PATCH 17/20] Switch to use label for maintainer --- flume-ng-dist/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flume-ng-dist/Dockerfile b/flume-ng-dist/Dockerfile index 62d41fe340..a96d799648 100644 --- a/flume-ng-dist/Dockerfile +++ b/flume-ng-dist/Dockerfile @@ -24,7 +24,7 @@ FROM ${IMAGE_NAME}:${IMAGE_TAG} ARG ASSEMBLY_VERSION ARG MAINTAINER="Apache Flume " -LABEL maintainer="${MAINTAINER}" +LABEL org.opencontainers.image.authors="${MAINTAINER}" LABEL site="https://flume.apache.org" COPY target/apache-flume-${ASSEMBLY_VERSION}-bin/apache-flume-${ASSEMBLY_VERSION}-bin/ /etc/flume/ From 67664abd976af065c5e4639032e2b659caf4f78f Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Wed, 9 Mar 2022 07:12:55 +0000 Subject: [PATCH 18/20] per HBASE-22318 switch HBase to use unbounded glassfish dependency --- flume-ng-sinks/flume-ng-hbase2-sink/pom.xml | 14 ++++++++++++++ pom.xml | 1 + 2 files changed, 15 insertions(+) diff --git a/flume-ng-sinks/flume-ng-hbase2-sink/pom.xml b/flume-ng-sinks/flume-ng-hbase2-sink/pom.xml index 0a445a6f26..a4cf863d72 100644 --- a/flume-ng-sinks/flume-ng-hbase2-sink/pom.xml +++ b/flume-ng-sinks/flume-ng-hbase2-sink/pom.xml @@ -85,6 +85,12 @@ hbase-server ${hbase2.version} test-jar + + + org.glassfish + javax.el + + @@ -114,6 +120,14 @@ ${hbase2.jetty.version} test + + + org.glassfish + javax.el + ${glassfish.version} + test + + diff --git a/pom.xml b/pom.xml index b48ee07371..c359ddabca 100644 --- a/pom.xml +++ b/pom.xml @@ -68,6 +68,7 @@ limitations under the License. 2.13.1 1.4 1.1.1 + 3.0.1-b12 2.2.2 18.0 11.0.2 From 28370c29a3f65102d4e41952cca2fc59557cf165 Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Sat, 9 Apr 2022 05:56:09 -0400 Subject: [PATCH 19/20] Revert "per HBASE-22318 switch HBase to use unbounded glassfish dependency" This reverts commit 67664abd976af065c5e4639032e2b659caf4f78f. --- flume-ng-sinks/flume-ng-hbase2-sink/pom.xml | 14 -------------- pom.xml | 1 - 2 files changed, 15 deletions(-) diff --git a/flume-ng-sinks/flume-ng-hbase2-sink/pom.xml b/flume-ng-sinks/flume-ng-hbase2-sink/pom.xml index a4cf863d72..0a445a6f26 100644 --- a/flume-ng-sinks/flume-ng-hbase2-sink/pom.xml +++ b/flume-ng-sinks/flume-ng-hbase2-sink/pom.xml @@ -85,12 +85,6 @@ hbase-server ${hbase2.version} test-jar - - - org.glassfish - javax.el - - @@ -120,14 +114,6 @@ ${hbase2.jetty.version} test - - - org.glassfish - javax.el - ${glassfish.version} - test - - diff --git a/pom.xml b/pom.xml index c359ddabca..b48ee07371 100644 --- a/pom.xml +++ b/pom.xml @@ -68,7 +68,6 @@ limitations under the License. 2.13.1 1.4 1.1.1 - 3.0.1-b12 2.2.2 18.0 11.0.2 From 7571738848ca778a4be66249481ac47e41d0cdcd Mon Sep 17 00:00:00 2001 From: Tristan Stevens Date: Wed, 11 May 2022 19:21:16 +0100 Subject: [PATCH 20/20] Update build.yml Add in github secrets and push to dockerhub --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2ec67f1081..7eb71bc45f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,6 +63,14 @@ jobs: -DtrimStackTrace=false \ -Dsurefire.rerunFailingTestsCount=2 + - name: Deploy Docker (Linux) + if: runner.os == 'Linux' && github.ref == 'refs/heads/trunk' + timeout-minutes: 90 + shell: bash + run: | + ./mvnw deploy -Ddockerfile.username=$${{ secrets.DOCKERHUB_USER }} \ + -Ddockerfile.password=... $${{ secrets.DOCKERHUB_TOKEN }} + - name: Build with Maven (MacOS) if: runner.os == 'macOS' timeout-minutes: 120