From f78cd9e4fd73408035565ef6a2cc6b4b90ee0a16 Mon Sep 17 00:00:00 2001 From: Kip Kohn Date: Wed, 13 Sep 2023 11:30:50 -0700 Subject: [PATCH] Add null-check to `JobConfigurationUtils.putStateIntoConfiguration` to avoid `IllegalArgumentException` --- .../java/org/apache/gobblin/writer/GobblinBaseOrcWriter.java | 5 ++--- .../org/apache/gobblin/runtime/mapreduce/MRJobLauncher.java | 3 ++- .../java/org/apache/gobblin/util/JobConfigurationUtils.java | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/gobblin-modules/gobblin-orc/src/main/java/org/apache/gobblin/writer/GobblinBaseOrcWriter.java b/gobblin-modules/gobblin-orc/src/main/java/org/apache/gobblin/writer/GobblinBaseOrcWriter.java index f7bc0c32582..618143b153a 100644 --- a/gobblin-modules/gobblin-orc/src/main/java/org/apache/gobblin/writer/GobblinBaseOrcWriter.java +++ b/gobblin-modules/gobblin-orc/src/main/java/org/apache/gobblin/writer/GobblinBaseOrcWriter.java @@ -39,6 +39,7 @@ import org.apache.gobblin.configuration.State; import org.apache.gobblin.state.ConstructState; +import org.apache.gobblin.util.JobConfigurationUtils; /** * A wrapper for ORC-core writer without dependency on Hive SerDe library. @@ -117,9 +118,7 @@ public GobblinBaseOrcWriter(FsDataWriterBuilder builder, State properties) // Create file-writer this.writerConfig = new Configuration(); // Populate job Configurations into Conf as well so that configurations related to ORC writer can be tuned easily. - for (Object key : properties.getProperties().keySet()) { - this.writerConfig.set((String) key, properties.getProp((String) key)); - } + JobConfigurationUtils.putStateIntoConfiguration(properties, this.writerConfig); OrcFile.WriterOptions options = OrcFile.writerOptions(properties.getProperties(), this.writerConfig); options.setSchema(typeDescription); diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncher.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncher.java index 1302cdab2a5..cd34fe1b5d6 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncher.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/mapreduce/MRJobLauncher.java @@ -316,7 +316,8 @@ protected void runWorkUnits(List workUnits) throws Exception { prepareHadoopJob(workUnits); if (this.shouldPersistWorkUnitsThenCancel) { - LOG.info("Cancelling job after persisting workunits beneath: " + this.jobInputPath); + // NOTE: `warn` level is hack for including path among automatic troubleshooter 'issues' + LOG.warn("Cancelling job after persisting workunits beneath: " + this.jobInputPath); jobState.setState(JobState.RunningState.CANCELLED); return; } diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/util/JobConfigurationUtils.java b/gobblin-utility/src/main/java/org/apache/gobblin/util/JobConfigurationUtils.java index 957444cac04..aad8d6aabd2 100644 --- a/gobblin-utility/src/main/java/org/apache/gobblin/util/JobConfigurationUtils.java +++ b/gobblin-utility/src/main/java/org/apache/gobblin/util/JobConfigurationUtils.java @@ -92,7 +92,10 @@ public static void putConfigurationIntoProperties(Configuration configuration, P */ public static void putStateIntoConfiguration(State state, Configuration configuration) { for (String key : state.getPropertyNames()) { - configuration.set(key, state.getProp(key)); + String value = state.getProp(key); + if (value != null) { // ignore `null`, to prevent `IllegalArgumentException` from `Configuration::set` + configuration.set(key, value); + } } }