From 1548350bdb3dedc10c66c4951684c75dc230c304 Mon Sep 17 00:00:00 2001 From: Zihan Li Date: Thu, 18 May 2023 17:34:02 -0700 Subject: [PATCH 01/30] [GOBBLIN-1833]Emit Completeness watermark information in snapshotCommitEvent (#3696) * address comments * use connectionmanager when httpclient is not cloesable * [GOBBLIN-1833]Emit Completeness watermark information in snapshotCommitEvent * address comments --------- Co-authored-by: Zihan Li --- .../apache/gobblin/iceberg/writer/IcebergMetadataWriter.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java index 1e88a83c7f4..3135a124351 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java @@ -788,7 +788,7 @@ private StructLike getIcebergPartitionVal(Collection specs, String fil * @return kafka topic name for this table */ protected String getTopicName(TableIdentifier tid, TableMetadata tableMetadata) { - if (tableMetadata.dataOffsetRange.isPresent()) { + if (tableMetadata.dataOffsetRange.isPresent() && tableMetadata.dataOffsetRange.get().size() != 0) { String topicPartitionString = tableMetadata.dataOffsetRange.get().keySet().iterator().next(); //In case the topic name is not the table name or the topic name contains '-' return topicPartitionString.substring(0, topicPartitionString.lastIndexOf('-')); @@ -1011,6 +1011,9 @@ private void submitSnapshotCommitEvent(Snapshot snapshot, TableMetadata tableMet gobblinTrackingEvent.addMetadata(entry.getKey(), entry.getValue()); } } + if (tableMetadata.completenessEnabled) { + gobblinTrackingEvent.addMetadata(COMPLETION_WATERMARK_KEY, Long.toString(tableMetadata.completionWatermark)); + } eventSubmitter.submit(gobblinTrackingEvent); } From 85b0a1e57402377f2baa35b9f7d0ca9a36d44a1a Mon Sep 17 00:00:00 2001 From: Zihan Li Date: Fri, 19 May 2023 14:35:15 -0700 Subject: [PATCH 02/30] [GOBBLIN-1830] Improving Container Transition Tracking in Streaming Data Ingestion (#3693) * address comments * use connectionmanager when httpclient is not cloesable * [GOBBLIN-1830]Improving Container Transition Tracking in Streaming Data Ingestion * emmit event with a different name * remove unnecessary log --------- Co-authored-by: Zihan Li --- .../kafka/KafkaExtractorStatsTracker.java | 17 +++++++++++++++++ .../extract/kafka/KafkaStreamingExtractor.java | 8 ++++++++ 2 files changed, 25 insertions(+) diff --git a/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/source/extractor/extract/kafka/KafkaExtractorStatsTracker.java b/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/source/extractor/extract/kafka/KafkaExtractorStatsTracker.java index b1bf19788b8..8ee8e5e8c96 100644 --- a/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/source/extractor/extract/kafka/KafkaExtractorStatsTracker.java +++ b/gobblin-modules/gobblin-kafka-common/src/main/java/org/apache/gobblin/source/extractor/extract/kafka/KafkaExtractorStatsTracker.java @@ -40,6 +40,7 @@ import org.apache.gobblin.configuration.WorkUnitState; import org.apache.gobblin.metrics.MetricContext; import org.apache.gobblin.metrics.event.EventSubmitter; +import org.apache.gobblin.metrics.event.GobblinEventBuilder; import org.apache.gobblin.runtime.api.TaskEventMetadataGenerator; import org.apache.gobblin.util.TaskEventMetadataUtils; @@ -56,6 +57,7 @@ public class KafkaExtractorStatsTracker { private static final String EMPTY_STRING = ""; private static final String GOBBLIN_KAFKA_NAMESPACE = "gobblin.kafka"; + private static final String KAFKA_EXTRACTOR_CONTAINER_TRANSITION_EVENT_NAME = "KafkaExtractorContainerTransitionEvent"; private static final String KAFKA_EXTRACTOR_TOPIC_METADATA_EVENT_NAME = "KafkaExtractorTopicMetadata"; private static final String LOW_WATERMARK = "lowWatermark"; private static final String ACTUAL_HIGH_WATERMARK = "actualHighWatermark"; @@ -497,6 +499,21 @@ public void emitTrackingEvents(MetricContext context, Map { @@ -271,6 +273,12 @@ public KafkaStreamingExtractor(WorkUnitState state) { this.workUnitState.getProp(KafkaSource.RECORD_CREATION_TIMESTAMP_UNIT, TimeUnit.MILLISECONDS.name())); } + private void submitEventToIndicateContainerTransition() { + if (this.isInstrumentationEnabled()) { + this.statsTracker.submitEventToIndicateContainerTransition(getMetricContext()); + } + } + private Map getTopicPartitionWatermarks(List topicPartitions) { List topicPartitionStrings = topicPartitions.stream().map(topicPartition -> topicPartition.toString()).collect(Collectors.toList()); From b4398001431050df51c593e88b0ccb45c7dd8fa4 Mon Sep 17 00:00:00 2001 From: Zihan Li Date: Mon, 22 May 2023 11:58:03 -0700 Subject: [PATCH 03/30] [GOBBLIN-1823] Improving Container Calculation and Allocation Methodology (#3692) * address comments * use connectionmanager when httpclient is not cloesable * [GOBBLIN-1823] Improving Container Calculation and Allocation Methodology * improve code style * add more un-retriable status that we want to log out * address comments * add log when mismatch happens for debuggability --------- Co-authored-by: Zihan Li --- .../gobblin/yarn/YarnAutoScalingManager.java | 6 +- .../org/apache/gobblin/yarn/YarnService.java | 66 +++++++++++++++++-- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java index 5f5c872c67e..e6683cfd383 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java @@ -88,7 +88,7 @@ public class YarnAutoScalingManager extends AbstractIdleService { private final String AUTO_SCALING_WINDOW_SIZE = AUTO_SCALING_PREFIX + "windowSize"; - private final static int DEFAULT_MAX_IDLE_TIME_BEFORE_SCALING_DOWN_MINUTES = 10; + public final static int DEFAULT_MAX_CONTAINER_IDLE_TIME_BEFORE_SCALING_DOWN_MINUTES = 10; private final Config config; private final HelixManager helixManager; @@ -97,9 +97,9 @@ public class YarnAutoScalingManager extends AbstractIdleService { private final int partitionsPerContainer; private final double overProvisionFactor; private final SlidingWindowReservoir slidingFixedSizeWindow; - private static int maxIdleTimeInMinutesBeforeScalingDown = DEFAULT_MAX_IDLE_TIME_BEFORE_SCALING_DOWN_MINUTES; + private static int maxIdleTimeInMinutesBeforeScalingDown = DEFAULT_MAX_CONTAINER_IDLE_TIME_BEFORE_SCALING_DOWN_MINUTES; private static final HashSet - UNUSUAL_HELIX_TASK_STATES = Sets.newHashSet(TaskPartitionState.ERROR, TaskPartitionState.DROPPED); + UNUSUAL_HELIX_TASK_STATES = Sets.newHashSet(TaskPartitionState.ERROR, TaskPartitionState.DROPPED, TaskPartitionState.COMPLETED, TaskPartitionState.TIMED_OUT); public YarnAutoScalingManager(GobblinApplicationMaster appMaster) { this.config = appMaster.getConfig(); diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java index fe0d93d2dfb..e1da50d94ee 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnService.java @@ -26,6 +26,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -200,6 +201,8 @@ public class YarnService extends AbstractIdleService { private final boolean isPurgingOfflineHelixInstancesEnabled; private final long helixPurgeLaggingThresholdMs; private final long helixPurgeStatusPollingRateMs; + private final ConcurrentMap containerIdleSince = Maps.newConcurrentMap(); + private final ConcurrentMap removedContainerID = Maps.newConcurrentMap(); private volatile YarnContainerRequestBundle yarnContainerRequest; private final AtomicInteger priorityNumGenerator = new AtomicInteger(0); @@ -473,11 +476,30 @@ public synchronized boolean requestTargetNumberOfContainers(YarnContainerRequest return false; } + //Correct the containerMap first as there is cases that handleContainerCompletion() is called before onContainersAllocated() + for (ContainerId removedId : this.removedContainerID.keySet()) { + ContainerInfo containerInfo = this.containerMap.remove(removedId); + if (containerInfo != null) { + String helixTag = containerInfo.getHelixTag(); + allocatedContainerCountMap.putIfAbsent(helixTag, new AtomicInteger(0)); + this.allocatedContainerCountMap.get(helixTag).decrementAndGet(); + this.removedContainerID.remove(removedId); + } + } + int numTargetContainers = yarnContainerRequestBundle.getTotalContainers(); // YARN can allocate more than the requested number of containers, compute additional allocations and deallocations // based on the max of the requested and actual allocated counts // Represents the number of containers allocated for across all helix tags int totalAllocatedContainers = this.containerMap.size(); + int totalContainersInContainerCountMap = 0; + for (AtomicInteger count: allocatedContainerCountMap.values()) { + totalContainersInContainerCountMap += count.get(); + } + if (totalContainersInContainerCountMap != totalAllocatedContainers) { + LOGGER.warn(String.format("Container number mismatch in containerMap and allocatedContainerCountMap, " + + "we have %s containers in containerMap while %s in allocatedContainerCountMap", totalAllocatedContainers, totalContainersInContainerCountMap)); + } // Request additional containers if the desired count is higher than the max of the current allocation or previously // requested amount. Note that there may be in-flight or additional allocations after numContainers has been computed @@ -501,31 +523,54 @@ public synchronized boolean requestTargetNumberOfContainers(YarnContainerRequest } } + //Iterate through all containers allocated and check whether the corresponding helix instance is still LIVE within the helix cluster. + // A container that has a bad connection to zookeeper will be dropped from the Helix cluster if the disconnection is greater than the specified timeout. + // In these cases, we want to release the container to get a new container because these containers won't be assigned tasks by Helix + + List containersToRelease = new ArrayList<>(); + HashSet idleContainerIdsToRelease = new HashSet<>(); + for (Map.Entry entry : this.containerMap.entrySet()) { + ContainerInfo containerInfo = entry.getValue(); + if (!HelixUtils.isInstanceLive(helixManager, containerInfo.getHelixParticipantId())) { + containerIdleSince.putIfAbsent(entry.getKey(), System.currentTimeMillis()); + if (System.currentTimeMillis() - containerIdleSince.get(entry.getKey()) + >= TimeUnit.MINUTES.toMillis(YarnAutoScalingManager.DEFAULT_MAX_CONTAINER_IDLE_TIME_BEFORE_SCALING_DOWN_MINUTES)) { + LOGGER.info("Releasing Container {} because the assigned participant {} has been in-active for more than {} minutes", + entry.getKey(), containerInfo.getHelixParticipantId(), YarnAutoScalingManager.DEFAULT_MAX_CONTAINER_IDLE_TIME_BEFORE_SCALING_DOWN_MINUTES); + containersToRelease.add(containerInfo.getContainer()); + idleContainerIdsToRelease.add(entry.getKey()); + } + } else { + containerIdleSince.remove(entry.getKey()); + } + } + // If the total desired is lower than the currently allocated amount then release free containers. // This is based on the currently allocated amount since containers may still be in the process of being allocated // and assigned work. Resizing based on numRequestedContainers at this point may release a container right before // or soon after it is assigned work. - if (numTargetContainers < totalAllocatedContainers) { - List containersToRelease = new ArrayList<>(); + if (numTargetContainers < totalAllocatedContainers - idleContainerIdsToRelease.size()) { int numToShutdown = totalAllocatedContainers - numTargetContainers; - LOGGER.info("Shrinking number of containers by {} because numTargetContainers < totalAllocatedContainers ({} < {})", - numToShutdown, numTargetContainers, totalAllocatedContainers); + LOGGER.info("Shrinking number of containers by {} because numTargetContainers < totalAllocatedContainers - idleContainersToRelease ({} < {} - {})", + totalAllocatedContainers - numTargetContainers - idleContainerIdsToRelease.size(), numTargetContainers, totalAllocatedContainers, idleContainerIdsToRelease.size()); // Look for eligible containers to release. If a container is in use then it is not released. for (Map.Entry entry : this.containerMap.entrySet()) { ContainerInfo containerInfo = entry.getValue(); - if (!inUseInstances.contains(containerInfo.getHelixParticipantId())) { + if (!inUseInstances.contains(containerInfo.getHelixParticipantId()) && !idleContainerIdsToRelease.contains(entry.getKey())) { containersToRelease.add(containerInfo.getContainer()); } - if (containersToRelease.size() == numToShutdown) { + if (containersToRelease.size() >= numToShutdown) { break; } } LOGGER.info("Shutting down {} containers. containersToRelease={}", containersToRelease.size(), containersToRelease); + } + if (!containersToRelease.isEmpty()) { this.eventBus.post(new ContainerReleaseRequest(containersToRelease)); } this.yarnContainerRequest = yarnContainerRequestBundle; @@ -721,9 +766,16 @@ protected void handleContainerCompletion(ContainerStatus containerStatus) { //Get the Helix instance name for the completed container. Because callbacks are processed asynchronously, we might //encounter situations where handleContainerCompletion() is called before onContainersAllocated(), resulting in the //containerId missing from the containersMap. + // We use removedContainerID to remember these containers and remove them from containerMap later when we call requestTargetNumberOfContainers method + if (completedContainerInfo == null) { + removedContainerID.putIfAbsent(containerStatus.getContainerId(), ""); + } String completedInstanceName = completedContainerInfo == null? UNKNOWN_HELIX_INSTANCE : completedContainerInfo.getHelixParticipantId(); + String helixTag = completedContainerInfo == null ? helixInstanceTags : completedContainerInfo.getHelixTag(); - allocatedContainerCountMap.get(helixTag).decrementAndGet(); + if (completedContainerInfo != null) { + allocatedContainerCountMap.get(helixTag).decrementAndGet(); + } LOGGER.info(String.format("Container %s running Helix instance %s with tag %s has completed with exit status %d", containerStatus.getContainerId(), completedInstanceName, helixTag, containerStatus.getExitStatus())); From 158d6fff546ee28ad7b542bfe78e5263af8843c6 Mon Sep 17 00:00:00 2001 From: meethngala Date: Wed, 24 May 2023 13:46:56 -0700 Subject: [PATCH 04/30] [GOBBLIN-1825]Hive retention job should fail if deleting underlying files fail (#3687) * fail hive retention if we fail to delete underlying hdfs files * address PR comments * address PR comments --------- Co-authored-by: Meeth Gala --- .../version/HiveDatasetVersionCleaner.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/retention/version/HiveDatasetVersionCleaner.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/retention/version/HiveDatasetVersionCleaner.java index ffcc3b3527e..1363be2e02a 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/retention/version/HiveDatasetVersionCleaner.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/retention/version/HiveDatasetVersionCleaner.java @@ -88,20 +88,31 @@ public void clean() throws IOException { Partition partition = hiveDatasetVersion.getPartition(); try { if (!cleanableHiveDataset.isSimulate()) { + // As part of the cleanup process, we want to delete both: hive partition and underlying hdfs files + // However, scenarios arise where hive partition is dropped, but hdfs files aren't, leading to dangling files + // Thus, we reverse the order of cleaning up hdfs files first and then drop hive partition + // In cases where HMS was unresponsive and hive partition couldn't be dropped + // re-running hive retention would drop the partition with no hdfs files found to be deleted + // or set the flag `isShouldDeleteData` to false + if (cleanableHiveDataset.isShouldDeleteData()) { + cleanableHiveDataset.getFsCleanableHelper().clean(hiveDatasetVersion, possiblyEmptyDirectories); + } client.get().dropPartition(partition.getTable().getDbName(), partition.getTable().getTableName(), partition.getValues(), false); log.info("Successfully dropped partition " + partition.getCompleteName()); } else { log.info("Simulating drop partition " + partition.getCompleteName()); } - if (cleanableHiveDataset.isShouldDeleteData()) { - cleanableHiveDataset.getFsCleanableHelper().clean(hiveDatasetVersion, possiblyEmptyDirectories); - } } catch (TException | IOException e) { log.warn(String.format("Failed to completely delete partition %s.", partition.getCompleteName()), e); throw new IOException(e); } } - cleanableHiveDataset.getFsCleanableHelper().cleanEmptyDirectories(possiblyEmptyDirectories, cleanableHiveDataset); + try { + cleanableHiveDataset.getFsCleanableHelper().cleanEmptyDirectories(possiblyEmptyDirectories, cleanableHiveDataset); + } catch (IOException ex) { + log.warn(String.format("Failed to delete at least one or more empty directories from total:{%s} with root path %s", possiblyEmptyDirectories.size(), cleanableHiveDataset.datasetRoot()), ex); + throw new IOException(ex); + } } @Override From 51a852d506b749b9ac33568aff47105e14972a57 Mon Sep 17 00:00:00 2001 From: vikram bohra Date: Fri, 26 May 2023 09:41:29 -0700 Subject: [PATCH 05/30] [GOBBLIN-1805] Check watermark for the most recent hour for quiet topics (#3698) * [GOBBLIN-1805] Check watermark for the most recent hour for quiet topics * fixed test case --- .../iceberg/writer/IcebergMetadataWriter.java | 8 ++++---- .../iceberg/writer/IcebergMetadataWriterTest.java | 14 +++++++++----- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java index 3135a124351..4cfb0bea056 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java @@ -23,6 +23,7 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -832,15 +833,14 @@ public void flush(String dbName, String tableName) throws IOException { tableMetadata.deleteFiles.get().commit(); } // Check and update completion watermark when there are no files to be registered, typically for quiet topics - // The logic is to check the next window from previous completion watermark and update the watermark if there are no audit counts + // The logic is to check the window [currentHour-1,currentHour] and update the watermark if there are no audit counts if(!tableMetadata.appendFiles.isPresent() && !tableMetadata.deleteFiles.isPresent() && tableMetadata.completenessEnabled) { if (tableMetadata.completionWatermark > DEFAULT_COMPLETION_WATERMARK) { log.info(String.format("Checking kafka audit for %s on change_property ", topicName)); SortedSet timestamps = new TreeSet<>(); - ZonedDateTime prevWatermarkDT = - Instant.ofEpochMilli(tableMetadata.completionWatermark).atZone(ZoneId.of(this.timeZone)); - timestamps.add(TimeIterator.inc(prevWatermarkDT, TimeIterator.Granularity.valueOf(this.auditCheckGranularity), 1)); + ZonedDateTime dtAtBeginningOfHour = ZonedDateTime.now(ZoneId.of(this.timeZone)).truncatedTo(ChronoUnit.HOURS); + timestamps.add(dtAtBeginningOfHour); checkAndUpdateCompletenessWatermark(tableMetadata, topicName, timestamps, props); } else { log.info(String.format("Need valid watermark, current watermark is %s, Not checking kafka audit for %s", diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java index 294ef08ab95..bc1c8ca5706 100644 --- a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java +++ b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java @@ -19,6 +19,9 @@ import java.io.File; import java.io.IOException; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; import java.util.*; import java.util.concurrent.TimeUnit; @@ -492,9 +495,9 @@ public void testWriteAddFileGMCECompleteness() throws IOException { @Test(dependsOnMethods={"testWriteAddFileGMCECompleteness"}, groups={"icebergMetadataWriterTest"}) public void testChangePropertyGMCECompleteness() throws IOException { - Table table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); - long watermark = Long.parseLong(table.properties().get(COMPLETION_WATERMARK_KEY)); - long expectedWatermark = watermark + TimeUnit.HOURS.toMillis(1); + ZonedDateTime expectedCWDt = ZonedDateTime.now(ZoneId.of(DEFAULT_TIME_ZONE)).truncatedTo(ChronoUnit.HOURS); + // For quiet topics, watermark should always be beginning of current hour + long expectedWatermark = expectedCWDt.toInstant().toEpochMilli(); File hourlyFile2 = new File(tmpDir, "testDB/testTopic/hourly/2021/09/16/11/data.avro"); gmce.setOldFilePrefixes(null); gmce.setNewFiles(Lists.newArrayList(DataFile.newBuilder() @@ -511,11 +514,12 @@ public void testChangePropertyGMCECompleteness() throws IOException { new LongWatermark(65L)))); KafkaAuditCountVerifier verifier = Mockito.mock(TestAuditCountVerifier.class); - Mockito.when(verifier.isComplete("testTopic", watermark, expectedWatermark)).thenReturn(true); + // For quiet topics always check for previous hour window + Mockito.when(verifier.isComplete("testTopic", expectedCWDt.minusHours(1).toInstant().toEpochMilli(), expectedWatermark)).thenReturn(true); ((IcebergMetadataWriter) gobblinMCEWriterWithCompletness.metadataWriters.iterator().next()).setAuditCountVerifier(verifier); gobblinMCEWriterWithCompletness.flush(); - table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); + Table table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); Assert.assertEquals(table.properties().get("offset.range.testTopic-1"), "0-7000"); Assert.assertEquals(table.spec().fields().get(1).name(), "late"); Assert.assertEquals(table.properties().get(TOPIC_NAME_KEY), "testTopic"); From a00b57cda36acf9e30f9856bdc69383528f70486 Mon Sep 17 00:00:00 2001 From: Zihan Li Date: Fri, 2 Jun 2023 13:53:58 -0700 Subject: [PATCH 06/30] [GOBBLIN-1836] Ensuring Task Reliability: Handling Job Cancellation and Graceful Exits for Error-Free Completion (#3699) * address comments * use connectionmanager when httpclient is not cloesable * [GOBBLIN-1836] Ensuring Task Reliability: Handling Job Cancellation and Graceful Exits for Error-Free Completion * add unit test --------- Co-authored-by: Zihan Li --- .../gobblin/cluster/GobblinHelixTask.java | 10 ++++++++ .../gobblin/cluster/GobblinHelixTaskTest.java | 23 +++++++++++++++++++ .../iceberg/writer/IcebergMetadataWriter.java | 2 ++ 3 files changed, 35 insertions(+) diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixTask.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixTask.java index 11a60959a2a..3e7d2707ede 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixTask.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixTask.java @@ -90,6 +90,7 @@ public class GobblinHelixTask implements Task { private SingleTask task; private String helixTaskId; private EventBus eventBus; + private boolean isCanceled; public GobblinHelixTask(TaskRunnerSuiteBase.Builder builder, TaskCallbackContext taskCallbackContext, @@ -161,12 +162,20 @@ private void getInfoFromTaskConfig() { @Override public TaskResult run() { this.taskMetrics.helixTaskTotalRunning.incrementAndGet(); + this.isCanceled = false; long startTime = System.currentTimeMillis(); log.info("Actual task {} started. [{} {}]", this.taskId, this.applicationName, this.instanceName); try (Closer closer = Closer.create()) { closer.register(MDC.putCloseable(ConfigurationKeys.JOB_NAME_KEY, this.jobName)); closer.register(MDC.putCloseable(ConfigurationKeys.JOB_KEY_KEY, this.jobKey)); this.task.run(); + // Since we enable gracefully cancel, when task get cancelled, we might not see any exception, + // so we check the isCanceled flag to make sure we return the correct task status + if (this.isCanceled) { + log.error("Actual task {} canceled.", this.taskId); + this.taskMetrics.helixTaskTotalCancelled.incrementAndGet(); + return new TaskResult(TaskResult.Status.CANCELED, ""); + } log.info("Actual task {} completed.", this.taskId); this.taskMetrics.helixTaskTotalCompleted.incrementAndGet(); return new TaskResult(TaskResult.Status.COMPLETED, ""); @@ -219,6 +228,7 @@ public void cancel() { log.info("Gobblin helix task cancellation invoked for jobId {}.", jobId); if (this.task != null ) { try { + this.isCanceled = true; this.task.cancel(); log.info("Gobblin helix task cancellation completed for jobId {}.", jobId); } catch (Throwable t) { diff --git a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixTaskTest.java b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixTaskTest.java index 8cdbbef759e..e7a8295dcac 100644 --- a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixTaskTest.java +++ b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixTaskTest.java @@ -94,6 +94,8 @@ public class GobblinHelixTaskTest { private GobblinHelixTask gobblinHelixTask; + private GobblinHelixTask gobblinHelixTaskForCancel; + private HelixManager helixManager; private FileSystem localFs; @@ -194,6 +196,8 @@ public void testPrepareTask() // Expect to go through. this.gobblinHelixTask = (GobblinHelixTask) gobblinHelixTaskFactory.createNewTask(taskCallbackContext); + this.gobblinHelixTaskForCancel = (GobblinHelixTask) gobblinHelixTaskFactory.createNewTask(taskCallbackContext); + // Mock the method getFs() which get called in SingleTask constructor, so that SingleTask could fail and trigger retry, // which would also fail eventually with timeout. TaskRunnerSuiteBase.Builder builderSpy = Mockito.spy(builder); @@ -256,6 +260,25 @@ public void testRun() throws IOException { TestHelper.assertGenericRecords(outputAvroFile, schema); } + @Test(dependsOnMethods = "testRun") + public void testCancel() throws IOException, InterruptedException { + + final TaskResult[] taskResult = new TaskResult[1]; + Thread thread = new Thread(){ + @Override + public void run() { + taskResult[0] = gobblinHelixTaskForCancel.run(); + } + }; + thread.start(); + Thread.sleep(3); + gobblinHelixTaskForCancel.cancel(); + thread.join(); + System.out.println(taskResult[0].getInfo()); + //We can see task failure or task cancelled as task status + Assert.assertNotEquals(taskResult[0].getStatus(), TaskResult.Status.COMPLETED); + } + @AfterClass public void tearDown() throws IOException { try { diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java index 4cfb0bea056..9c3d30e9e50 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java @@ -521,6 +521,8 @@ protected Table createTable(GobblinMetadataChangeEvent gmce, HiveSpec spec) thro try (Timer.Context context = metricContext.timer(CREATE_TABLE_TIME).time()) { icebergTable = catalog.createTable(tid, tableSchema, partitionSpec, tableLocation, IcebergUtils.getTableProperties(table)); + // We should set the avro schema literal when creating the table. + icebergTable.updateProperties().set(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName(), schema).commit(); log.info("Created table {}, schema: {} partition spec: {}", tid, tableSchema, partitionSpec); } catch (AlreadyExistsException e) { log.warn("table {} already exist, there may be some other process try to create table concurrently", tid); From 586371a250e6b28629b875f96a6ba9d1847cc0ea Mon Sep 17 00:00:00 2001 From: Abhishek Tiwari Date: Tue, 13 Jun 2023 20:33:19 -0700 Subject: [PATCH 07/30] Reserving 0.18.0 version for next release --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 01f4457a6b7..2101483e2d2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -31,7 +31,7 @@ org.gradle.parallel=true ide.recursive=true # Apache release specific -version=0.17.0 +version=0.18.0 group=org.apache.gobblin release=false From fc508ca272cc2ea0a9f5cdd28e72f60ce3c7912b Mon Sep 17 00:00:00 2001 From: Abhishek Tiwari Date: Tue, 13 Jun 2023 22:04:52 -0700 Subject: [PATCH 08/30] Update CHANGELOG to reflect changes in 0.17.0 --- CHANGELOG.md | 237 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 418cfb1f1ca..51abe3e66d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,240 @@ +GOBBLIN 0.17.0 +-------------- + +### Created Date: 06/13/2023 + +* [GOBBLIN-1836] Ensure Task Reliability: Handle Job Cancellation and Graceful Exits for Error-Free Completion +* [GOBBLIN-1805] Support watermark for the most recent hour for quiet topics +* [GOBBLIN-1833] Emit Completeness watermark information in SnapshotCommitEvent +* [GOBBLIN-1832] Emit warning instead of failing job in retention +* [GOBBLIN-1831] Use flowexecutionid in kafka monitor and jobnames +* [GOBBLIN-1830] Improved Container Transition Tracking in Streaming Data Ingestion +* [GOBBLIN-1823] Improved Container Calculation and Allocation Methodology +* [GOBBLIN-1829] Fixed bug where the wrong workunit event was being tracked +* [GOBBLIN-1828] Implement Timeout for Creating Writer Functionality +* [GOBBLIN-1827] Added check that if nested field is optional and has a non-null default +* [GOBBLIN-1826] Changed isAssignableFrom() to isSuperTypeOf() per Guava 20 javadocs +* [GOBBLIN-1825] Fail Hive retention job if deleting underlying files fail +* [GOBBLIN-1824] Improved the Efficiency of Work Planning in Manifest-Based DistCp Jobs +* [GOBBLIN-1822] Logging for Abnormal Helix Task States +* [GOBBLIN-1821] Allow flow execution ID propagate to the Job ID if it exists +* [GOBBLIN-1820] Added null default value to observability events +* [GOBBLIN-1819] Log helix workflow information and timeout information during submission wait / polling +* [GOBBLIN-1810] Support general Iceberg catalog (support configurable behavior for metadata retention policy) +* [GOBBLIN-1818] Initilaize yarn clients in yarn app launcher so that a child class can override the yarn client creation logic +* [GOBBLIN-1817] Changed some deprecated code and fix minor codestyle +* [GOBBLIN-1813] Helix workflows submission timeouts made configurable +* [GOBBLIN-1816] Added job properties and GaaS instance ID to observability event +* [GOBBLIN-1814] Added MRJobLauncher configurability for any failing mapper to be fatal to the MR job +* [GOBBLIN-1811] Fix Iceberg Registration Serialization +* [GOBBLIN-1810] Support general Iceberg catalog in IcebergMetadataWriter +* [GOBBLIN-1815] Refactor yarn app launchers to support extending these classes +* [GOBBLIN-1809] Add new lookback version finder for use with iceberg retention +* [GOBBLIN-1808] Bump Guava version from 15.0 to 20.0 +* [GOBBLIN-1807] Replaced conjars.org with conjars.wensel.net +* [GOBBLIN-1806] Submit dataset summary event post commit and integrate them into GaaSObservabilityEvent +* [GOBBLIN-1805] Check watermark for the most recent hour for quiet topics +* [GOBBLIN-1804] Merge similar logic between FlowConfig{,V2}ResourceLocalHandler.update into single base class impl. +* [GOBBLIN-1804] Reject flow config updates that would fail compilation by returning service error +* [GOBBLIN-1802] Register iceberg table metadata update with destination side catalog +* [GOBBLIN-1799] Fix add spec and actual number flows scheduled metrics +* [GOBBLIN-1798] Add backoff retry when we access mysql db for flow spec or dag action +* [GOBBLIN-1797] Skip scheduling flows far into future +* [GOBBLIN-1796] Log startup command when container fails to startup +* [GOBBLIN-1795] Make Manifest based copy to support facl +* [GOBBLIN-1794] Add defaults to newly added fields in observability events +* [GOBBLIN-1793] Add metrics to measure and isolate bottleneck for init +* [GOBBLIN-1792] Upgrade Mockito to 4.* +* [GOBBLIN-1791] Prevent the adding of flowspec compilation errors to the scheduler +* [GOBBLIN-1790] Add and change appropriate job status fields for observability events +* [GOBBLIN-1779] Ability to filter datasets that contain non optional unions +* [GOBBLIN-1789] Create Generic Iceberg Data Node to Support Different Types of Catalogs +* [GOBBLIN-1787] Ability to delete multiple watermarks in a state store +* [GOBBLIN-1786] Support Other Catalog Types for Iceberg Distcp +* [GOBBLIN-1785] Add MR_JARS_BASE_DIR and logic to delete old mr jar dirs +* [GOBBLIN-1784] Only clean dags from the dag manager if a flow event is received +* [GOBBLIN-1783] Initialize scheduler with batch gets instead of individual get per flow +* [GOBBLIN-1782] Fix Merge State for Flow Pending Resume statuses +* [GOBBLIN-1781] Make Helix offline instance purging thread safe in the yarn service +* [GOBBLIN-1780] Refactor/rename YarnServiceIT to YarnServiceTest +* [GOBBLIN-1773] Fix bugs in quota manager +* [GOBBLIN-1778] Add house keeping thread in DagManager to periodically sync in memory state with mysql table +* [GOBBLIN-1777] Register gauge metrics for change monitors +* [GOBBLIN-1775] Make GMIP Hive metadatawriter gracefully fail +* [GOBBLIN-1774] Util for detecting non optional uniontype columns based on Hive Table metadata +* [GOBBLIN-1771] Clean up logs for dataset commit and file cleanup +* [GOBBLIN-1770] Allow null values for fields in GaaSObservabilityEvent.Issue fields which are optional +* [GOBBLIN-1769] Change a noisy log that indicates that the queue capacity is almost full +* [GOBBLIN-1768] Fix constructor in KafkaJobStatusMonitorFactory so that it can be injected +* [GOBBLIN-1767] Update references to deprecated Mysql connector/j driver to new name +* [GOBBLIN-1766] Define metric to measure lag from producer to consumer +* [GOBBLIN-1765] Add support to sync metadata for dir in manifest based copy +* [GOBBLIN-1764] Emit observability event +* [GOBBLIN-1763] D2 markup/down for all live GaaS services not only leader +* [GOBBLIN-1762] Upgrade Gobblin OSS Hadoop version to 2.10.0 +* [GOBBLIN-1761] Update Gobblin OSS Slack channel link to a never-expire link +* [GOBBLIN-1758] Disable flaky HiveMaterializerTest on CI/CD +* [GOBBLIN-1757] Refactor manifest, add reader/writer and iterator for efficient reading +* [GOBBLIN-1756] Fix the issue that causes skipping flows for multihop jobs +* [GOBBLIN-1755] Support extended ACLs and sticky bit for file based distcp +* [GOBBLIN-1754] Fixes for mysql store change monitors +* [GOBBLIN-1759] Add error reporting when attempting to resolve flow configs +* [GOBBLIN-1753] Migrate DB connection pool from o.a.commons.dbcp/dbcp2 to HikariCP +* [GOBBLIN-1752] Fix race condition where FSTemplateCatalog would update at the same +* [GOBBLIN-1750] Add schemas for observability events in GaaS +* [GOBBLIN-1749] Add dependency for handling xz-compressed Avro file +* [GOBBLIN-1748] Add logs to debug multi-hop flows creation, progression, and cleanup +* [GOBBLIN-1747] Add job.name and job.id to kafka and compaction workunits +* [GOBBLIN-1746] Add fs.uri to FsDatasetDescriptor to support copy between volumes in GaaS +* [GOBBLIN-1745] Fix bug in SimpleKafkaSpecProducer +* [GOBBLIN-1744] Improve handling of null value edge cases when querying Helix +* [GOBBLIN-1743] Ensure GobblinTaskRunner works without Yarn use +* [GOBBLIN-1742] Do not close DestinationDatasetHandlerService prematurely +* [GOBBLIN-1741] Create manifest based dataset finder +* [GOBBLIN-1739] Define Datanodes and Dataset Descriptor for Iceberg +* [GOBBLIN-1737] Fix bug when using mysql user quota manager +* [GOBBLIN-1738] Move dataset handler code before cleaning up staging data +* [GOBBLIN-1736] Add metrics for change stream monitor and mysql quota manager +* [GOBBLIN-1734] Make DestinationDatasetHandler work on streaming sources +* [GOBBLIN-1735] Correct a log line and GTE with correct number of total task count +* [GOBBLIN-1733] Support multiple node types in shared flowgraph, fix logs +* [GOBBLIN-1732] Search for dummy file in writer directory +* [GOBBLIN-1730] Include flow execution id when try to cancel/submit job using SimpleKafkaSpecProducer +* [GOBBLIN-1731] Enable HiveMetadataWriter to override table schema +* [GOBBLIN-1728] Fix YarnService incorrect container allocation behavior +* [GOBBLIN-1729] Use root cause for checking if exception is transient +* [GOBBLIN-1727] Use delete API to delete the helix job instead of stop it +* [GOBBLIN-1724] Support a shared flowgraph layout in GaaS +* [GOBBLIN-1725] Fix bugs in gaas warm standby mode +* [GOBBLIN-1726] Avro 1.9 upgrade of Gobblin OSS +* [GOBBLIN-1721] Give option to cancel helix workflow through Delete API +* [GOBBLIN-1723] Ignore AlreadyExistsException in hive writer +* [GOBBLIN-1722] Add log line for committing/retrieving watermarks in streaming +* [GOBBLIN-1720] Add ancestors owner permissions preservations for iceberg distcp +* [GOBBLIN-1712] Fail GMIP container for known transient exceptions to avoid data loss +* [GOBBLIN-1707] Enhance IcebergDataset to detect when files already at dest then proceed with only delta +* [GOBBLIN-1719] Replace moveToTrash with moveToAppropriateTrash for hadoop trash +* [GOBBLIN-1718] Define DagActionStoreMonitor to listen for kill/resume +* [GOBBLIN-1717] Correct semantics of IcebergDatasetTest and streamline both impl and test code +* [GOBBLIN-1716] Refactor HighLevelConsumer to make consumer initiatlization configurable +* [GOBBLIN-1707] Update IcebergDataset to incorporate all snapshots, not only the current one +* [GOBBLIN-1714] Use FileNotFoundException when determining files in source/target instead of generic IOException +* [GOBBLIN-1713] Add missing sql source validation +* [GOBBLIN-1712] Fail container for known transient exceptions to avoid data loss +* [GOBBLIN-1707] Add IcebergTableTest unit test +* [GOBBLIN-1708] Improve TimeAwareRecursiveCopyableDataset to lookback only into datefolders that match range +* [GOBBLIN-1710] Make Codecov optional in CI and not fail +* [GOBBLIN-1704] Purge offline helix instances during startup +* [GOBBLIN-1709] Create Iceberg Datasets Finder, Iceberg Dataset and FileSet to generate Copy Entities to support Distcp for Iceberg +* [GOBBLIN-1706] Add DagActionStore to store the action to kill/resume one flow execution +* [GOBBLIN-1705] New consumer service to monitor changes to FlowSpecStore +* [GOBBLIN-1702] Fix helix job wait completion bug when job goes to STOPPING state +* [GOBBLIN-1700] Remove unused coveralls-gradle-plugin dependency +* [GOBBLIN-1701] Replace jcenter with either maven central or gradle plugin portal +* [GOBBLIN-1699] Log progress of reducer task for visibility with slow compaction jobs +* [GOBBLIN-1695] Fix: Failure to add spec executors doesn't block deployment +* [GOBBLIN-1703] Avoid double quota increase for adhoc flows +* [GOBBLIN-1697] Have a separate resource handler to rely on CDC stream to do message forwarding +* [GOBBLIN-1696] Implement file based flowgraph that detects changes to the underlying files +* [GOBBLIN-1694] Add GMCE topic explicitly to hive commit event +* [GOBBLIN-1691] Add MysqlUserQuotaManager +* [GOBBLIN-1689] Decouple compiler from scheduler in warm standby mode +* [GOBBLIN-1690] Improve logging in ORC Writer +* [GOBBLIN-1698] Fast fail during work unit generation based on config +* [GOBBLIN-1686] Allow all Iceberg exceptions to be fault tolerant +* [GOBBLIN-1684] Stub for FileSystem based message buffer +* [GOBBLIN-1673]* [GOBBLIN-1683] Skeleton code for handling messages between task runner / application master for Dynamic work unit allocation +* [GOBBLIN-1681] Guard against exists fs call as well +* [GOBBLIN-1678] Refactor git flowgraph component to be extensible +* [GOBBLIN-1677] Fix timezone property to read from key correctly +* [GOBBLIN-1675] Add pagination for GaaS on server side +* [GOBBLIN-1672] Refactor metrics from DagManager into its own class, add metrics +* [GOBBLIN-1671] Fix gobblin.sh script to add external jars as colon separated to HADOOP_CLASSPATH +* [GOBBLIN-1670] Remove rat tasks and unneeded checkstyles blocking build pipeline +* [GOBBLIN-1669] Clean up TimeAwareRecursiveCopyableDataset to support seconds in time +* [GOBBLIN-1668] Add audit counts for iceberg registration +* [GOBBLIN-1667] Create new predicate - ExistingPartitionSkipPredicate +* [GOBBLIN-1667] Supporting for true ABORT on existing entity +* [GOBBLIN-1664] Allow table to flush after write failure +* [GOBBLIN-1663] Add some debug log lines around GMIP hive commit events +* [GOBBLIN-1662] Fix running counts for retried flows +* [GOBBLIN-1657] Update completion watermark on change_property in IcebergMetadataWriter +* [GOBBLIN-1656] Return a http status 503 on GaaS when quota is exceeded for user or flowgroup +* [GOBBLIN-1654] Add capacity floor to avoid aggressively requesting resource and small files +* [GOBBLIN-1653] Shorten job name length if it exceeds 255 characters +* [GOBBLIN-1652] Add more log in the KafkaJobStatusMonitor in case it fails to process one GobblinTrackingEvent +* [GOBBLIN-1651] Add config to set close timeout in HiveRegister +* [GOBBLIN-1650] Implement flowGroup quotas for the DagManager +* [GOBBLIN-1648] Complete use of JDBC DataSource 'read-only' validation query by incorporating where previously omitted +* [GOBBLIN-1647] Add hive commit GTE to HiveMetadataWriter +* [GOBBLIN-1644] Log assigned participant when helix participant check fails +* [GOBBLIN-1645] Change the prefix of dagManager heartbeat to make it consistent with other metrics +* [GOBBLIN-1641] Add meter for sla exceeded flows +* [GOBBLIN-1640] Add an API in AbstractBaseKafkaConsumerClient to list selected topics +* [GOBBLIN-1639] Prevent metrics reporting if configured, clean up workunit count metric +* [GOBBLIN-1638] Fix unbalanced running count metrics due to Azkaban failures +* [GOBBLIN-1637] Add writer, operation, and partition info to failed metadata writer events +* [GOBBLIN-1636] Close DatasetCleaner after clean task +* [GOBBLIN-1635] Avoid loading env configuration when using config store to improve the performance +* [GOBBLIN-1634] Add retries on flow sla kills +* [GOBBLIN-1633] Fix compaction actions on job failure not retried if compaction succeeds +* [GOBBLIN-1632] Use data node aliases to figure out data node names before using DMAS +* [GOBBLIN-1631] Emit heartbeat for dagManagerThread +* [GOBBLIN-1630] Remove flow level metrics for adhoc flows +* [GOBBLIN-1613] Add metadata writers field to GMCE schema +* [GOBBLIN-1629] Make GobblinMCEWriter be able to catch error when calculating hive specs +* [GOBBLIN-1620] Make yarn container allocation group by helix tag +* [GOBBLIN-1616] Add close connection logic in salseforceSource +* [GOBBLIN-1628] Add/fix some fields of MetadataWriterFailureEvent +* [GOBBLIN-1627] Provide option to convert datanodes names +* [GOBBLIN-1626] Use user supplied props to create FileSystem in DatasetCleanerTask +* [GOBBLIN-1625] Add coverage for edge cases when table paths do not exist, check parents +* [GOBBLIN-1624] Refactor quota management, fix various bugs in accounting of running jobs +* [GOBBLIN-1623] Fix NPE when try to close RestApiConnector +* [GOBBLIN-1622] Clear bad mysql packages from cache in CI/CD machines +* [GOBBLIN-1621] Make HelixRetriggeringJobCallable emit job skip event when job is dropped due to previous job is running +* [GOBBLIN-1619] WriterUtils.mkdirsWithRecursivePermission contains race condition and puts unnecessary load on filesystem +* [GOBBLIN-1617] Pass configurations to some HadoopUtils APIs +* [GOBBLIN-1616] Make RestApiConnector be able to close the connection finally +* [GOBBLIN-1615] Add config to set log level for any class +* [GOBBLIN-1614] Fix bug where partitioned tables would always return the wrong equali… +* [GOBBLIN-1612] Add description about downloading gradle wrapper +* [GOBBLIN-1611] Fix a wrong value for writer.codec.type in the document +* [GOBBLIN-1609] Don't flush on change_property operation +* [GOBBLIN-1608] Fix case where error GTE is incorrectly sent from MCE writer +* [GOBBLIN-1606] Change DEFAULT_GOBBLIN_COPY_CHECK_FILESIZE value +* [GOBBLIN-1605] Fix mysql ubuntu download 404 not found for Github Actions CI/CD +* [GOBBLIN-1604] Throw exception if there are no allocated requests due to lack of resources +* [GOBBLIN-1603] Throws error if configured when encountering an IO exception +* [GOBBLIN-1601] Implement ChangePermissionCommitStep +* [GOBBLIN-1598] Fix metrics already exist issue in dag manager +* [GOBBLIN-1597] Add error handling in dagmanager to continue if dag fails to process +* [GOBBLIN-1596] Ignore already exists exception if the table has already been created +* [GOBBLIN-1594] Add guard in DagManager for improperly formed SLA +* [GOBBLIN-1593] Fix bugs in dag manager about metric reporting and job status monitor +* [GOBBLIN-1592] Make hive copy be able to apply filter on directory +* [GOBBLIN-1591] Lazily initialize FileContext and do not hold a reference to it +* [GOBBLIN-1590] Add low/high watermark information in event emitted by Gobblin cluster +* [GOBBLIN-1589] Add FileContextFactory to cache FileContext instances +* [GOBBLIN-1588] Send failure events for write failures when watermark is advanced in MCE writer +* [GOBBLIN-1587] Bump version of code cov plugin +* [GOBBLIN-1585] Fix for GaaS (DagManager) keep retrying a failed job beyond max attempt number +* [GOBBLIN-1584] Add replace record logic for Mysql writer +* [GOBBLIN-1583] Add System level job start SLA +* [GOBBLIN-1582] Fill low/high watermark info in SourceState for QueryBasedSource +* [GOBBLIN-1581] Iterate over Sql ResultSet in Only the Forward Direction +* [GOBBLIN-1580] Check table exists instead of call create table directly to make sure table exists +* [GOBBLIN-1578] Avoid deletion of data while dropping a hive table +* [GOBBLIN-1577] Change the multiplier used in ExponentialWaitStrategy +* [GOBBLIN-1576] Skip appending record count to staging file +* [GOBBLIN-1575] Use reference count in helix manager, so that connect/disconnect are called once and at the right time +* [GOBBLIN-1574] Added whitelist for iceberg tables to add new partition +* [GOBBLIN-1573] Fix the ClassNotFoundException in streaming test pipeline +* [GOBBLIN-1565] Make GMCEWriter fault tolerant so that one topic failure will not affect other topics in the same container +* [GOBBLIN-1564] Codestyle changes, typo corrections, improved javadoc +* [GOBBLIN-1552] Determine flow status correctly when dag manager is disabled +* [GOBBLIN-1492] Optimize flowspec keys on configToProperties + GOBBLIN 0.16.0 -------------- From 949b01927e642a40a1f5c208926925988910c064 Mon Sep 17 00:00:00 2001 From: meethngala Date: Wed, 14 Jun 2023 16:39:12 -0700 Subject: [PATCH 09/30] [GOBBLIN-1835]Upgrade Iceberg Version from 0.11.1 to 1.2.0 (#3697) * upgrade iceberg version to 1.2.0 * address PR comments * resolve merge conflicts * fix checkstyle * update assert to verfiy error map elements in testFaultTolerant unit test --------- Co-authored-by: Meeth Gala --- gobblin-data-management/build.gradle | 2 +- .../management/copy/iceberg/IcebergTable.java | 2 +- gobblin-iceberg/build.gradle | 2 +- .../gobblin/iceberg/Utils/IcebergUtils.java | 2 + .../publisher/GobblinMCEPublisher.java | 2 +- .../iceberg/writer/IcebergMetadataWriter.java | 5 +- .../writer/HiveMetadataWriterTest.java | 1 - .../writer/IcebergMetadataWriterTest.java | 74 +++++++++---------- gradle/scripts/defaultBuildProperties.gradle | 2 +- gradle/scripts/dependencyDefinitions.gradle | 1 + 10 files changed, 48 insertions(+), 45 deletions(-) diff --git a/gobblin-data-management/build.gradle b/gobblin-data-management/build.gradle index c3b3129ea62..8b25f856246 100644 --- a/gobblin-data-management/build.gradle +++ b/gobblin-data-management/build.gradle @@ -45,7 +45,7 @@ dependencies { compile externalDependency.junit compile externalDependency.jacksonMapperAsl - testCompile(group: 'org.apache.iceberg', name: 'iceberg-hive-metastore', version: '0.11.1', classifier: 'tests') { + testCompile(externalDependency.icebergHiveMetastoreTest) { transitive = false } testCompile('org.apache.hadoop:hadoop-common:2.6.0') diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergTable.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergTable.java index 6671ebdeb64..529f53a45e1 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergTable.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/iceberg/IcebergTable.java @@ -161,7 +161,7 @@ protected IcebergSnapshotInfo createSnapshotInfo(Snapshot snapshot, Optional calcManifestFileInfo(m, tableOps.io()))` due to checked exception - skipManifestFileInfo ? Lists.newArrayList() : calcAllManifestFileInfos(snapshot.allManifests(), tableOps.io()) + skipManifestFileInfo ? Lists.newArrayList() : calcAllManifestFileInfos(snapshot.allManifests(tableOps.io()), tableOps.io()) ); } diff --git a/gobblin-iceberg/build.gradle b/gobblin-iceberg/build.gradle index fe030f80caf..c6626b299d3 100644 --- a/gobblin-iceberg/build.gradle +++ b/gobblin-iceberg/build.gradle @@ -48,7 +48,7 @@ dependencies { compile externalDependency.findBugsAnnotations compile externalDependency.avroMapredH2 - testCompile(group: 'org.apache.iceberg', name: 'iceberg-hive-metastore', version: '0.11.1', classifier: 'tests') { + testCompile(externalDependency.icebergHiveMetastoreTest) { transitive = false } testCompile('org.apache.hadoop:hadoop-common:2.6.0') diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/Utils/IcebergUtils.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/Utils/IcebergUtils.java index 65fb551ec8e..c7b528cb522 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/Utils/IcebergUtils.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/Utils/IcebergUtils.java @@ -248,6 +248,8 @@ public static DataFile getIcebergDataFileWithMetric(org.apache.gobblin.metadata. IcebergUtils.getMapFromIntegerLongPairs(file.getFileMetrics().getColumnSizes(), schemaIdMap), IcebergUtils.getMapFromIntegerLongPairs(file.getFileMetrics().getValueCounts(), schemaIdMap), IcebergUtils.getMapFromIntegerLongPairs(file.getFileMetrics().getNullValueCounts(), schemaIdMap), + // TODO: If required, handle NaN value count File metric conversion in ORC metrics with iceberg upgrade + IcebergUtils.getMapFromIntegerLongPairs(Lists.newArrayList(), schemaIdMap), // metric value will be null since Nan values are supported from avro version 1.10.* IcebergUtils.getMapFromIntegerBytesPairs(file.getFileMetrics().getLowerBounds(), schemaIdMap), IcebergUtils.getMapFromIntegerBytesPairs(file.getFileMetrics().getUpperBounds(), schemaIdMap)); return dataFileBuilder.withMetrics(metrics).build(); diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/publisher/GobblinMCEPublisher.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/publisher/GobblinMCEPublisher.java index 47616529f86..ffeefaa0866 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/publisher/GobblinMCEPublisher.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/publisher/GobblinMCEPublisher.java @@ -219,7 +219,7 @@ public static Metrics getMetrics(State state, Path path, Configuration conf, Nam } case AVRO: { try { - return new Metrics(100000000L, null, null, null); + return new Metrics(100000000L, null, null, null, null); } catch (Exception e) { throw new RuntimeException("Cannot get file information for file " + path.toString(), e); } diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java index 9c3d30e9e50..ecd528323e2 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java @@ -58,6 +58,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils; import org.apache.iceberg.AppendFiles; +import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFiles; import org.apache.iceberg.ExpireSnapshots; @@ -77,7 +78,7 @@ import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; -import org.apache.iceberg.hive.HiveCatalogs; +import org.apache.iceberg.hive.HiveCatalog; import org.apache.iceberg.types.Types; import org.joda.time.DateTime; import org.joda.time.format.PeriodFormatter; @@ -253,7 +254,7 @@ protected void setAuditCountVerifier(KafkaAuditCountVerifier verifier) { } protected void initializeCatalog() { - catalog = HiveCatalogs.loadCatalog(conf); + catalog = CatalogUtil.loadCatalog(HiveCatalog.class.getName(), "HiveCatalog", new HashMap<>(), conf); } private org.apache.iceberg.Table getIcebergTable(TableIdentifier tid) throws NoSuchTableException { diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java index 4225ce5ce4a..f50bc81aa9a 100644 --- a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java +++ b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java @@ -128,7 +128,6 @@ public void clean() throws Exception { @BeforeSuite public void setUp() throws Exception { Class.forName("org.apache.derby.jdbc.EmbeddedDriver").newInstance(); - startMetastore(); State state = ConfigUtils.configToState(ConfigUtils.propertiesToConfig(hiveConf.getAllProperties())); Optional metastoreUri = Optional.fromNullable(state.getProperties().getProperty(HiveRegister.HIVE_METASTORE_URI_KEY)); hc = HiveMetastoreClientPool.get(state.getProperties(), metastoreUri); diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java index bc1c8ca5706..bf7dcb3833f 100644 --- a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java +++ b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java @@ -124,8 +124,6 @@ public void clean() throws Exception { @BeforeClass public void setUp() throws Exception { Class.forName("org.apache.derby.jdbc.EmbeddedDriver").newInstance(); - startMetastore(); - tmpDir = Files.createTempDir(); hourlyDataFile_1 = new File(tmpDir, "testDB/testTopic/hourly/2020/03/17/08/data.avro"); Files.createParentDirs(hourlyDataFile_1); @@ -236,7 +234,7 @@ public void testWriteAddFileGMCE() throws IOException { gobblinMCEWriter.flush(); table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); Assert.assertEquals(table.properties().get("offset.range.testTopic-1"), "0-2000"); - Assert.assertEquals(table.currentSnapshot().allManifests().size(), 1); + Assert.assertEquals(table.currentSnapshot().allManifests(table.io()).size(), 1); // Assert low watermark and high watermark set properly Assert.assertEquals(table.properties().get("gmce.low.watermark.GobblinMetadataChangeEvent_test-1"), "9"); Assert.assertEquals(table.properties().get("gmce.high.watermark.GobblinMetadataChangeEvent_test-1"), "20"); @@ -256,7 +254,7 @@ public void testWriteAddFileGMCE() throws IOException { gobblinMCEWriter.flush(); table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); Assert.assertEquals(table.properties().get("offset.range.testTopic-1"), "0-3000"); - Assert.assertEquals(table.currentSnapshot().allManifests().size(), 2); + Assert.assertEquals(table.currentSnapshot().allManifests(table.io()).size(), 2); Assert.assertEquals(table.properties().get("gmce.low.watermark.GobblinMetadataChangeEvent_test-1"), "20"); Assert.assertEquals(table.properties().get("gmce.high.watermark.GobblinMetadataChangeEvent_test-1"), "30"); @@ -269,7 +267,7 @@ public void testWriteAddFileGMCE() throws IOException { gobblinMCEWriter.flush(); table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); Assert.assertEquals(table.properties().get("offset.range.testTopic-1"), "0-3000"); - Assert.assertEquals(table.currentSnapshot().allManifests().size(), 2); + Assert.assertEquals(table.currentSnapshot().allManifests(table.io()).size(), 2); } //Make sure hive test execute later and close the metastore @@ -290,7 +288,7 @@ public void testWriteRewriteFileGMCE() throws IOException { Table table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); Iterator result = FindFiles.in(table).withMetadataMatching(Expressions.startsWith("file_path", filePath_1)).collect().iterator(); - Assert.assertEquals(table.currentSnapshot().allManifests().size(), 2); + Assert.assertEquals(table.currentSnapshot().allManifests(table.io()).size(), 2); Assert.assertTrue(result.hasNext()); GenericRecord genericGmce = GenericData.get().deepCopy(gmce.getSchema(), gmce); gobblinMCEWriter.writeEnvelope(new RecordEnvelope<>(genericGmce, @@ -313,7 +311,7 @@ public void testWriteRewriteFileGMCE() throws IOException { public void testChangeProperty() throws IOException { Table table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); Assert.assertEquals(table.properties().get("offset.range.testTopic-1"), "0-3000"); - Assert.assertEquals(table.currentSnapshot().allManifests().size(), 3); + Assert.assertEquals(table.currentSnapshot().allManifests(table.io()).size(), 3); Assert.assertEquals(table.properties().get("gmce.low.watermark.GobblinMetadataChangeEvent_test-1"), "30"); Assert.assertEquals(table.properties().get("gmce.high.watermark.GobblinMetadataChangeEvent_test-1"), "40"); @@ -335,7 +333,7 @@ public void testChangeProperty() throws IOException { table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); // Assert the offset has been updated Assert.assertEquals(table.properties().get("offset.range.testTopic-1"), "0-4000"); - Assert.assertEquals(table.currentSnapshot().allManifests().size(), 3); + Assert.assertEquals(table.currentSnapshot().allManifests(table.io()).size(), 3); // Assert low watermark and high watermark set properly Assert.assertEquals(table.properties().get("gmce.low.watermark.GobblinMetadataChangeEvent_test-1"), "40"); Assert.assertEquals(table.properties().get("gmce.high.watermark.GobblinMetadataChangeEvent_test-1"), "45"); @@ -367,11 +365,7 @@ public void testFaultTolerant() throws Exception { Assert.assertEquals(gobblinMCEWriter.getDatasetErrorMap().values().iterator().next().size(), 1); Assert.assertEquals(gobblinMCEWriter.getDatasetErrorMap() .get(new File(tmpDir, "testDB/testTopic").getAbsolutePath()) - .get("hivedb.testTopic").get(0).lowWatermark, 50L); - Assert.assertEquals(gobblinMCEWriter.getDatasetErrorMap() - .get(new File(tmpDir, "testDB/testTopic").getAbsolutePath()) - .get("hivedb.testTopic").get(0).highWatermark, 52L); - + .get("hivedb.testTopicCompleteness").get(0).getMessage(), "failed to flush table hivedb, testTopicCompleteness"); // No events sent yet since the topic has not been flushed Assert.assertEquals(eventsSent.size(), 0); @@ -381,7 +375,7 @@ public void testFaultTolerant() throws Exception { // Since this topic has been flushed, there should be an event sent for previous failure, and the table // should be removed from the error map Assert.assertEquals(eventsSent.size(), 1); - Assert.assertEquals(eventsSent.get(0).getMetadata().get(MetadataWriterKeys.TABLE_NAME_KEY), "testTopic"); + Assert.assertEquals(eventsSent.get(0).getMetadata().get(MetadataWriterKeys.TABLE_NAME_KEY), "testTopicCompleteness"); Assert.assertEquals(eventsSent.get(0).getMetadata().get(MetadataWriterKeys.GMCE_LOW_WATERMARK), "50"); Assert.assertEquals(eventsSent.get(0).getMetadata().get(MetadataWriterKeys.GMCE_HIGH_WATERMARK), "52"); Assert.assertEquals(gobblinMCEWriter.getDatasetErrorMap().values().iterator().next().size(), 0); @@ -401,7 +395,7 @@ public void testWriteAddFileGMCECompleteness() throws IOException { // Creating a copy of gmce with static type in GenericRecord to work with writeEnvelop method // without risking running into type cast runtime error. gmce.setOperationType(OperationType.add_files); - File hourlyFile = new File(tmpDir, "testDB/testTopic/hourly/2021/09/16/10/data.avro"); + File hourlyFile = new File(tmpDir, "testDB/testTopicCompleteness/hourly/2021/09/16/10/data.avro"); long timestampMillis = 1631811600000L; Files.createParentDirs(hourlyFile); writeRecord(hourlyFile); @@ -410,25 +404,23 @@ public void testWriteAddFileGMCECompleteness() throws IOException { .setFileFormat("avro") .setFileMetrics(DataMetrics.newBuilder().setRecordCount(10L).build()) .build())); - gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopic-1", "3000-4000").build()); + gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopicCompleteness-1", "3000-4000").build()); GenericRecord genericGmce_3000_4000 = GenericData.get().deepCopy(gmce.getSchema(), gmce); gobblinMCEWriterWithCompletness.writeEnvelope(new RecordEnvelope<>(genericGmce_3000_4000, new KafkaStreamingExtractor.KafkaWatermark( new KafkaPartition.Builder().withTopicName("GobblinMetadataChangeEvent_test").withId(1).build(), new LongWatermark(50L)))); - - Table table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); - Assert.assertEquals(table.properties().get("offset.range.testTopic-1"), "0-4000"); + Table table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(1)); Assert.assertTrue(table.spec().fields().size() == 2); Assert.assertEquals(table.spec().fields().get(1).name(), "late"); // Test when completeness watermark = -1 bootstrap case KafkaAuditCountVerifier verifier = Mockito.mock(TestAuditCountVerifier.class); - Mockito.when(verifier.isComplete("testTopic", timestampMillis - TimeUnit.HOURS.toMillis(1), timestampMillis)).thenReturn(true); + Mockito.when(verifier.isComplete("testTopicCompleteness", timestampMillis - TimeUnit.HOURS.toMillis(1), timestampMillis)).thenReturn(true); IcebergMetadataWriter imw = (IcebergMetadataWriter) gobblinMCEWriterWithCompletness.metadataWriters.iterator().next(); imw.setAuditCountVerifier(verifier); gobblinMCEWriterWithCompletness.flush(); - table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); + table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(1)); //completeness watermark = "2020-09-16-10" Assert.assertEquals(table.properties().get(TOPIC_NAME_KEY), "testTopic"); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_TIMEZONE_KEY), "America/Los_Angeles"); @@ -440,7 +432,7 @@ public void testWriteAddFileGMCECompleteness() throws IOException { Assert.assertTrue(dfl.hasNext()); // Test when completeness watermark is still "2021-09-16-10" but have a late file for "2021-09-16-09" - File hourlyFile1 = new File(tmpDir, "testDB/testTopic/hourly/2021/09/16/09/data1.avro"); + File hourlyFile1 = new File(tmpDir, "testDB/testTopicCompleteness/hourly/2021/09/16/09/data1.avro"); Files.createParentDirs(hourlyFile1); writeRecord(hourlyFile1); gmce.setNewFiles(Lists.newArrayList(DataFile.newBuilder() @@ -448,14 +440,14 @@ public void testWriteAddFileGMCECompleteness() throws IOException { .setFileFormat("avro") .setFileMetrics(DataMetrics.newBuilder().setRecordCount(10L).build()) .build())); - gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopic-1", "4000-5000").build()); + gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopicCompleteness-1", "4000-5000").build()); GenericRecord genericGmce_4000_5000 = GenericData.get().deepCopy(gmce.getSchema(), gmce); gobblinMCEWriterWithCompletness.writeEnvelope(new RecordEnvelope<>(genericGmce_4000_5000, new KafkaStreamingExtractor.KafkaWatermark( new KafkaPartition.Builder().withTopicName("GobblinMetadataChangeEvent_test").withId(1).build(), new LongWatermark(55L)))); gobblinMCEWriterWithCompletness.flush(); - table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); + table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(1)); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_KEY), String.valueOf(timestampMillis)); dfl = FindFiles.in(table).withMetadataMatching(Expressions.startsWith("file_path", hourlyFile1.getAbsolutePath())).collect().iterator(); @@ -463,7 +455,7 @@ public void testWriteAddFileGMCECompleteness() throws IOException { Assert.assertEquals((int) dfl.next().partition().get(1, Integer.class), 1); // Test when completeness watermark will advance to "2021-09-16-11" - File hourlyFile2 = new File(tmpDir, "testDB/testTopic/hourly/2021/09/16/11/data.avro"); + File hourlyFile2 = new File(tmpDir, "testDB/testTopicCompleteness/hourly/2021/09/16/11/data.avro"); long timestampMillis1 = timestampMillis + TimeUnit.HOURS.toMillis(1); Files.createParentDirs(hourlyFile2); writeRecord(hourlyFile2); @@ -472,16 +464,16 @@ public void testWriteAddFileGMCECompleteness() throws IOException { .setFileFormat("avro") .setFileMetrics(DataMetrics.newBuilder().setRecordCount(10L).build()) .build())); - gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopic-1", "5000-6000").build()); + gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopicCompleteness-1", "5000-6000").build()); GenericRecord genericGmce_5000_6000 = GenericData.get().deepCopy(gmce.getSchema(), gmce); gobblinMCEWriterWithCompletness.writeEnvelope(new RecordEnvelope<>(genericGmce_5000_6000, new KafkaStreamingExtractor.KafkaWatermark( new KafkaPartition.Builder().withTopicName("GobblinMetadataChangeEvent_test").withId(1).build(), new LongWatermark(60L)))); - Mockito.when(verifier.isComplete("testTopic", timestampMillis1 - TimeUnit.HOURS.toMillis(1), timestampMillis1)).thenReturn(true); + Mockito.when(verifier.isComplete("testTopicCompleteness", timestampMillis1 - TimeUnit.HOURS.toMillis(1), timestampMillis1)).thenReturn(true); gobblinMCEWriterWithCompletness.flush(); - table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); + table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(1)); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_KEY), String.valueOf(timestampMillis1)); // watermark 1631815200000L correspond to 2021-09-16-11 in PT Assert.assertEquals(imw.state.getPropAsLong(String.format(STATE_COMPLETION_WATERMARK_KEY_OF_TABLE, table.name().toLowerCase(Locale.ROOT))), 1631815200000L); @@ -498,7 +490,7 @@ public void testChangePropertyGMCECompleteness() throws IOException { ZonedDateTime expectedCWDt = ZonedDateTime.now(ZoneId.of(DEFAULT_TIME_ZONE)).truncatedTo(ChronoUnit.HOURS); // For quiet topics, watermark should always be beginning of current hour long expectedWatermark = expectedCWDt.toInstant().toEpochMilli(); - File hourlyFile2 = new File(tmpDir, "testDB/testTopic/hourly/2021/09/16/11/data.avro"); + File hourlyFile2 = new File(tmpDir, "testDB/testTopicCompleteness/hourly/2021/09/16/11/data.avro"); gmce.setOldFilePrefixes(null); gmce.setNewFiles(Lists.newArrayList(DataFile.newBuilder() .setFilePath(hourlyFile2.toString()) @@ -506,7 +498,7 @@ public void testChangePropertyGMCECompleteness() throws IOException { .setFileMetrics(DataMetrics.newBuilder().setRecordCount(10L).build()) .build())); gmce.setOperationType(OperationType.change_property); - gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopic-1", "6000-7000").build()); + gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopicCompleteness-1", "6000-7000").build()); GenericRecord genericGmce = GenericData.get().deepCopy(gmce.getSchema(), gmce); gobblinMCEWriterWithCompletness.writeEnvelope(new RecordEnvelope<>(genericGmce, new KafkaStreamingExtractor.KafkaWatermark( @@ -515,12 +507,12 @@ public void testChangePropertyGMCECompleteness() throws IOException { KafkaAuditCountVerifier verifier = Mockito.mock(TestAuditCountVerifier.class); // For quiet topics always check for previous hour window - Mockito.when(verifier.isComplete("testTopic", expectedCWDt.minusHours(1).toInstant().toEpochMilli(), expectedWatermark)).thenReturn(true); + Mockito.when(verifier.isComplete("testTopicCompleteness", expectedCWDt.minusHours(1).toInstant().toEpochMilli(), expectedWatermark)).thenReturn(true); ((IcebergMetadataWriter) gobblinMCEWriterWithCompletness.metadataWriters.iterator().next()).setAuditCountVerifier(verifier); gobblinMCEWriterWithCompletness.flush(); - Table table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); - Assert.assertEquals(table.properties().get("offset.range.testTopic-1"), "0-7000"); + Table table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(1)); + Assert.assertEquals(table.properties().get("offset.range.testTopicCompleteness-1"), "3000-7000"); Assert.assertEquals(table.spec().fields().get(1).name(), "late"); Assert.assertEquals(table.properties().get(TOPIC_NAME_KEY), "testTopic"); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_TIMEZONE_KEY), "America/Los_Angeles"); @@ -562,15 +554,20 @@ protected Optional getPartition(Path path, HiveTable table) throw partitionValue = "2020-03-17-00"; } return Optional.of(new HivePartition.Builder().withPartitionValues(Lists.newArrayList(partitionValue)) - .withDbName("hivedb").withTableName("testTopic").build()); + .withDbName("hivedb").withTableName(table.getTableName()).build()); } @Override protected List getTables(Path path) throws IOException { List tables = super.getTables(path); for (HiveTable table : tables) { - table.setPartitionKeys(ImmutableList.of( - new HiveRegistrationUnit.Column("datepartition", serdeConstants.STRING_TYPE_NAME, StringUtils.EMPTY))); - //table.setLocation(tmpDir.getAbsolutePath()); + if (table.getTableName().equals("testTopicCompleteness")) { + table.setPartitionKeys(ImmutableList.of( + new HiveRegistrationUnit.Column("datepartition", serdeConstants.STRING_TYPE_NAME, StringUtils.EMPTY) + , new HiveRegistrationUnit.Column("late", serdeConstants.INT_TYPE_NAME, StringUtils.EMPTY))); + } else { + table.setPartitionKeys(ImmutableList.of(new HiveRegistrationUnit.Column("datepartition", serdeConstants.STRING_TYPE_NAME, StringUtils.EMPTY))); + //table.setLocation(tmpDir.getAbsolutePath()); + } } return tables; } @@ -581,6 +578,9 @@ protected List getTableNames(Optional dbPrefix, Path path) { if (path.toString().contains("testFaultTolerant")) { return Lists.newArrayList("testFaultTolerantIcebergTable"); } + else if (path.toString().contains("testTopicCompleteness")) { + return Lists.newArrayList("testTopicCompleteness"); + } return Lists.newArrayList("testTopic"); } } diff --git a/gradle/scripts/defaultBuildProperties.gradle b/gradle/scripts/defaultBuildProperties.gradle index 950c8780f55..f7b5f9cb196 100644 --- a/gradle/scripts/defaultBuildProperties.gradle +++ b/gradle/scripts/defaultBuildProperties.gradle @@ -31,7 +31,7 @@ def BuildProperties BUILD_PROPERTIES = new BuildProperties(project) .register(new BuildProperty("gobblinFlavor", "standard", "Build flavor (see http://gobblin.readthedocs.io/en/latest/developer-guide/GobblinModules/)")) .register(new BuildProperty("hadoopVersion", "2.10.0", "Hadoop dependencies version")) .register(new BuildProperty("hiveVersion", "1.0.1-avro", "Hive dependencies version")) - .register(new BuildProperty("icebergVersion", "0.11.1", "Iceberg dependencies version")) + .register(new BuildProperty("icebergVersion", "1.2.0", "Iceberg dependencies version")) .register(new BuildProperty("jdkVersion", JavaVersion.VERSION_1_8.toString(), "Java languange compatibility; supported versions: " + JavaVersion.VERSION_1_8)) .register(new BuildProperty("kafka08Version", "0.8.2.2", "Kafka 0.8 dependencies version")) diff --git a/gradle/scripts/dependencyDefinitions.gradle b/gradle/scripts/dependencyDefinitions.gradle index 95f8f6c3287..9c34ef02890 100644 --- a/gradle/scripts/dependencyDefinitions.gradle +++ b/gradle/scripts/dependencyDefinitions.gradle @@ -82,6 +82,7 @@ ext.externalDependency = [ "httpcore": "org.apache.httpcomponents:httpcore:4.4.11", "httpasyncclient": "org.apache.httpcomponents:httpasyncclient:4.1.3", "icebergHive": "org.apache.iceberg:iceberg-hive-runtime:" + icebergVersion, + "icebergHiveMetastoreTest": "org.apache.iceberg:iceberg-hive-metastore:" + icebergVersion + ":tests", "jgit": "org.eclipse.jgit:org.eclipse.jgit:5.1.1.201809181055-r", "jmh": "org.openjdk.jmh:jmh-core:1.17.3", "jmhAnnotations": "org.openjdk.jmh:jmh-generator-annprocess:1.17.3", From faa3a4f60b6ef763d620245b1535b75728a9802e Mon Sep 17 00:00:00 2001 From: umustafi Date: Thu, 15 Jun 2023 10:10:07 -0700 Subject: [PATCH 10/30] [GOBBLIN-1837] Implement multi-active, non blocking for leader host (#3700) * basic outline of changes to make, started SchedulerLeaseDeterminationStore * multiple query options for store * add launch type as column * wip for scheduler abstractions * non blocking algo impl, dag action store updates * DagActionMonitor changes to handle LAUNCH events * clean up comments, add docstrings * redefined lease arbiter & algo handler to separate scheduler specific logic from general lease handler * Address second round of review comments * Cleanup in response to review, fix failing test * small clean ups --------- Co-authored-by: Urmi Mustafi --- .../configuration/ConfigurationKeys.java | 15 + .../gobblin/service/ServiceConfigKeys.java | 1 + .../main/avro/DagActionStoreChangeEvent.avsc | 18 + .../gobblin/runtime/api/DagActionStore.java | 55 +-- .../runtime/api/MultiActiveLeaseArbiter.java | 103 +++++ .../api/MysqlMultiActiveLeaseArbiter.java | 391 ++++++++++++++++++ .../dag_action_store/MysqlDagActionStore.java | 64 ++- .../runtime/metrics/RuntimeMetrics.java | 3 + .../gobblin/runtime/util/InjectionNames.java | 3 + .../gobblin/scheduler/JobScheduler.java | 15 +- .../MysqlDagActionStoreTest.java | 43 +- .../core/GobblinServiceConfiguration.java | 4 + .../core/GobblinServiceGuiceModule.java | 12 + .../modules/orchestration/DagManager.java | 21 +- .../orchestration/FlowTriggerHandler.java | 179 ++++++++ .../modules/orchestration/Orchestrator.java | 62 ++- .../orchestration/TimingEventUtils.java | 4 +- ...ecutionResourceHandlerWithWarmStandby.java | 53 +-- .../scheduler/GobblinServiceJobScheduler.java | 20 +- .../DagActionStoreChangeMonitor.java | 89 ++-- .../DagActionStoreChangeMonitorFactory.java | 17 +- .../SpecStoreChangeMonitorFactory.java | 2 +- .../orchestration/DagManagerFlowTest.java | 6 +- .../orchestration/OrchestratorTest.java | 10 +- .../GobblinServiceJobSchedulerTest.java | 10 +- 25 files changed, 1000 insertions(+), 200 deletions(-) create mode 100644 gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MultiActiveLeaseArbiter.java create mode 100644 gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java create mode 100644 gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java diff --git a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java index f0e15bf94a4..b155e8089bb 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java @@ -95,6 +95,21 @@ public class ConfigurationKeys { public static final int DEFAULT_LOAD_SPEC_BATCH_SIZE = 500; public static final String SKIP_SCHEDULING_FLOWS_AFTER_NUM_DAYS = "skip.scheduling.flows.after.num.days"; public static final int DEFAULT_NUM_DAYS_TO_SKIP_AFTER = 365; + // Scheduler lease determination store configuration + public static final String MYSQL_LEASE_ARBITER_PREFIX = "MysqlMultiActiveLeaseArbiter"; + public static final String MULTI_ACTIVE_SCHEDULER_CONSTANTS_DB_TABLE_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".constantsTable"; + public static final String DEFAULT_MULTI_ACTIVE_SCHEDULER_CONSTANTS_DB_TABLE = MYSQL_LEASE_ARBITER_PREFIX + ".gobblin_multi_active_scheduler_constants_store"; + public static final String SCHEDULER_LEASE_DETERMINATION_STORE_DB_TABLE_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".schedulerLeaseArbiterTable"; + public static final String DEFAULT_SCHEDULER_LEASE_DETERMINATION_STORE_DB_TABLE = MYSQL_LEASE_ARBITER_PREFIX + ".gobblin_scheduler_lease_determination_store"; + public static final String SCHEDULER_EVENT_TO_REVISIT_TIMESTAMP_MILLIS_KEY = "eventToRevisitTimestampMillis"; + public static final String SCHEDULER_EVENT_TO_TRIGGER_TIMESTAMP_MILLIS_KEY = "triggerEventTimestampMillis"; + public static final String SCHEDULER_EVENT_EPSILON_MILLIS_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".epsilonMillis"; + public static final int DEFAULT_SCHEDULER_EVENT_EPSILON_MILLIS = 5000; + // Note: linger should be on the order of seconds even though we measure in millis + public static final String SCHEDULER_EVENT_LINGER_MILLIS_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".lingerMillis"; + public static final int DEFAULT_SCHEDULER_EVENT_LINGER_MILLIS = 30000; + public static final String SCHEDULER_MAX_BACKOFF_MILLIS_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".maxBackoffMillis"; + public static final int DEFAULT_SCHEDULER_MAX_BACKOFF_MILLIS = 5000; // Job executor thread pool size public static final String JOB_EXECUTOR_THREAD_POOL_SIZE_KEY = "jobexecutor.threadpool.size"; diff --git a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java index ef4323538d9..21b32b58cc0 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java @@ -41,6 +41,7 @@ public class ServiceConfigKeys { public static final boolean DEFAULT_GOBBLIN_SERVICE_DAG_MANAGER_ENABLED = false; public static final String GOBBLIN_SERVICE_JOB_STATUS_MONITOR_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "jobStatusMonitor.enabled"; public static final String GOBBLIN_SERVICE_WARM_STANDBY_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "warmStandby.enabled"; + public static final String GOBBLIN_SERVICE_MULTI_ACTIVE_SCHEDULER_ENABLED_KEY = GOBBLIN_SERVICE_PREFIX + "multiActiveScheduler.enabled"; // If true, will mark up/down d2 servers on leadership so that all requests will be routed to the leader node public static final String GOBBLIN_SERVICE_D2_ONLY_ANNOUNCE_LEADER = GOBBLIN_SERVICE_PREFIX + "d2.onlyAnnounceLeader"; diff --git a/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/DagActionStoreChangeEvent.avsc b/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/DagActionStoreChangeEvent.avsc index 268f18ad049..b628f17146e 100644 --- a/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/DagActionStoreChangeEvent.avsc +++ b/gobblin-metrics-libs/gobblin-metrics-base/src/main/avro/DagActionStoreChangeEvent.avsc @@ -23,6 +23,24 @@ "type" : "string", "doc" : "flow execution id for the dag action", "compliance" : "NONE" + }, { + "name" : "dagAction", + "type": { + "type": "enum", + "name": "DagActionValue", + "symbols": [ + "KILL", + "RESUME", + "LAUNCH" + ], + "symbolDocs": { + "KILL": "Kill the flow corresponding to this dag", + "RESUME": "Resume or start a new flow corresponding to this dag", + "LAUNCH": "Launch a new execution of the flow corresponding to this dag" + } + }, + "doc" : "type of dag action", + "compliance" : "NONE" } ] } \ No newline at end of file diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/DagActionStore.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/DagActionStore.java index 5da8e6d31d8..a1a0ea237e6 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/DagActionStore.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/DagActionStore.java @@ -20,29 +20,26 @@ import java.io.IOException; import java.sql.SQLException; import java.util.Collection; -import lombok.EqualsAndHashCode; -import lombok.Getter; + +import lombok.Data; public interface DagActionStore { - enum DagActionValue { - KILL, - RESUME + enum FlowActionType { + KILL, // Kill invoked through API call + RESUME, // Resume flow invoked through API call + LAUNCH, // Launch new flow execution invoked adhoc or through scheduled trigger + RETRY, // Invoked through DagManager for flows configured to allow retries + CANCEL, // Invoked through DagManager if flow has been stuck in Orchestrated state for a while + ADVANCE // Launch next step in multi-hop dag } - @Getter - @EqualsAndHashCode + @Data class DagAction { - String flowGroup; - String flowName; - String flowExecutionId; - DagActionValue dagActionValue; - public DagAction(String flowGroup, String flowName, String flowExecutionId, DagActionValue dagActionValue) { - this.flowGroup = flowGroup; - this.flowName = flowName; - this.flowExecutionId = flowExecutionId; - this.dagActionValue = dagActionValue; - } + final String flowGroup; + final String flowName; + final String flowExecutionId; + final FlowActionType flowActionType; } @@ -51,40 +48,28 @@ public DagAction(String flowGroup, String flowName, String flowExecutionId, DagA * @param flowGroup flow group for the dag action * @param flowName flow name for the dag action * @param flowExecutionId flow execution for the dag action + * @param flowActionType the value of the dag action * @throws IOException */ - boolean exists(String flowGroup, String flowName, String flowExecutionId) throws IOException, SQLException; + boolean exists(String flowGroup, String flowName, String flowExecutionId, FlowActionType flowActionType) throws IOException, SQLException; /** * Persist the dag action in {@link DagActionStore} for durability * @param flowGroup flow group for the dag action * @param flowName flow name for the dag action * @param flowExecutionId flow execution for the dag action - * @param dagActionValue the value of the dag action + * @param flowActionType the value of the dag action * @throws IOException */ - void addDagAction(String flowGroup, String flowName, String flowExecutionId, DagActionValue dagActionValue) throws IOException; + void addDagAction(String flowGroup, String flowName, String flowExecutionId, FlowActionType flowActionType) throws IOException; /** * delete the dag action from {@link DagActionStore} - * @param flowGroup flow group for the dag action - * @param flowName flow name for the dag action - * @param flowExecutionId flow execution for the dag action + * @param DagAction containing all information needed to identify dag and specific action value * @throws IOException * @return true if we successfully delete one record, return false if the record does not exist */ - boolean deleteDagAction(String flowGroup, String flowName, String flowExecutionId) throws IOException; - - /*** - * Retrieve action value by the flow group, flow name and flow execution id from the {@link DagActionStore}. - * @param flowGroup flow group for the dag action - * @param flowName flow name for the dag action - * @param flowExecutionId flow execution for the dag action - * @throws IOException Exception in retrieving the {@link DagAction}. - * @throws SpecNotFoundException If {@link DagAction} being retrieved is not present in store. - */ - DagAction getDagAction(String flowGroup, String flowName, String flowExecutionId) throws IOException, SpecNotFoundException, - SQLException; + boolean deleteDagAction(DagAction dagAction) throws IOException; /*** * Get all {@link DagAction}s from the {@link DagActionStore}. diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MultiActiveLeaseArbiter.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MultiActiveLeaseArbiter.java new file mode 100644 index 00000000000..ab9e03599b5 --- /dev/null +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MultiActiveLeaseArbiter.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.runtime.api; + +import java.io.IOException; + +import lombok.Data; + + +/** + * This interface defines a generic approach to a non-blocking, multiple active thread or host system, in which one or + * more active participants compete to take responsiblity for a particular flow's event. The type of flow event in + * question does not impact the algorithm other than to uniquely identify the flow event. Each participant uses the + * interface to initiate an attempt at ownership over the flow event and receives a response indicating the status of + * the attempt. + * + * At a high level the lease arbiter works as follows: + * 1. Multiple participants independently learn of a flow action event to act upon + * 2. Each participant attempts to acquire rights or `a lease` to be the sole participant acting on the event by + * calling the tryAcquireLease method below and receives the resulting status. The status indicates whether this + * participant has + * a) LeaseObtainedStatus -> this participant will attempt to carry out the required action before the lease expires + * b) LeasedToAnotherStatus -> another will attempt to carry out the required action before the lease expires + * c) NoLongerLeasingStatus -> flow event no longer needs to be acted upon (terminal state) + * 3. If another participant has acquired the lease before this one could, then the present participant must check back + * in at the time of lease expiry to see if it needs to attempt the lease again [status (b) above]. + * 4. Once the participant which acquired the lease completes its work on the flow event, it calls recordLeaseSuccess + * to indicate to all other participants that the flow event no longer needs to be acted upon [status (c) above] + */ +public interface MultiActiveLeaseArbiter { + /** + * This method attempts to insert an entry into store for a particular flow action event if one does not already + * exist in the store for the flow action or has expired. Regardless of the outcome it also reads the lease + * acquisition timestamp of the entry for that flow action event (it could have pre-existed in the table or been newly + * added by the previous write). Based on the transaction results, it will return {@link LeaseAttemptStatus} to + * determine the next action. + * @param flowAction uniquely identifies the flow and the present action upon it + * @param eventTimeMillis is the time this flow action was triggered + * @return LeaseAttemptStatus + * @throws IOException + */ + LeaseAttemptStatus tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis) throws IOException; + + /** + * This method is used to indicate the owner of the lease has successfully completed required actions while holding + * the lease of the flow action event. It marks the lease as "no longer leasing", if the eventTimeMillis and + * leaseAcquisitionTimeMillis values have not changed since this owner acquired the lease (indicating the lease did + * not expire). + * @return true if successfully updated, indicating no further actions need to be taken regarding this event. + * false if failed to update the lease properly, the caller should continue seeking to acquire the lease as + * if any actions it did successfully accomplish, do not count + */ + boolean recordLeaseSuccess(LeaseObtainedStatus status) throws IOException; + + /* + Class used to encapsulate status of lease acquisition attempt and derivations should contain information specific to + the status that results. + */ + abstract class LeaseAttemptStatus {} + + class NoLongerLeasingStatus extends LeaseAttemptStatus {} + + /* + The participant calling this method acquired the lease for the event in question. The class contains the + `eventTimestamp` associated with the lease as well as the time the caller obtained the lease or + `leaseAcquisitionTimestamp`. + */ + @Data + class LeaseObtainedStatus extends LeaseAttemptStatus { + private final DagActionStore.DagAction flowAction; + private final long eventTimestamp; + private final long leaseAcquisitionTimestamp; + } + + /* + This flow action event already has a valid lease owned by another participant. + `eventTimeMillis` is the timestamp the lease is associated with, which may be a different timestamp for the same flow + action corresponding to the same instance of the event or a distinct one. + `minimumLingerDurationMillis` is the minimum amount of time to wait before this participant should return to check if + the lease has completed or expired + */ + @Data + class LeasedToAnotherStatus extends LeaseAttemptStatus { + private final DagActionStore.DagAction flowAction; + private final long eventTimeMillis; + private final long minimumLingerDurationMillis; +} +} diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java new file mode 100644 index 00000000000..8a40c71b2e5 --- /dev/null +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.runtime.api; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; + +import com.google.inject.Inject; +import com.typesafe.config.Config; +import com.zaxxer.hikari.HikariDataSource; + +import javax.sql.DataSource; +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.broker.SharedResourcesBrokerFactory; +import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.metastore.MysqlDataSourceFactory; +import org.apache.gobblin.service.ServiceConfigKeys; +import org.apache.gobblin.util.ConfigUtils; + + +/** + * MySQL based implementation of the {@link MultiActiveLeaseArbiter} which uses a MySQL store to resolve ownership of + * a flow event amongst multiple competing participants. A MySQL table is used to store flow identifying information as + * well as the flow action associated with it. It uses two additional values of the `event_timestamp` and + * `lease_acquisition_timestamp` to indicate an active lease, expired lease, and state of no longer leasing. The table + * schema is as follows: + * [flow_group | flow_name | flow_execution_id | flow_action | event_timestamp | lease_acquisition_timestamp] + * (----------------------primary key------------------------) + * We also maintain another table in the database with two constants that allow us to coordinate between participants and + * ensure they are using the same values to base their coordination off of. + * [epsilon | linger] + * `epsilon` - time within we consider to timestamps to be the same, to account for between-host clock drift + * `linger` - minimum time to occur before another host may attempt a lease on a flow event. It should be much greater + * than epsilon and encapsulate executor communication latency including retry attempts + * + * The `event_timestamp` is the time of the flow_action event request. + * ---Event consolidation--- + * Note that for the sake of simplification, we only allow one event associated with a particular flow's flow_action + * (ie: only one LAUNCH for example of flow FOO, but there can be a LAUNCH, KILL, & RESUME for flow FOO at once) during + * the time it takes to execute the flow action. In most cases, the execution time should be so negligible that this + * event consolidation of duplicate flow action requests is not noticed and even during executor downtime this behavior + * is acceptable as the user generally expects a timely execution of the most recent request rather than one execution + * per request. + * + * The `lease_acquisition_timestamp` is the time a host acquired ownership of this flow action, and it is valid for + * `linger` period of time after which it expires and any host can re-attempt ownership. In most cases, the original + * host should actually complete its work while having the lease and then mark the flow action as NULL to indicate no + * further leasing should be done for the event. + */ +@Slf4j +public class MysqlMultiActiveLeaseArbiter implements MultiActiveLeaseArbiter { + /** `j.u.Function` variant for an operation that may @throw IOException or SQLException: preserves method signature checked exceptions */ + @FunctionalInterface + protected interface CheckedFunction { + R apply(T t) throws IOException, SQLException; + } + + protected final DataSource dataSource; + private final String leaseArbiterTableName; + private final String constantsTableName; + private final int epsilon; + private final int linger; + + // TODO: define retention on this table + private static final String CREATE_LEASE_ARBITER_TABLE_STATEMENT = "CREATE TABLE IF NOT EXISTS %S (" + + "flow_group varchar(" + ServiceConfigKeys.MAX_FLOW_GROUP_LENGTH + ") NOT NULL, flow_name varchar(" + + ServiceConfigKeys.MAX_FLOW_GROUP_LENGTH + ") NOT NULL, " + "flow_execution_id varchar(" + + ServiceConfigKeys.MAX_FLOW_EXECUTION_ID_LENGTH + ") NOT NULL, flow_action varchar(100) NOT NULL, " + + "event_timestamp TIMESTAMP, " + + "lease_acquisition_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP," + + "PRIMARY KEY (flow_group,flow_name,flow_execution_id,flow_action))"; + private static final String CREATE_CONSTANTS_TABLE_STATEMENT = "CREATE TABLE IF NOT EXISTS %s " + + "(epsilon INT, linger INT), PRIMARY KEY (epsilon, linger); INSERT INTO %s (epsilon, linger) VALUES (?,?)"; + protected static final String WHERE_CLAUSE_TO_MATCH_KEY = "WHERE flow_group=? AND flow_name=? AND flow_execution_id=?" + + " AND flow_action=?"; + protected static final String WHERE_CLAUSE_TO_MATCH_ROW = WHERE_CLAUSE_TO_MATCH_KEY + + " AND event_timestamp=? AND lease_acquisition_timestamp=?"; + protected static final String SELECT_AFTER_INSERT_STATEMENT = "SELECT ROW_COUNT() AS rows_inserted_count, " + + "lease_acquisition_timestamp, linger FROM %s, %s " + WHERE_CLAUSE_TO_MATCH_KEY; + // Does a cross join between the two tables to have epsilon and linger values available. Returns the following values: + // event_timestamp, lease_acquisition_timestamp, isWithinEpsilon (boolean if event_timestamp in table is within + // epsilon), leaseValidityStatus (1 if lease has not expired, 2 if expired, 3 if column is NULL or no longer leasing) + protected static final String GET_EVENT_INFO_STATEMENT = "SELECT event_timestamp, lease_acquisition_timestamp, " + + "abs(event_timestamp - ?) <= epsilon as isWithinEpsilon, CASE " + + "WHEN CURRENT_TIMESTAMP < (lease_acquisition_timestamp + linger) then 1" + + "WHEN CURRENT_TIMESTAMP >= (lease_acquisition_timestamp + linger) then 2" + + "ELSE 3 END as leaseValidityStatus, linger FROM %s, %s " + WHERE_CLAUSE_TO_MATCH_KEY; + // Insert or update row to acquire lease if values have not changed since the previous read + // Need to define three separate statements to handle cases where row does not exist or has null values to check + protected static final String CONDITIONALLY_ACQUIRE_LEASE_IF_NEW_ROW_STATEMENT = "INSERT INTO %s " + + "(flow_group, flow_name, flow_execution_id, flow_action, event_timestamp) VALUES (?, ?, ?, ?, ?)"; + protected static final String CONDITIONALLY_ACQUIRE_LEASE_IF_FINISHED_LEASING_STATEMENT = "UPDATE %s " + + "SET event_timestamp=?" + WHERE_CLAUSE_TO_MATCH_KEY + + " AND event_timestamp=? AND lease_acquisition_timestamp is NULL"; + protected static final String CONDITIONALLY_ACQUIRE_LEASE_IF_MATCHING_ALL_COLS_STATEMENT = "UPDATE %s " + + "SET event_timestamp=?" + WHERE_CLAUSE_TO_MATCH_ROW + + " AND event_timestamp=? AND lease_acquisition_timestamp=?"; + // Complete lease acquisition if values have not changed since lease was acquired + protected static final String CONDITIONALLY_COMPLETE_LEASE_STATEMENT = "UPDATE %s SET " + + "lease_acquisition_timestamp = NULL " + WHERE_CLAUSE_TO_MATCH_ROW; + + @Inject + public MysqlMultiActiveLeaseArbiter(Config config) throws IOException { + if (config.hasPath(ConfigurationKeys.MYSQL_LEASE_ARBITER_PREFIX)) { + config = config.getConfig(ConfigurationKeys.MYSQL_LEASE_ARBITER_PREFIX).withFallback(config); + } else { + throw new IOException(String.format("Please specify the config for MysqlMultiActiveLeaseArbiter using prefix %s " + + "before all properties", ConfigurationKeys.MYSQL_LEASE_ARBITER_PREFIX)); + } + + this.leaseArbiterTableName = ConfigUtils.getString(config, ConfigurationKeys.SCHEDULER_LEASE_DETERMINATION_STORE_DB_TABLE_KEY, + ConfigurationKeys.DEFAULT_SCHEDULER_LEASE_DETERMINATION_STORE_DB_TABLE); + this.constantsTableName = ConfigUtils.getString(config, ConfigurationKeys.MULTI_ACTIVE_SCHEDULER_CONSTANTS_DB_TABLE_KEY, + ConfigurationKeys.DEFAULT_MULTI_ACTIVE_SCHEDULER_CONSTANTS_DB_TABLE); + this.epsilon = ConfigUtils.getInt(config, ConfigurationKeys.SCHEDULER_EVENT_EPSILON_MILLIS_KEY, + ConfigurationKeys.DEFAULT_SCHEDULER_EVENT_EPSILON_MILLIS); + this.linger = ConfigUtils.getInt(config, ConfigurationKeys.SCHEDULER_EVENT_LINGER_MILLIS_KEY, + ConfigurationKeys.DEFAULT_SCHEDULER_EVENT_LINGER_MILLIS); + this.dataSource = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); + try (Connection connection = dataSource.getConnection(); + PreparedStatement createStatement = connection.prepareStatement(String.format( + CREATE_LEASE_ARBITER_TABLE_STATEMENT, leaseArbiterTableName))) { + createStatement.executeUpdate(); + connection.commit(); + } catch (SQLException e) { + throw new IOException("Table creation failure for " + leaseArbiterTableName, e); + } + withPreparedStatement(String.format(CREATE_CONSTANTS_TABLE_STATEMENT, this.constantsTableName, this.constantsTableName), + createStatement -> { + int i = 0; + createStatement.setInt(++i, epsilon); + createStatement.setInt(++i, linger); + return createStatement.executeUpdate();}, true); + } + + @Override + public LeaseAttemptStatus tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis) + throws IOException { + // Check table for an existing entry for this flow action and event time + ResultSet resultSet = withPreparedStatement( + String.format(GET_EVENT_INFO_STATEMENT, this.leaseArbiterTableName, this.constantsTableName), + getInfoStatement -> { + int i = 0; + getInfoStatement.setTimestamp(++i, new Timestamp(eventTimeMillis)); + getInfoStatement.setString(++i, flowAction.getFlowGroup()); + getInfoStatement.setString(++i, flowAction.getFlowName()); + getInfoStatement.setString(++i, flowAction.getFlowExecutionId()); + getInfoStatement.setString(++i, flowAction.getFlowActionType().toString()); + return getInfoStatement.executeQuery(); + }, true); + + String formattedSelectAfterInsertStatement = + String.format(SELECT_AFTER_INSERT_STATEMENT, this.leaseArbiterTableName, this.constantsTableName); + try { + // CASE 1: If no existing row for this flow action, then go ahead and insert + if (!resultSet.next()) { + String formattedAcquireLeaseNewRowStatement = + String.format(CONDITIONALLY_ACQUIRE_LEASE_IF_NEW_ROW_STATEMENT, this.leaseArbiterTableName); + ResultSet rs = withPreparedStatement( + formattedAcquireLeaseNewRowStatement + "; " + formattedSelectAfterInsertStatement, + insertStatement -> { + completeInsertPreparedStatement(insertStatement, flowAction, eventTimeMillis); + return insertStatement.executeQuery(); + }, true); + return handleResultFromAttemptedLeaseObtainment(rs, flowAction, eventTimeMillis); + } + + // Extract values from result set + Timestamp dbEventTimestamp = resultSet.getTimestamp("event_timestamp"); + Timestamp dbLeaseAcquisitionTimestamp = resultSet.getTimestamp("lease_acquisition_timestamp"); + boolean isWithinEpsilon = resultSet.getBoolean("isWithinEpsilon"); + int leaseValidityStatus = resultSet.getInt("leaseValidityStatus"); + int dbLinger = resultSet.getInt("linger"); + + // CASE 2: If our event timestamp is older than the last event in db, then skip this trigger + if (eventTimeMillis < dbEventTimestamp.getTime()) { + return new NoLongerLeasingStatus(); + } + // Lease is valid + if (leaseValidityStatus == 1) { + // CASE 3: Same event, lease is valid + if (isWithinEpsilon) { + // Utilize db timestamp for reminder + return new LeasedToAnotherStatus(flowAction, dbEventTimestamp.getTime(), + dbLeaseAcquisitionTimestamp.getTime() + dbLinger - System.currentTimeMillis()); + } + // CASE 4: Distinct event, lease is valid + // Utilize db timestamp for wait time, but be reminded of own event timestamp + return new LeasedToAnotherStatus(flowAction, eventTimeMillis, + dbLeaseAcquisitionTimestamp.getTime() + dbLinger - System.currentTimeMillis()); + } + // CASE 5: Lease is out of date (regardless of whether same or distinct event) + else if (leaseValidityStatus == 2) { + if (isWithinEpsilon) { + log.warn("Lease should not be out of date for the same trigger event since epsilon << linger for flowAction" + + " {}, db eventTimestamp {}, db leaseAcquisitionTimestamp {}, linger {}", flowAction, + dbEventTimestamp, dbLeaseAcquisitionTimestamp, dbLinger); + } + // Use our event to acquire lease, check for previous db eventTimestamp and leaseAcquisitionTimestamp + String formattedAcquireLeaseIfMatchingAllStatement = + String.format(CONDITIONALLY_ACQUIRE_LEASE_IF_MATCHING_ALL_COLS_STATEMENT, this.leaseArbiterTableName); + ResultSet rs = withPreparedStatement( + formattedAcquireLeaseIfMatchingAllStatement + "; " + formattedSelectAfterInsertStatement, + updateStatement -> { + completeUpdatePreparedStatement(updateStatement, flowAction, eventTimeMillis, true, + true, dbEventTimestamp, dbLeaseAcquisitionTimestamp); + return updateStatement.executeQuery(); + }, true); + return handleResultFromAttemptedLeaseObtainment(rs, flowAction, eventTimeMillis); + } // No longer leasing this event + // CASE 6: Same event, no longer leasing event in db: terminate + if (isWithinEpsilon) { + return new NoLongerLeasingStatus(); + } + // CASE 7: Distinct event, no longer leasing event in db + // Use our event to acquire lease, check for previous db eventTimestamp and NULL leaseAcquisitionTimestamp + String formattedAcquireLeaseIfFinishedStatement = + String.format(CONDITIONALLY_ACQUIRE_LEASE_IF_FINISHED_LEASING_STATEMENT, this.leaseArbiterTableName); + ResultSet rs = withPreparedStatement( + formattedAcquireLeaseIfFinishedStatement + "; " + formattedSelectAfterInsertStatement, + updateStatement -> { + completeUpdatePreparedStatement(updateStatement, flowAction, eventTimeMillis, true, + false, dbEventTimestamp, null); + return updateStatement.executeQuery(); + }, true); + return handleResultFromAttemptedLeaseObtainment(rs, flowAction, eventTimeMillis); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + /** + * Attempt lease by insert or update following a read based on the condition the state of the table has not changed + * since the read. Parse the result to return the corresponding status based on successful insert/update or not. + * @param resultSet + * @param eventTimeMillis + * @return LeaseAttemptStatus + * @throws SQLException + * @throws IOException + */ + protected LeaseAttemptStatus handleResultFromAttemptedLeaseObtainment(ResultSet resultSet, + DagActionStore.DagAction flowAction, long eventTimeMillis) + throws SQLException, IOException { + if (!resultSet.next()) { + throw new IOException("Expected num rows and lease_acquisition_timestamp returned from query but received nothing"); + } + int numRowsUpdated = resultSet.getInt(1); + long leaseAcquisitionTimeMillis = resultSet.getTimestamp(2).getTime(); + int dbLinger = resultSet.getInt(3); + if (numRowsUpdated == 1) { + return new LeaseObtainedStatus(flowAction, eventTimeMillis, leaseAcquisitionTimeMillis); + } + // Another participant acquired lease in between + return new LeasedToAnotherStatus(flowAction, eventTimeMillis, + leaseAcquisitionTimeMillis + dbLinger - System.currentTimeMillis()); + } + + /** + * Complete the INSERT statement for a new flow action lease where the flow action is not present in the table + * @param statement + * @param flowAction + * @param eventTimeMillis + * @throws SQLException + */ + protected void completeInsertPreparedStatement(PreparedStatement statement, DagActionStore.DagAction flowAction, + long eventTimeMillis) throws SQLException { + int i = 0; + // Values to set in new row + statement.setString(++i, flowAction.getFlowGroup()); + statement.setString(++i, flowAction.getFlowName()); + statement.setString(++i, flowAction.getFlowExecutionId()); + statement.setString(++i, flowAction.getFlowActionType().toString()); + statement.setTimestamp(++i, new Timestamp(eventTimeMillis)); + // Values to check if existing row matches previous read + statement.setString(++i, flowAction.getFlowGroup()); + statement.setString(++i, flowAction.getFlowName()); + statement.setString(++i, flowAction.getFlowExecutionId()); + statement.setString(++i, flowAction.getFlowActionType().toString()); + // Values to select for return + statement.setString(++i, flowAction.getFlowGroup()); + statement.setString(++i, flowAction.getFlowName()); + statement.setString(++i, flowAction.getFlowExecutionId()); + statement.setString(++i, flowAction.getFlowActionType().toString()); + } + + /** + * Complete the UPDATE prepared statements for a flow action that already exists in the table that needs to be + * updated. + * @param statement + * @param flowAction + * @param eventTimeMillis + * @param needEventTimeCheck true if need to compare `originalEventTimestamp` with db event_timestamp + * @param needLeaseAcquisitionTimeCheck true if need to compare `originalLeaseAcquisitionTimestamp` with db one + * @param originalEventTimestamp value to compare to db one, null if not needed + * @param originalLeaseAcquisitionTimestamp value to compare to db one, null if not needed + * @throws SQLException + */ + protected void completeUpdatePreparedStatement(PreparedStatement statement, DagActionStore.DagAction flowAction, + long eventTimeMillis, boolean needEventTimeCheck, boolean needLeaseAcquisitionTimeCheck, + Timestamp originalEventTimestamp, Timestamp originalLeaseAcquisitionTimestamp) throws SQLException { + int i = 0; + // Value to update + statement.setTimestamp(++i, new Timestamp(eventTimeMillis)); + // Values to check if existing row matches previous read + statement.setString(++i, flowAction.getFlowGroup()); + statement.setString(++i, flowAction.getFlowName()); + statement.setString(++i, flowAction.getFlowExecutionId()); + statement.setString(++i, flowAction.getFlowActionType().toString()); + // Values that may be needed depending on the insert statement + if (needEventTimeCheck) { + statement.setTimestamp(++i, originalEventTimestamp); + } + if (needLeaseAcquisitionTimeCheck) { + statement.setTimestamp(++i, originalLeaseAcquisitionTimestamp); + } + // Values to select for return + statement.setString(++i, flowAction.getFlowGroup()); + statement.setString(++i, flowAction.getFlowName()); + statement.setString(++i, flowAction.getFlowExecutionId()); + statement.setString(++i, flowAction.getFlowActionType().toString()); + } + + @Override + public boolean recordLeaseSuccess(LeaseObtainedStatus status) + throws IOException { + DagActionStore.DagAction flowAction = status.getFlowAction(); + String flowGroup = flowAction.getFlowGroup(); + String flowName = flowAction.getFlowName(); + String flowExecutionId = flowAction.getFlowExecutionId(); + DagActionStore.FlowActionType flowActionType = flowAction.getFlowActionType(); + return withPreparedStatement(String.format(CONDITIONALLY_COMPLETE_LEASE_STATEMENT, leaseArbiterTableName), + updateStatement -> { + int i = 0; + updateStatement.setString(++i, flowGroup); + updateStatement.setString(++i, flowName); + updateStatement.setString(++i, flowExecutionId); + updateStatement.setString(++i, flowActionType.toString()); + updateStatement.setTimestamp(++i, new Timestamp(status.getEventTimestamp())); + updateStatement.setTimestamp(++i, new Timestamp(status.getLeaseAcquisitionTimestamp())); + int numRowsUpdated = updateStatement.executeUpdate(); + if (numRowsUpdated == 0) { + log.info("Multi-active lease arbiter lease attempt: [%s, eventTimestamp: %s] - FAILED to complete because " + + "lease expired or event cleaned up before host completed required actions", flowAction, + status.getEventTimestamp()); + return false; + } + if( numRowsUpdated == 1) { + log.info("Multi-active lease arbiter lease attempt: [%s, eventTimestamp: %s] - COMPLETED, no longer leasing" + + " this event after this.", flowAction, status.getEventTimestamp()); + return true; + }; + throw new IOException(String.format("Attempt to complete lease use: [%s, eventTimestamp: %s] - updated more " + + "rows than expected", flowAction, status.getEventTimestamp())); + }, true); + } + + /** Abstracts recurring pattern around resource management and exception re-mapping. */ + protected T withPreparedStatement(String sql, CheckedFunction f, boolean shouldCommit) throws IOException { + try (Connection connection = this.dataSource.getConnection(); + PreparedStatement statement = connection.prepareStatement(sql)) { + T result = f.apply(statement); + if (shouldCommit) { + connection.commit(); + } + return result; + } catch (SQLException e) { + log.warn("Received SQL exception that can result from invalid connection. Checking if validation query is set {} Exception is {}", ((HikariDataSource) this.dataSource).getConnectionTestQuery(), e); + throw new IOException(e); + } + } +} diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStore.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStore.java index d3f4db11bbb..ab5faee8ca0 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStore.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStore.java @@ -43,23 +43,21 @@ public class MysqlDagActionStore implements DagActionStore { public static final String CONFIG_PREFIX = "MysqlDagActionStore"; - private static final long GET_DAG_ACTION_INITIAL_WAIT_AFTER_FAILURE = 1000L; - protected final DataSource dataSource; private final String tableName; - private static final String EXISTS_STATEMENT = "SELECT EXISTS(SELECT * FROM %s WHERE flow_group = ? AND flow_name =? AND flow_execution_id = ?)"; + private static final String EXISTS_STATEMENT = "SELECT EXISTS(SELECT * FROM %s WHERE flow_group = ? AND flow_name =? AND flow_execution_id = ? AND dag_action = ?)"; - protected static final String INSERT_STATEMENT = "INSERT INTO %s (flow_group, flow_name, flow_execution_id, dag_action ) " + protected static final String INSERT_STATEMENT = "INSERT INTO %s (flow_group, flow_name, flow_execution_id, dag_action) " + "VALUES (?, ?, ?, ?)"; - private static final String DELETE_STATEMENT = "DELETE FROM %s WHERE flow_group = ? AND flow_name =? AND flow_execution_id = ?"; - private static final String GET_STATEMENT = "SELECT flow_group, flow_name, flow_execution_id, dag_action FROM %s WHERE flow_group = ? AND flow_name =? AND flow_execution_id = ?"; + private static final String DELETE_STATEMENT = "DELETE FROM %s WHERE flow_group = ? AND flow_name =? AND flow_execution_id = ? AND dag_action = ?"; + private static final String GET_STATEMENT = "SELECT flow_group, flow_name, flow_execution_id, dag_action FROM %s WHERE flow_group = ? AND flow_name =? AND flow_execution_id = ? AND dag_action = ?"; private static final String GET_ALL_STATEMENT = "SELECT flow_group, flow_name, flow_execution_id, dag_action FROM %s"; private static final String CREATE_TABLE_STATEMENT = "CREATE TABLE IF NOT EXISTS %s (" + "flow_group varchar(" + ServiceConfigKeys.MAX_FLOW_GROUP_LENGTH + ") NOT NULL, flow_name varchar(" + ServiceConfigKeys.MAX_FLOW_GROUP_LENGTH + ") NOT NULL, " + "flow_execution_id varchar(" + ServiceConfigKeys.MAX_FLOW_EXECUTION_ID_LENGTH + ") NOT NULL, " + "dag_action varchar(100) NOT NULL, modified_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP on update CURRENT_TIMESTAMP NOT NULL, " - + "PRIMARY KEY (flow_group,flow_name,flow_execution_id))"; + + "PRIMARY KEY (flow_group,flow_name,flow_execution_id, dag_action))"; private final int getDagActionMaxRetries; @@ -86,7 +84,7 @@ public MysqlDagActionStore(Config config) throws IOException { } @Override - public boolean exists(String flowGroup, String flowName, String flowExecutionId) throws IOException, SQLException { + public boolean exists(String flowGroup, String flowName, String flowExecutionId, FlowActionType flowActionType) throws IOException, SQLException { ResultSet rs = null; try (Connection connection = this.dataSource.getConnection(); PreparedStatement existStatement = connection.prepareStatement(String.format(EXISTS_STATEMENT, tableName))) { @@ -94,12 +92,13 @@ public boolean exists(String flowGroup, String flowName, String flowExecutionId) existStatement.setString(++i, flowGroup); existStatement.setString(++i, flowName); existStatement.setString(++i, flowExecutionId); + existStatement.setString(++i, flowActionType.toString()); rs = existStatement.executeQuery(); rs.next(); return rs.getBoolean(1); } catch (SQLException e) { - throw new IOException(String.format("Failure checking existence for table %s of flow with flow group:%s, flow name:%s and flow execution id:%s", - tableName, flowGroup, flowName, flowExecutionId), e); + throw new IOException(String.format("Failure checking existence of DagAction: %s in table %s", + new DagAction(flowGroup, flowName, flowExecutionId, flowActionType), tableName), e); } finally { if (rs != null) { rs.close(); @@ -108,7 +107,7 @@ public boolean exists(String flowGroup, String flowName, String flowExecutionId) } @Override - public void addDagAction(String flowGroup, String flowName, String flowExecutionId, DagActionValue dagActionValue) + public void addDagAction(String flowGroup, String flowName, String flowExecutionId, FlowActionType flowActionType) throws IOException { try (Connection connection = this.dataSource.getConnection(); PreparedStatement insertStatement = connection.prepareStatement(String.format(INSERT_STATEMENT, tableName))) { @@ -116,33 +115,35 @@ public void addDagAction(String flowGroup, String flowName, String flowExecution insertStatement.setString(++i, flowGroup); insertStatement.setString(++i, flowName); insertStatement.setString(++i, flowExecutionId); - insertStatement.setString(++i, dagActionValue.toString()); + insertStatement.setString(++i, flowActionType.toString()); insertStatement.executeUpdate(); connection.commit(); } catch (SQLException e) { - throw new IOException(String.format("Failure to adding action for table %s of flow with flow group:%s, flow name:%s and flow execution id:%s", - tableName, flowGroup, flowName, flowExecutionId), e); + throw new IOException(String.format("Failure adding action for DagAction: %s in table %s", + new DagAction(flowGroup, flowName, flowExecutionId, flowActionType), tableName), e); } } @Override - public boolean deleteDagAction(String flowGroup, String flowName, String flowExecutionId) throws IOException { + public boolean deleteDagAction(DagAction dagAction) throws IOException { try (Connection connection = this.dataSource.getConnection(); PreparedStatement deleteStatement = connection.prepareStatement(String.format(DELETE_STATEMENT, tableName))) { int i = 0; - deleteStatement.setString(++i, flowGroup); - deleteStatement.setString(++i, flowName); - deleteStatement.setString(++i, flowExecutionId); + deleteStatement.setString(++i, dagAction.getFlowGroup()); + deleteStatement.setString(++i, dagAction.getFlowName()); + deleteStatement.setString(++i, dagAction.getFlowExecutionId()); + deleteStatement.setString(++i, dagAction.getFlowActionType().toString()); int result = deleteStatement.executeUpdate(); connection.commit(); return result != 0; } catch (SQLException e) { - throw new IOException(String.format("Failure to delete action for table %s of flow with flow group:%s, flow name:%s and flow execution id:%s", - tableName, flowGroup, flowName, flowExecutionId), e); + throw new IOException(String.format("Failure deleting action for DagAction: %s in table %s", dagAction, + tableName), e); } } - private DagAction getDagActionWithRetry(String flowGroup, String flowName, String flowExecutionId, ExponentialBackoff exponentialBackoff) + // TODO: later change this to getDagActions relating to a particular flow execution if it makes sense + private DagAction getDagActionWithRetry(String flowGroup, String flowName, String flowExecutionId, FlowActionType flowActionType, ExponentialBackoff exponentialBackoff) throws IOException, SQLException { ResultSet rs = null; try (Connection connection = this.dataSource.getConnection(); @@ -151,20 +152,22 @@ private DagAction getDagActionWithRetry(String flowGroup, String flowName, Strin getStatement.setString(++i, flowGroup); getStatement.setString(++i, flowName); getStatement.setString(++i, flowExecutionId); + getStatement.setString(++i, flowActionType.toString()); rs = getStatement.executeQuery(); if (rs.next()) { - return new DagAction(rs.getString(1), rs.getString(2), rs.getString(3), DagActionValue.valueOf(rs.getString(4))); + return new DagAction(rs.getString(1), rs.getString(2), rs.getString(3), FlowActionType.valueOf(rs.getString(4))); } else { if (exponentialBackoff.awaitNextRetryIfAvailable()) { - return getDagActionWithRetry(flowGroup, flowName, flowExecutionId, exponentialBackoff); + return getDagActionWithRetry(flowGroup, flowName, flowExecutionId, flowActionType, exponentialBackoff); } else { - log.warn(String.format("Can not find dag action with flowGroup: %s, flowName: %s, flowExecutionId: %s",flowGroup, flowName, flowExecutionId)); + log.warn(String.format("Can not find dag action: %s with flowGroup: %s, flowName: %s, flowExecutionId: %s", + flowActionType, flowGroup, flowName, flowExecutionId)); return null; } } } catch (SQLException | InterruptedException e) { - throw new IOException(String.format("Failure get dag action from table %s of flow with flow group:%s, flow name:%s and flow execution id:%s", - tableName, flowGroup, flowName, flowExecutionId), e); + throw new IOException(String.format("Failure get %s from table %s", new DagAction(flowGroup, flowName, flowExecutionId, + flowActionType), tableName), e); } finally { if (rs != null) { rs.close(); @@ -173,13 +176,6 @@ private DagAction getDagActionWithRetry(String flowGroup, String flowName, Strin } - @Override - public DagAction getDagAction(String flowGroup, String flowName, String flowExecutionId) - throws IOException, SQLException { - ExponentialBackoff exponentialBackoff = ExponentialBackoff.builder().initialDelay(GET_DAG_ACTION_INITIAL_WAIT_AFTER_FAILURE).maxRetries(this.getDagActionMaxRetries).build(); - return getDagActionWithRetry(flowGroup, flowName, flowExecutionId, exponentialBackoff); - } - @Override public Collection getDagActions() throws IOException { HashSet result = new HashSet<>(); @@ -188,7 +184,7 @@ public Collection getDagActions() throws IOException { ResultSet rs = getAllStatement.executeQuery()) { while (rs.next()) { result.add( - new DagAction(rs.getString(1), rs.getString(2), rs.getString(3), DagActionValue.valueOf(rs.getString(4)))); + new DagAction(rs.getString(1), rs.getString(2), rs.getString(3), FlowActionType.valueOf(rs.getString(4)))); } if (rs != null) { rs.close(); diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/metrics/RuntimeMetrics.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/metrics/RuntimeMetrics.java index 3d9e9b5c55d..dfccb0c0715 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/metrics/RuntimeMetrics.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/metrics/RuntimeMetrics.java @@ -46,6 +46,7 @@ public class RuntimeMetrics { public static final String GOBBLIN_DAG_ACTION_STORE_MONITOR_KILLS_INVOKED = ServiceMetricNames.GOBBLIN_SERVICE_PREFIX + ".dagActionStoreMonitor.kills.invoked"; public static final String GOBBLIN_DAG_ACTION_STORE_MONITOR_MESSAGE_PROCESSED= ServiceMetricNames.GOBBLIN_SERVICE_PREFIX + ".dagActionStoreMonitor.message.processed"; public static final String GOBBLIN_DAG_ACTION_STORE_MONITOR_RESUMES_INVOKED = ServiceMetricNames.GOBBLIN_SERVICE_PREFIX + ".dagActionStoreMonitor.resumes.invoked"; + public static final String GOBBLIN_DAG_ACTION_STORE_MONITOR_FLOWS_LAUNCHED = ServiceMetricNames.GOBBLIN_SERVICE_PREFIX + ".dagActionStoreMonitor.flows.launched"; public static final String GOBBLIN_DAG_ACTION_STORE_MONITOR_UNEXPECTED_ERRORS = ServiceMetricNames.GOBBLIN_SERVICE_PREFIX + ".dagActionStoreMonitor.unexpected.errors"; public static final String GOBBLIN_DAG_ACTION_STORE_PRODUCE_TO_CONSUME_DELAY_MILLIS = ServiceMetricNames.GOBBLIN_SERVICE_PREFIX + ".dagActionStoreMonitor.produce.to.consume.delay"; @@ -72,6 +73,8 @@ public class RuntimeMetrics { public static final String GOBBLIN_JOB_SCHEDULER_TOTAL_GET_SPEC_TIME_NANOS = ServiceMetricNames.GOBBLIN_SERVICE_PREFIX + ".jobScheduler.totalGetSpecTimeNanos"; public static final String GOBBLIN_JOB_SCHEDULER_TOTAL_ADD_SPEC_TIME_NANOS = ServiceMetricNames.GOBBLIN_SERVICE_PREFIX + ".jobScheduler.totalAddSpecTimeNanos"; public static final String GOBBLIN_JOB_SCHEDULER_NUM_JOBS_SCHEDULED_DURING_STARTUP = ServiceMetricNames.GOBBLIN_SERVICE_PREFIX + ".jobScheduler.numJobsScheduledDuringStartup"; + // Metrics Used to Track flowTriggerHandlerProgress + public static final String GOBBLIN_FLOW_TRIGGER_HANDLER_NUM_FLOWS_SUBMITTED = ServiceMetricNames.GOBBLIN_SERVICE_PREFIX + ".flowTriggerHandler.numFlowsSubmitted"; // Metadata keys public static final String TOPIC = "topic"; public static final String GROUP_ID = "groupId"; diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/InjectionNames.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/InjectionNames.java index d0e42f525f4..b9ff94f8a56 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/InjectionNames.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/InjectionNames.java @@ -25,5 +25,8 @@ public final class InjectionNames { public static final String SERVICE_NAME = "serviceName"; public static final String FORCE_LEADER = "forceLeader"; public static final String FLOW_CATALOG_LOCAL_COMMIT = "flowCatalogLocalCommit"; + + // TODO: Rename `warm_standby_enabled` config to `message_forwarding_enabled` since it's a misnomer. public static final String WARM_STANDBY_ENABLED = "statelessRestAPIEnabled"; + public static final String MULTI_ACTIVE_SCHEDULER_ENABLED = "multiActiveSchedulerEnabled"; } diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/JobScheduler.java b/gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/JobScheduler.java index 1ae73e9345b..56b1ac8c045 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/JobScheduler.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/JobScheduler.java @@ -396,7 +396,7 @@ public void scheduleJob(Properties jobProps, JobListener jobListener, Map + long triggerTimestampMillis = trigger.getPreviousFireTime().getTime(); + jobProps.setProperty(ConfigurationKeys.SCHEDULER_EVENT_TO_TRIGGER_TIMESTAMP_MILLIS_KEY, + String.valueOf(triggerTimestampMillis)); try { jobScheduler.runJob(jobProps, jobListener); diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStoreTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStoreTest.java index 0c65f224113..255dd07898f 100644 --- a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStoreTest.java +++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/dag_action_store/MysqlDagActionStoreTest.java @@ -61,40 +61,43 @@ public void setUp() throws Exception { @Test public void testAddAction() throws Exception { - this.mysqlDagActionStore.addDagAction(flowGroup, flowName, flowExecutionId, DagActionStore.DagActionValue.KILL); - //Should not be able to add again when previous one exist + this.mysqlDagActionStore.addDagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.KILL); + //Should not be able to add KILL again when previous one exist Assert.expectThrows(IOException.class, - () -> this.mysqlDagActionStore.addDagAction(flowGroup, flowName, flowExecutionId, DagActionStore.DagActionValue.RESUME)); - //Should be able to add un-exist one - this.mysqlDagActionStore.addDagAction(flowGroup, flowName, flowExecutionId_2, DagActionStore.DagActionValue.RESUME); + () -> this.mysqlDagActionStore.addDagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.KILL)); + //Should be able to add a RESUME action for same execution as well as KILL for another execution of the flow + this.mysqlDagActionStore.addDagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.RESUME); + this.mysqlDagActionStore.addDagAction(flowGroup, flowName, flowExecutionId_2, DagActionStore.FlowActionType.KILL); } @Test(dependsOnMethods = "testAddAction") public void testExists() throws Exception { - Assert.assertTrue(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId)); - Assert.assertTrue(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId_2)); - Assert.assertFalse(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId_3)); + Assert.assertTrue(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.KILL)); + Assert.assertTrue(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.RESUME)); + Assert.assertTrue(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId_2, DagActionStore.FlowActionType.KILL)); + Assert.assertFalse(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId_3, DagActionStore.FlowActionType.RESUME)); + Assert.assertFalse(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId_3, DagActionStore.FlowActionType.KILL)); } @Test(dependsOnMethods = "testExists") - public void testGetAction() throws IOException, SQLException { - Assert.assertEquals(new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, DagActionStore.DagActionValue.KILL), this.mysqlDagActionStore.getDagAction(flowGroup, flowName, flowExecutionId)); - Assert.assertEquals(new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId_2, DagActionStore.DagActionValue.RESUME), this.mysqlDagActionStore.getDagAction(flowGroup, flowName, flowExecutionId_2)); + public void testGetActions() throws IOException { Collection dagActions = this.mysqlDagActionStore.getDagActions(); - Assert.assertEquals(2, dagActions.size()); + Assert.assertEquals(3, dagActions.size()); HashSet set = new HashSet<>(); - set.add(new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, DagActionStore.DagActionValue.KILL)); - set.add(new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId_2, DagActionStore.DagActionValue.RESUME)); + set.add(new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.KILL)); + set.add(new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.RESUME)); + set.add(new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId_2, DagActionStore.FlowActionType.KILL)); Assert.assertEquals(dagActions, set); } - @Test(dependsOnMethods = "testGetAction") + @Test(dependsOnMethods = "testGetActions") public void testDeleteAction() throws IOException, SQLException { - this.mysqlDagActionStore.deleteDagAction(flowGroup, flowName, flowExecutionId); - Assert.assertEquals(this.mysqlDagActionStore.getDagActions().size(), 1); - Assert.assertFalse(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId)); - Assert.assertTrue(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId_2)); - Assert.assertNull( this.mysqlDagActionStore.getDagAction(flowGroup, flowName, flowExecutionId)); + this.mysqlDagActionStore.deleteDagAction( + new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.KILL)); + Assert.assertEquals(this.mysqlDagActionStore.getDagActions().size(), 2); + Assert.assertFalse(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.KILL)); + Assert.assertTrue(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.RESUME)); + Assert.assertTrue(this.mysqlDagActionStore.exists(flowGroup, flowName, flowExecutionId_2, DagActionStore.FlowActionType.KILL)); } } \ No newline at end of file diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceConfiguration.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceConfiguration.java index 0a3640da5bf..03081b3ba2d 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceConfiguration.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceConfiguration.java @@ -43,6 +43,9 @@ public class GobblinServiceConfiguration { @Getter private final boolean isWarmStandbyEnabled; + @Getter + private final boolean isMultiActiveSchedulerEnabled; + @Getter private final boolean isTopologyCatalogEnabled; @@ -107,6 +110,7 @@ public GobblinServiceConfiguration(String serviceName, String serviceId, Config } this.isWarmStandbyEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_WARM_STANDBY_ENABLED_KEY, false); + this.isMultiActiveSchedulerEnabled = ConfigUtils.getBoolean(config, ServiceConfigKeys.GOBBLIN_SERVICE_MULTI_ACTIVE_SCHEDULER_ENABLED_KEY, false); this.isHelixManagerEnabled = config.hasPath(ServiceConfigKeys.ZK_CONNECTION_STRING_KEY); this.isDagManagerEnabled = diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java index 642f78f118e..c0f140a9fe4 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java @@ -20,7 +20,10 @@ import java.util.Objects; import org.apache.gobblin.runtime.api.DagActionStore; +import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter; +import org.apache.gobblin.runtime.api.MysqlMultiActiveLeaseArbiter; import org.apache.gobblin.runtime.dag_action_store.MysqlDagActionStore; +import org.apache.gobblin.service.modules.orchestration.FlowTriggerHandler; import org.apache.gobblin.service.modules.orchestration.UserQuotaManager; import org.apache.gobblin.service.modules.restli.GobblinServiceFlowConfigV2ResourceHandlerWithWarmStandby; import org.apache.gobblin.service.modules.restli.GobblinServiceFlowExecutionResourceHandlerWithWarmStandby; @@ -147,6 +150,9 @@ public void configure(Binder binder) { binder.bindConstant() .annotatedWith(Names.named(InjectionNames.WARM_STANDBY_ENABLED)) .to(serviceConfig.isWarmStandbyEnabled()); + binder.bindConstant() + .annotatedWith(Names.named(InjectionNames.MULTI_ACTIVE_SCHEDULER_ENABLED)) + .to(serviceConfig.isMultiActiveSchedulerEnabled()); OptionalBinder.newOptionalBinder(binder, DagActionStore.class); if (serviceConfig.isWarmStandbyEnabled()) { binder.bind(DagActionStore.class).to(MysqlDagActionStore.class); @@ -159,6 +165,12 @@ public void configure(Binder binder) { binder.bind(FlowExecutionResourceHandler.class).to(GobblinServiceFlowExecutionResourceHandler.class); } + OptionalBinder.newOptionalBinder(binder, MultiActiveLeaseArbiter.class); + OptionalBinder.newOptionalBinder(binder, FlowTriggerHandler.class); + if (serviceConfig.isMultiActiveSchedulerEnabled()) { + binder.bind(MysqlMultiActiveLeaseArbiter.class); + binder.bind(FlowTriggerHandler.class); + } binder.bind(FlowConfigsResource.class); binder.bind(FlowConfigsV2Resource.class); diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java index f47cbde507a..80da8a9e99d 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java @@ -277,8 +277,9 @@ protected void startUp() { * @param dag {@link Dag} to be added * @param persist whether to persist the dag to the {@link DagStateStore} * @param setStatus if true, set all jobs in the dag to pending + * Note this should only be called from the {@link Orchestrator} or {@link org.apache.gobblin.service.monitoring.DagActionStoreChangeMonitor} */ - synchronized void addDag(Dag dag, boolean persist, boolean setStatus) throws IOException { + public synchronized void addDag(Dag dag, boolean persist, boolean setStatus) throws IOException { if (persist) { //Persist the dag this.dagStateStore.writeCheckpoint(dag); @@ -425,7 +426,7 @@ public synchronized void setActive(boolean active) { if (dagActionStore.isPresent()) { Collection dagActions = dagActionStore.get().getDagActions(); for (DagActionStore.DagAction action : dagActions) { - switch (action.getDagActionValue()) { + switch (action.getFlowActionType()) { case KILL: this.handleKillFlowEvent(new KillFlowEvent(action.getFlowGroup(), action.getFlowName(), Long.parseLong(action.getFlowExecutionId()))); break; @@ -433,7 +434,7 @@ public synchronized void setActive(boolean active) { this.handleResumeFlowEvent(new ResumeFlowEvent(action.getFlowGroup(), action.getFlowName(), Long.parseLong(action.getFlowExecutionId()))); break; default: - log.warn("Unsupported dagAction: " + action.getDagActionValue().toString()); + log.warn("Unsupported dagAction: " + action.getFlowActionType().toString()); } } } @@ -578,9 +579,10 @@ public void run() { } } - private void clearUpDagAction(DagId dagId) throws IOException { + private void removeDagActionFromStore(DagId dagId, DagActionStore.FlowActionType flowActionType) throws IOException { if (this.dagActionStore.isPresent()) { - this.dagActionStore.get().deleteDagAction(dagId.flowGroup, dagId.flowName, dagId.flowExecutionId); + this.dagActionStore.get().deleteDagAction( + new DagActionStore.DagAction(dagId.flowGroup, dagId.flowName, dagId.flowExecutionId, flowActionType)); } } @@ -592,13 +594,13 @@ private void beginResumingDag(DagId dagIdToResume) throws IOException { String dagId= dagIdToResume.toString(); if (!this.failedDagIds.contains(dagId)) { log.warn("No dag found with dagId " + dagId + ", so cannot resume flow"); - clearUpDagAction(dagIdToResume); + removeDagActionFromStore(dagIdToResume, DagActionStore.FlowActionType.RESUME); return; } Dag dag = this.failedDagStateStore.getDag(dagId); if (dag == null) { log.error("Dag " + dagId + " was found in memory but not found in failed dag state store"); - clearUpDagAction(dagIdToResume); + removeDagActionFromStore(dagIdToResume, DagActionStore.FlowActionType.RESUME); return; } @@ -649,7 +651,7 @@ private void finishResumingDags() throws IOException { if (dagReady) { this.dagStateStore.writeCheckpoint(dag.getValue()); this.failedDagStateStore.cleanUp(dag.getValue()); - clearUpDagAction(DagManagerUtils.generateDagId(dag.getValue())); + removeDagActionFromStore(DagManagerUtils.generateDagId(dag.getValue()), DagActionStore.FlowActionType.RESUME); this.failedDagIds.remove(dag.getKey()); this.resumingDags.remove(dag.getKey()); initialize(dag.getValue()); @@ -678,7 +680,8 @@ private void cancelDag(DagId dagId) throws ExecutionException, InterruptedExcept } else { log.warn("Did not find Dag with id {}, it might be already cancelled/finished.", dagToCancel); } - clearUpDagAction(dagId); + // Called after a KILL request is received + removeDagActionFromStore(dagId, DagActionStore.FlowActionType.KILL); } private void cancelDagNode(DagNode dagNodeToCancel) throws ExecutionException, InterruptedException { diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java new file mode 100644 index 00000000000..42ab0af96fa --- /dev/null +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.service.modules.orchestration; + +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoUnit; +import java.util.Locale; +import java.util.Properties; +import java.util.Random; + +import org.quartz.JobKey; +import org.quartz.SchedulerException; +import org.quartz.Trigger; + +import com.typesafe.config.Config; + +import javax.inject.Inject; +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.instrumented.Instrumented; +import org.apache.gobblin.metrics.ContextAwareMeter; +import org.apache.gobblin.metrics.MetricContext; +import org.apache.gobblin.runtime.api.DagActionStore; +import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter; +import org.apache.gobblin.runtime.api.MysqlMultiActiveLeaseArbiter; +import org.apache.gobblin.runtime.metrics.RuntimeMetrics; +import org.apache.gobblin.scheduler.JobScheduler; +import org.apache.gobblin.scheduler.SchedulerService; +import org.apache.gobblin.util.ConfigUtils; + + +/** + * Handler used to coordinate multiple hosts with enabled schedulers to respond to flow action events. It uses the + * {@link MysqlMultiActiveLeaseArbiter} to determine a single lease owner at a given time + * for a flow action event. After acquiring the lease, it persists the flow action event to the {@link DagActionStore} + * to be eventually acted upon by the host with the active DagManager. Once it has completed this action, it will mark + * the lease as completed by calling the + * {@link MysqlMultiActiveLeaseArbiter.recordLeaseSuccess()} method. Hosts that do not gain the lease for the event, + * instead schedule a reminder using the {@link SchedulerService} to check back in on the previous lease owner's + * completion status after the lease should expire to ensure the event is handled in failure cases. + */ +@Slf4j +public class FlowTriggerHandler { + private final int schedulerMaxBackoffMillis; + private static Random random = new Random(); + protected MultiActiveLeaseArbiter multiActiveLeaseArbiter; + protected SchedulerService schedulerService; + protected DagActionStore dagActionStore; + private MetricContext metricContext; + private ContextAwareMeter numFlowsSubmitted; + + @Inject + public FlowTriggerHandler(Config config, MultiActiveLeaseArbiter leaseDeterminationStore, + SchedulerService schedulerService, DagActionStore dagActionStore) { + this.schedulerMaxBackoffMillis = ConfigUtils.getInt(config, ConfigurationKeys.SCHEDULER_MAX_BACKOFF_MILLIS_KEY, + ConfigurationKeys.DEFAULT_SCHEDULER_MAX_BACKOFF_MILLIS); + this.multiActiveLeaseArbiter = leaseDeterminationStore; + this.schedulerService = schedulerService; + this.dagActionStore = dagActionStore; + this.metricContext = Instrumented.getMetricContext(new org.apache.gobblin.configuration.State(ConfigUtils.configToProperties(config)), + this.getClass()); + this.numFlowsSubmitted = metricContext.contextAwareMeter(RuntimeMetrics.GOBBLIN_FLOW_TRIGGER_HANDLER_NUM_FLOWS_SUBMITTED); + } + + /** + * This method is used in the multi-active scheduler case for one or more hosts to respond to a flow action event + * by attempting a lease for the flow event and processing the result depending on the status of the attempt. + * @param jobProps + * @param flowAction + * @param eventTimeMillis + * @throws IOException + */ + public void handleTriggerEvent(Properties jobProps, DagActionStore.DagAction flowAction, long eventTimeMillis) + throws IOException { + MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = + multiActiveLeaseArbiter.tryAcquireLease(flowAction, eventTimeMillis); + // TODO: add a log event or metric for each of these cases + if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeaseObtainedStatus) { + MultiActiveLeaseArbiter.LeaseObtainedStatus leaseObtainedStatus = (MultiActiveLeaseArbiter.LeaseObtainedStatus) leaseAttemptStatus; + if (persistFlowAction(leaseObtainedStatus)) { + return; + } + // If persisting the flow action failed, then we set another trigger for this event to occur immediately to + // re-attempt handling the event + scheduleReminderForEvent(jobProps, new MultiActiveLeaseArbiter.LeasedToAnotherStatus(flowAction, + leaseObtainedStatus.getEventTimestamp(), 0L), eventTimeMillis); + return; + } else if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeasedToAnotherStatus) { + scheduleReminderForEvent(jobProps, (MultiActiveLeaseArbiter.LeasedToAnotherStatus) leaseAttemptStatus, + eventTimeMillis); + return; + } else if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.NoLongerLeasingStatus) { + return; + } + throw new RuntimeException(String.format("Received type of leaseAttemptStatus: %s not handled by this method", + leaseAttemptStatus.getClass().getName())); + } + + // Called after obtaining a lease to persist the flow action to {@link DagActionStore} and mark the lease as done + private boolean persistFlowAction(MultiActiveLeaseArbiter.LeaseObtainedStatus leaseStatus) { + try { + DagActionStore.DagAction flowAction = leaseStatus.getFlowAction(); + this.dagActionStore.addDagAction(flowAction.getFlowGroup(), flowAction.getFlowName(), + flowAction.getFlowExecutionId(), flowAction.getFlowActionType()); + // If the flow action has been persisted to the {@link DagActionStore} we can close the lease + this.numFlowsSubmitted.mark(); + return this.multiActiveLeaseArbiter.recordLeaseSuccess(leaseStatus); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * This method is used by {@link FlowTriggerHandler.handleTriggerEvent} to schedule a self-reminder to check on + * the other participant's progress to finish acting on a flow action after the time the lease should expire. + * @param jobProps + * @param status used to extract event to be reminded for and the minimum time after which reminder should occur + * @param originalEventTimeMillis the event timestamp we were originally handling + */ + private void scheduleReminderForEvent(Properties jobProps, MultiActiveLeaseArbiter.LeasedToAnotherStatus status, + long originalEventTimeMillis) { + DagActionStore.DagAction flowAction = status.getFlowAction(); + // Add a small randomization to the minimum reminder wait time to avoid 'thundering herd' issue + String cronExpression = createCronFromDelayPeriod(status.getMinimumLingerDurationMillis() + + random.nextInt(schedulerMaxBackoffMillis)); + jobProps.setProperty(ConfigurationKeys.JOB_SCHEDULE_KEY, cronExpression); + // Ensure we save the event timestamp that we're setting reminder for to have for debugging purposes + // in addition to the event we want to initiate + jobProps.setProperty(ConfigurationKeys.SCHEDULER_EVENT_TO_REVISIT_TIMESTAMP_MILLIS_KEY, + String.valueOf(status.getEventTimeMillis())); + jobProps.setProperty(ConfigurationKeys.SCHEDULER_EVENT_TO_TRIGGER_TIMESTAMP_MILLIS_KEY, + String.valueOf(originalEventTimeMillis)); + JobKey key = new JobKey(flowAction.getFlowName(), flowAction.getFlowGroup()); + // Create a new trigger for the flow in job scheduler that is set to fire at the minimum reminder wait time calculated + Trigger trigger = JobScheduler.createTriggerForJob(key, jobProps); + try { + log.info("Flow Trigger Handler - [%s, eventTimestamp: %s] - attempting to schedule reminder for event %s in %s millis", + flowAction, originalEventTimeMillis, status.getEventTimeMillis(), trigger.getNextFireTime()); + this.schedulerService.getScheduler().scheduleJob(trigger); + } catch (SchedulerException e) { + log.warn("Failed to add job reminder due to SchedulerException for job %s trigger event %s ", key, status.getEventTimeMillis(), e); + } + log.info(String.format("Flow Trigger Handler - [%s, eventTimestamp: %s] - SCHEDULED REMINDER for event %s in %s millis", + flowAction, originalEventTimeMillis, status.getEventTimeMillis(), trigger.getNextFireTime())); + } + + /** + * These methods should only be called from the Orchestrator or JobScheduler classes as it directly adds jobs to the + * Quartz scheduler + * @param delayPeriodMillis + * @return + */ + protected static String createCronFromDelayPeriod(long delayPeriodMillis) { + LocalDateTime now = LocalDateTime.now(ZoneId.of("UTC")); + LocalDateTime timeToScheduleReminder = now.plus(delayPeriodMillis, ChronoUnit.MILLIS); + // TODO: investigate potentially better way of generating cron expression that does not make it US dependent + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("ss mm HH dd MM ? yyyy", Locale.US); + return timeToScheduleReminder.format(formatter); + } +} diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java index c9a21560c38..a0c19678955 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java @@ -56,6 +56,7 @@ import org.apache.gobblin.metrics.Tag; import org.apache.gobblin.metrics.event.EventSubmitter; import org.apache.gobblin.metrics.event.TimingEvent; +import org.apache.gobblin.runtime.api.DagActionStore; import org.apache.gobblin.runtime.api.FlowSpec; import org.apache.gobblin.runtime.api.JobSpec; import org.apache.gobblin.runtime.api.Spec; @@ -103,7 +104,7 @@ public class Orchestrator implements SpecCatalogListener, Instrumentable { private FlowStatusGenerator flowStatusGenerator; private UserQuotaManager quotaManager; - + private Optional flowTriggerHandler; private final ClassAliasResolver aliasResolver; @@ -111,13 +112,13 @@ public class Orchestrator implements SpecCatalogListener, Instrumentable { public Orchestrator(Config config, Optional topologyCatalog, Optional dagManager, Optional log, - FlowStatusGenerator flowStatusGenerator, boolean instrumentationEnabled) { + FlowStatusGenerator flowStatusGenerator, boolean instrumentationEnabled, Optional flowTriggerHandler) { _log = log.isPresent() ? log.get() : LoggerFactory.getLogger(getClass()); this.aliasResolver = new ClassAliasResolver<>(SpecCompiler.class); this.topologyCatalog = topologyCatalog; this.dagManager = dagManager; this.flowStatusGenerator = flowStatusGenerator; - + this.flowTriggerHandler = flowTriggerHandler; try { String specCompilerClassName = ServiceConfigKeys.DEFAULT_GOBBLIN_SERVICE_FLOWCOMPILER_CLASS; if (config.hasPath(ServiceConfigKeys.GOBBLIN_SERVICE_FLOWCOMPILER_CLASS_KEY)) { @@ -160,8 +161,8 @@ public Orchestrator(Config config, Optional topologyCatalog, Op @Inject public Orchestrator(Config config, FlowStatusGenerator flowStatusGenerator, Optional topologyCatalog, - Optional dagManager, Optional log) { - this(config, topologyCatalog, dagManager, log, flowStatusGenerator, true); + Optional dagManager, Optional log, Optional flowTriggerHandler) { + this(config, topologyCatalog, dagManager, log, flowStatusGenerator, true, flowTriggerHandler); } @@ -222,7 +223,7 @@ public void onUpdateSpec(Spec updatedSpec) { } - public void orchestrate(Spec spec) throws Exception { + public void orchestrate(Spec spec, Properties jobProps, long triggerTimestampMillis) throws Exception { // Add below waiting because TopologyCatalog and FlowCatalog service can be launched at the same time this.topologyCatalog.get().getInitComplete().await(); @@ -310,19 +311,28 @@ public void orchestrate(Spec spec) throws Exception { flowCompilationTimer.get().stop(flowMetadata); } - if (this.dagManager.isPresent()) { - try { - //Send the dag to the DagManager. - this.dagManager.get().addDag(jobExecutionPlanDag, true, true); - } catch (Exception ex) { + // If multi-active scheduler is enabled do not pass onto DagManager, otherwise scheduler forwards it directly + if (flowTriggerHandler.isPresent()) { + // If triggerTimestampMillis is 0, then it was not set by the job trigger handler, and we cannot handle this event + if (triggerTimestampMillis == 0L) { + _log.warn("Skipping execution of spec: {} because missing trigger timestamp in job properties", + jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY)); + flowMetadata.put(TimingEvent.METADATA_MESSAGE, "Flow orchestration skipped because no trigger timestamp " + + "associated with flow action."); if (this.eventSubmitter.isPresent()) { - // pronounce failed before stack unwinds, to ensure flow not marooned in `COMPILED` state; (failure likely attributable to DB connection/failover) - String failureMessage = "Failed to add Job Execution Plan due to: " + ex.getMessage(); - flowMetadata.put(TimingEvent.METADATA_MESSAGE, failureMessage); new TimingEvent(this.eventSubmitter.get(), TimingEvent.FlowTimings.FLOW_FAILED).stop(flowMetadata); } - throw ex; + return; } + + String flowExecutionId = flowMetadata.get(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD); + DagActionStore.DagAction flowAction = + new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.LAUNCH); + flowTriggerHandler.get().handleTriggerEvent(jobProps, flowAction, triggerTimestampMillis); + _log.info("Multi-active scheduler finished handling trigger event: [%s, triggerEventTimestamp: %s]", flowAction, + triggerTimestampMillis); + } else if (this.dagManager.isPresent()) { + submitFlowToDagManager((FlowSpec) spec, jobExecutionPlanDag); } else { // Schedule all compiled JobSpecs on their respective Executor for (Dag.DagNode dagNode : jobExecutionPlanDag.getNodes()) { @@ -364,6 +374,28 @@ public void orchestrate(Spec spec) throws Exception { Instrumented.updateTimer(this.flowOrchestrationTimer, System.nanoTime() - startTime, TimeUnit.NANOSECONDS); } + public void submitFlowToDagManager(FlowSpec flowSpec) + throws IOException { + submitFlowToDagManager(flowSpec, specCompiler.compileFlow(flowSpec)); + } + + public void submitFlowToDagManager(FlowSpec flowSpec, Dag jobExecutionPlanDag) + throws IOException { + try { + //Send the dag to the DagManager. + this.dagManager.get().addDag(jobExecutionPlanDag, true, true); + } catch (Exception ex) { + if (this.eventSubmitter.isPresent()) { + // pronounce failed before stack unwinds, to ensure flow not marooned in `COMPILED` state; (failure likely attributable to DB connection/failover) + String failureMessage = "Failed to add Job Execution Plan due to: " + ex.getMessage(); + Map flowMetadata = TimingEventUtils.getFlowMetadata(flowSpec); + flowMetadata.put(TimingEvent.METADATA_MESSAGE, failureMessage); + new TimingEvent(this.eventSubmitter.get(), TimingEvent.FlowTimings.FLOW_FAILED).stop(flowMetadata); + } + throw ex; + } + } + /** * Check if a FlowSpec instance is allowed to run. * diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/TimingEventUtils.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/TimingEventUtils.java index 65b464c888a..99661305fd3 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/TimingEventUtils.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/TimingEventUtils.java @@ -30,8 +30,8 @@ import org.apache.gobblin.util.ConfigUtils; -class TimingEventUtils { - static Map getFlowMetadata(FlowSpec flowSpec) { +public class TimingEventUtils { + public static Map getFlowMetadata(FlowSpec flowSpec) { return getFlowMetadata(flowSpec.getConfig()); } diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/restli/GobblinServiceFlowExecutionResourceHandlerWithWarmStandby.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/restli/GobblinServiceFlowExecutionResourceHandlerWithWarmStandby.java index 3919a3a7d74..0b5d1cdc7e9 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/restli/GobblinServiceFlowExecutionResourceHandlerWithWarmStandby.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/restli/GobblinServiceFlowExecutionResourceHandlerWithWarmStandby.java @@ -30,7 +30,6 @@ import javax.inject.Named; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.runtime.api.DagActionStore; -import org.apache.gobblin.runtime.api.SpecNotFoundException; import org.apache.gobblin.runtime.util.InjectionNames; import org.apache.gobblin.service.FlowExecutionResourceLocalHandler; import org.apache.gobblin.service.modules.core.GobblinServiceManager; @@ -54,34 +53,29 @@ public void resume(ComplexResourceKey quotaManager; + protected final Optional flowTriggerHandler; @Getter protected final Map scheduledFlowSpecs; @Getter @@ -163,7 +165,8 @@ public GobblinServiceJobScheduler(@Named(InjectionNames.SERVICE_NAME) String ser Config config, Optional helixManager, Optional flowCatalog, Optional topologyCatalog, Orchestrator orchestrator, SchedulerService schedulerService, Optional quotaManager, Optional log, - @Named(InjectionNames.WARM_STANDBY_ENABLED) boolean warmStandbyEnabled) throws Exception { + @Named(InjectionNames.WARM_STANDBY_ENABLED) boolean warmStandbyEnabled, + Optional flowTriggerHandler) throws Exception { super(ConfigUtils.configToProperties(config), schedulerService); _log = log.isPresent() ? log.get() : LoggerFactory.getLogger(getClass()); @@ -179,6 +182,7 @@ public GobblinServiceJobScheduler(@Named(InjectionNames.SERVICE_NAME) String ser && config.hasPath(GOBBLIN_SERVICE_SCHEDULER_DR_NOMINATED); this.warmStandbyEnabled = warmStandbyEnabled; this.quotaManager = quotaManager; + this.flowTriggerHandler = flowTriggerHandler; // Check that these metrics do not exist before adding, mainly for testing purpose which creates multiple instances // of the scheduler. If one metric exists, then the others should as well. MetricFilter filter = MetricFilter.contains(RuntimeMetrics.GOBBLIN_JOB_SCHEDULER_GET_SPECS_DURING_STARTUP_PER_SPEC_RATE_NANOS); @@ -198,11 +202,13 @@ public GobblinServiceJobScheduler(@Named(InjectionNames.SERVICE_NAME) String ser } public GobblinServiceJobScheduler(String serviceName, Config config, FlowStatusGenerator flowStatusGenerator, - Optional helixManager, - Optional flowCatalog, Optional topologyCatalog, Optional dagManager, Optional quotaManager, - SchedulerService schedulerService, Optional log, boolean warmStandbyEnabled) throws Exception { + Optional helixManager, Optional flowCatalog, Optional topologyCatalog, + Optional dagManager, Optional quotaManager, SchedulerService schedulerService, + Optional log, boolean warmStandbyEnabled, Optional flowTriggerHandler) + throws Exception { this(serviceName, config, helixManager, flowCatalog, topologyCatalog, - new Orchestrator(config, flowStatusGenerator, topologyCatalog, dagManager, log), schedulerService, quotaManager, log, warmStandbyEnabled); + new Orchestrator(config, flowStatusGenerator, topologyCatalog, dagManager, log, flowTriggerHandler), + schedulerService, quotaManager, log, warmStandbyEnabled, flowTriggerHandler); } public synchronized void setActive(boolean isActive) { @@ -440,7 +446,9 @@ public synchronized void scheduleJob(Properties jobProps, JobListener jobListene public void runJob(Properties jobProps, JobListener jobListener) throws JobException { try { Spec flowSpec = this.scheduledFlowSpecs.get(jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY)); - this.orchestrator.orchestrate(flowSpec); + String triggerTimestampMillis = jobProps.getProperty( + ConfigurationKeys.SCHEDULER_EVENT_TO_TRIGGER_TIMESTAMP_MILLIS_KEY, "0"); + this.orchestrator.orchestrate(flowSpec, jobProps, Long.parseLong(triggerTimestampMillis)); } catch (Exception e) { throw new JobException("Failed to run Spec: " + jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY), e); } diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/DagActionStoreChangeMonitor.java b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/DagActionStoreChangeMonitor.java index 732697038fd..456851612af 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/DagActionStoreChangeMonitor.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/DagActionStoreChangeMonitor.java @@ -18,7 +18,8 @@ package org.apache.gobblin.service.monitoring; import java.io.IOException; -import java.sql.SQLException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.UUID; import java.util.concurrent.TimeUnit; @@ -34,10 +35,14 @@ import org.apache.gobblin.metrics.ContextAwareGauge; import org.apache.gobblin.metrics.ContextAwareMeter; import org.apache.gobblin.runtime.api.DagActionStore; +import org.apache.gobblin.runtime.api.FlowSpec; import org.apache.gobblin.runtime.api.SpecNotFoundException; import org.apache.gobblin.runtime.kafka.HighLevelConsumer; import org.apache.gobblin.runtime.metrics.RuntimeMetrics; +import org.apache.gobblin.runtime.spec_catalog.FlowCatalog; +import org.apache.gobblin.service.FlowId; import org.apache.gobblin.service.modules.orchestration.DagManager; +import org.apache.gobblin.service.modules.orchestration.Orchestrator; /** @@ -52,6 +57,7 @@ public class DagActionStoreChangeMonitor extends HighLevelConsumer { // Metrics private ContextAwareMeter killsInvoked; private ContextAwareMeter resumesInvoked; + private ContextAwareMeter flowsLaunched; private ContextAwareMeter unexpectedErrors; private ContextAwareMeter messageProcessedMeter; private ContextAwareGauge produceToConsumeDelayMillis; // Reports delay from all partitions in one gauge @@ -71,17 +77,23 @@ public String load(String key) throws Exception { protected DagActionStore dagActionStore; protected DagManager dagManager; + protected Orchestrator orchestrator; + protected boolean isMultiActiveSchedulerEnabled; + protected FlowCatalog flowCatalog; // Note that the topic is an empty string (rather than null to avoid NPE) because this monitor relies on the consumer // client itself to determine all Kafka related information dynamically rather than through the config. public DagActionStoreChangeMonitor(String topic, Config config, DagActionStore dagActionStore, DagManager dagManager, - int numThreads) { + int numThreads, FlowCatalog flowCatalog, Orchestrator orchestrator, boolean isMultiActiveSchedulerEnabled) { // Differentiate group id for each host super(topic, config.withValue(GROUP_ID_KEY, ConfigValueFactory.fromAnyRef(DAG_ACTION_CHANGE_MONITOR_PREFIX + UUID.randomUUID().toString())), numThreads); this.dagActionStore = dagActionStore; this.dagManager = dagManager; + this.flowCatalog = flowCatalog; + this.orchestrator = orchestrator; + this.isMultiActiveSchedulerEnabled = isMultiActiveSchedulerEnabled; } @Override @@ -93,7 +105,7 @@ protected void assignTopicPartitions() { @Override /* - This class is multi-threaded and this message will be called by multiple threads, however any given message will be + This class is multithreaded and this method will be called by multiple threads, however any given message will be partitioned and processed by only one thread (and corresponding queue). */ protected void processMessage(DecodeableKafkaRecord message) { @@ -109,6 +121,8 @@ protected void processMessage(DecodeableKafkaRecord message) { String flowName = value.getFlowName(); String flowExecutionId = value.getFlowExecutionId(); + DagActionStore.FlowActionType dagActionType = DagActionStore.FlowActionType.valueOf(value.getDagAction().toString()); + produceToConsumeDelayValue = calcMillisSince(produceTimestamp); log.debug("Processing Dag Action message for flow group: {} name: {} executionId: {} tid: {} operation: {} lag: {}", flowGroup, flowName, flowExecutionId, tid, operation, produceToConsumeDelayValue); @@ -119,47 +133,37 @@ protected void processMessage(DecodeableKafkaRecord message) { return; } - // Retrieve the Dag Action taken from MySQL table unless operation is DELETE - DagActionStore.DagActionValue dagAction = null; - if (!operation.equals("DELETE")) { - try { - dagAction = dagActionStore.getDagAction(flowGroup, flowName, flowExecutionId).getDagActionValue(); - } catch (IOException e) { - log.error("Encountered IOException trying to retrieve dagAction for flow group: {} name: {} executionId: {}. " + "Exception: {}", flowGroup, flowName, flowExecutionId, e); - this.unexpectedErrors.mark(); - return; - } catch (SpecNotFoundException e) { - log.error("DagAction not found for flow group: {} name: {} executionId: {} Exception: {}", flowGroup, flowName, - flowExecutionId, e); - this.unexpectedErrors.mark(); - return; - } catch (SQLException throwables) { - log.error("Encountered SQLException trying to retrieve dagAction for flow group: {} name: {} executionId: {}. " + "Exception: {}", flowGroup, flowName, flowExecutionId, throwables); - return; - } - } - - // We only expert INSERT and DELETE operations done to this table. INSERTs correspond to resume or delete flow - // requests that have to be processed. DELETEs require no action. + // We only expect INSERT and DELETE operations done to this table. INSERTs correspond to any type of + // {@link DagActionStore.FlowActionType} flow requests that have to be processed. DELETEs require no action. try { if (operation.equals("INSERT")) { - if (dagAction.equals(DagActionStore.DagActionValue.RESUME)) { + if (dagActionType.equals(DagActionStore.FlowActionType.RESUME)) { log.info("Received insert dag action and about to send resume flow request"); dagManager.handleResumeFlowRequest(flowGroup, flowName,Long.parseLong(flowExecutionId)); this.resumesInvoked.mark(); - } else if (dagAction.equals(DagActionStore.DagActionValue.KILL)) { + } else if (dagActionType.equals(DagActionStore.FlowActionType.KILL)) { log.info("Received insert dag action and about to send kill flow request"); dagManager.handleKillFlowRequest(flowGroup, flowName, Long.parseLong(flowExecutionId)); this.killsInvoked.mark(); + } else if (dagActionType.equals(DagActionStore.FlowActionType.LAUNCH)) { + // If multi-active scheduler is NOT turned on we should not receive these type of events + if (!this.isMultiActiveSchedulerEnabled) { + this.unexpectedErrors.mark(); + throw new RuntimeException(String.format("Received LAUNCH dagAction while not in multi-active scheduler " + + "mode for flowAction: %s", + new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, dagActionType))); + } + log.info("Received insert dag action and about to forward launch request to DagManager"); + submitFlowToDagManagerHelper(flowGroup, flowName); } else { - log.warn("Received unsupported dagAction {}. Expected to be a KILL or RESUME", dagAction); + log.warn("Received unsupported dagAction {}. Expected to be a KILL, RESUME, or LAUNCH", dagActionType); this.unexpectedErrors.mark(); return; } } else if (operation.equals("UPDATE")) { log.warn("Received an UPDATE action to the DagActionStore when values in this store are never supposed to be " + "updated. Flow group: {} name {} executionId {} were updated to action {}", flowGroup, flowName, - flowExecutionId, dagAction); + flowExecutionId, dagActionType); this.unexpectedErrors.mark(); } else if (operation.equals("DELETE")) { log.debug("Deleted flow group: {} name: {} executionId {} from DagActionStore", flowGroup, flowName, flowExecutionId); @@ -177,15 +181,40 @@ protected void processMessage(DecodeableKafkaRecord message) { dagActionsSeenCache.put(changeIdentifier, changeIdentifier); } + protected void submitFlowToDagManagerHelper(String flowGroup, String flowName) { + // Retrieve job execution plan by recompiling the flow spec to send to the DagManager + FlowId flowId = new FlowId().setFlowGroup(flowGroup).setFlowName(flowName); + FlowSpec spec = null; + try { + URI flowUri = FlowSpec.Utils.createFlowSpecUri(flowId); + spec = (FlowSpec) flowCatalog.getSpecs(flowUri); + this.orchestrator.submitFlowToDagManager(spec); + } catch (URISyntaxException e) { + log.warn("Could not create URI object for flowId {} due to error {}", flowId, e.getMessage()); + this.unexpectedErrors.mark(); + return; + } catch (SpecNotFoundException e) { + log.warn("Spec not found for flow group: {} name: {} Exception: {}", flowGroup, flowName, e); + this.unexpectedErrors.mark(); + return; + } catch (IOException e) { + log.warn("Failed to add Job Execution Plan for flow group: {} name: {} due to error {}", flowGroup, flowName, e); + this.unexpectedErrors.mark(); + return; + } + // Only mark this if the dag was successfully added + this.flowsLaunched.mark(); + } + @Override protected void createMetrics() { super.createMetrics(); this.killsInvoked = this.getMetricContext().contextAwareMeter(RuntimeMetrics.GOBBLIN_DAG_ACTION_STORE_MONITOR_KILLS_INVOKED); this.resumesInvoked = this.getMetricContext().contextAwareMeter(RuntimeMetrics.GOBBLIN_DAG_ACTION_STORE_MONITOR_RESUMES_INVOKED); + this.flowsLaunched = this.getMetricContext().contextAwareMeter(RuntimeMetrics.GOBBLIN_DAG_ACTION_STORE_MONITOR_FLOWS_LAUNCHED); this.unexpectedErrors = this.getMetricContext().contextAwareMeter(RuntimeMetrics.GOBBLIN_DAG_ACTION_STORE_MONITOR_UNEXPECTED_ERRORS); this.messageProcessedMeter = this.getMetricContext().contextAwareMeter(RuntimeMetrics.GOBBLIN_DAG_ACTION_STORE_MONITOR_MESSAGE_PROCESSED); this.produceToConsumeDelayMillis = this.getMetricContext().newContextAwareGauge(RuntimeMetrics.GOBBLIN_DAG_ACTION_STORE_PRODUCE_TO_CONSUME_DELAY_MILLIS, () -> produceToConsumeDelayValue); this.getMetricContext().register(this.produceToConsumeDelayMillis); } - } diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/DagActionStoreChangeMonitorFactory.java b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/DagActionStoreChangeMonitorFactory.java index d4a0656b3f2..5806949a8b1 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/DagActionStoreChangeMonitorFactory.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/monitoring/DagActionStoreChangeMonitorFactory.java @@ -22,11 +22,15 @@ import com.typesafe.config.Config; import javax.inject.Inject; +import javax.inject.Named; import javax.inject.Provider; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.runtime.api.DagActionStore; +import org.apache.gobblin.runtime.spec_catalog.FlowCatalog; +import org.apache.gobblin.runtime.util.InjectionNames; import org.apache.gobblin.service.modules.orchestration.DagManager; +import org.apache.gobblin.service.modules.orchestration.Orchestrator; import org.apache.gobblin.util.ConfigUtils; @@ -40,12 +44,20 @@ public class DagActionStoreChangeMonitorFactory implements Providerabsent(), Optional.of(logger)); + this.mockStatusGenerator, Optional.of(this.topologyCatalog), Optional.absent(), Optional.of(logger), + Optional.of(this._mockFlowTriggerHandler)); this.topologyCatalog.addListener(orchestrator); this.flowCatalog.addListener(orchestrator); // Start application @@ -341,13 +343,13 @@ public void doNotRegisterMetricsAdhocFlows() throws Exception { flowProps.put("gobblin.flow.destinationIdentifier", "destination"); flowProps.put("flow.allowConcurrentExecution", false); FlowSpec adhocSpec = new FlowSpec(URI.create("flow0/group0"), "1", "", ConfigUtils.propertiesToConfig(flowProps) , flowProps, Optional.absent(), Optional.absent()); - this.orchestrator.orchestrate(adhocSpec); + this.orchestrator.orchestrate(adhocSpec, flowProps, 0); String metricName = MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX, "group0", "flow0", ServiceMetricNames.COMPILED); Assert.assertNull(metricContext.getParent().get().getGauges().get(metricName)); flowProps.setProperty("job.schedule", "0/2 * * * * ?"); FlowSpec scheduledSpec = new FlowSpec(URI.create("flow0/group0"), "1", "", ConfigUtils.propertiesToConfig(flowProps) , flowProps, Optional.absent(), Optional.absent()); - this.orchestrator.orchestrate(scheduledSpec); + this.orchestrator.orchestrate(scheduledSpec, flowProps, 0); Assert.assertNotNull(metricContext.getParent().get().getGauges().get(metricName)); } } \ No newline at end of file diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobSchedulerTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobSchedulerTest.java index 22f060bbd79..6db4a83e33f 100644 --- a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobSchedulerTest.java +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobSchedulerTest.java @@ -60,6 +60,7 @@ import org.apache.gobblin.service.modules.orchestration.AbstractUserQuotaManager; import org.apache.gobblin.service.modules.orchestration.InMemoryUserQuotaManager; import org.apache.gobblin.service.modules.orchestration.Orchestrator; +import org.apache.gobblin.service.modules.orchestration.FlowTriggerHandler; import org.apache.gobblin.service.modules.orchestration.UserQuotaManager; import org.apache.gobblin.service.modules.spec.JobExecutionPlan; import org.apache.gobblin.service.modules.spec.JobExecutionPlanDagFactory; @@ -348,7 +349,8 @@ public void testJobSchedulerAddFlowQuotaExceeded() throws Exception { SchedulerService schedulerService = new SchedulerService(new Properties()); // Mock a GaaS scheduler not in warm standby mode GobblinServiceJobScheduler scheduler = new GobblinServiceJobScheduler("testscheduler", - ConfigFactory.empty(), Optional.absent(), Optional.of(flowCatalog), null, mockOrchestrator, schedulerService, Optional.of(new InMemoryUserQuotaManager(quotaConfig)), Optional.absent(), false); + ConfigFactory.empty(), Optional.absent(), Optional.of(flowCatalog), null, mockOrchestrator, schedulerService, Optional.of(new InMemoryUserQuotaManager(quotaConfig)), Optional.absent(), false, Optional.of(Mockito.mock( + FlowTriggerHandler.class))); schedulerService.startAsync().awaitRunning(); scheduler.startUp(); @@ -366,7 +368,8 @@ public void testJobSchedulerAddFlowQuotaExceeded() throws Exception { //Mock a GaaS scheduler in warm standby mode, where we don't check quota GobblinServiceJobScheduler schedulerWithWarmStandbyEnabled = new GobblinServiceJobScheduler("testscheduler", - ConfigFactory.empty(), Optional.absent(), Optional.of(flowCatalog), null, mockOrchestrator, schedulerService, Optional.of(new InMemoryUserQuotaManager(quotaConfig)), Optional.absent(), true); + ConfigFactory.empty(), Optional.absent(), Optional.of(flowCatalog), null, mockOrchestrator, schedulerService, Optional.of(new InMemoryUserQuotaManager(quotaConfig)), Optional.absent(), true, Optional.of(Mockito.mock( + FlowTriggerHandler.class))); schedulerWithWarmStandbyEnabled.startUp(); schedulerWithWarmStandbyEnabled.setActive(true); @@ -388,7 +391,8 @@ class TestGobblinServiceJobScheduler extends GobblinServiceJobScheduler { public TestGobblinServiceJobScheduler(String serviceName, Config config, Optional flowCatalog, Optional topologyCatalog, Orchestrator orchestrator, Optional quotaManager, SchedulerService schedulerService, boolean isWarmStandbyEnabled) throws Exception { - super(serviceName, config, Optional.absent(), flowCatalog, topologyCatalog, orchestrator, schedulerService, quotaManager, Optional.absent(), isWarmStandbyEnabled); + super(serviceName, config, Optional.absent(), flowCatalog, topologyCatalog, orchestrator, schedulerService, quotaManager, Optional.absent(), isWarmStandbyEnabled, Optional.of(Mockito.mock( + FlowTriggerHandler.class))); if (schedulerService != null) { hasScheduler = true; } From ee17b1576759c0669bf7bc46c9faf6868544ceb9 Mon Sep 17 00:00:00 2001 From: Matthew Ho Date: Thu, 15 Jun 2023 11:34:43 -0700 Subject: [PATCH 11/30] [GOBBLIN-1843] Utility for detecting non optional unions should convert dataset urn to hive compatible format (#3705) * [GOBBLIN-1843] Utility for detecting non optional unions should convert dataset urn to hive compatible format * Logs for dataset decorator and hivemetastore utils * Add util method for converting topic to table name --- .../gobblin/compaction/hive/HiveTable.java | 3 +- .../DatasetsFinderFilteringDecorator.java | 5 +++ .../hive/metastore/HiveMetaStoreUtils.java | 16 +++++++-- ...setHiveSchemaContainsNonOptionalUnion.java | 3 +- ...iveSchemaContainsNonOptionalUnionTest.java | 33 +++++++++++-------- 5 files changed, 42 insertions(+), 18 deletions(-) diff --git a/gobblin-compaction/src/main/java/org/apache/gobblin/compaction/hive/HiveTable.java b/gobblin-compaction/src/main/java/org/apache/gobblin/compaction/hive/HiveTable.java index 1ff18265012..7880de2d208 100644 --- a/gobblin-compaction/src/main/java/org/apache/gobblin/compaction/hive/HiveTable.java +++ b/gobblin-compaction/src/main/java/org/apache/gobblin/compaction/hive/HiveTable.java @@ -27,6 +27,7 @@ import com.google.common.base.Splitter; +import org.apache.gobblin.hive.metastore.HiveMetaStoreUtils; import org.apache.gobblin.util.HiveJdbcConnector; @@ -42,7 +43,7 @@ public abstract class HiveTable { protected List attributes; public static class Builder> { - protected String name = UUID.randomUUID().toString().replaceAll("-", "_"); + protected String name = HiveMetaStoreUtils.getHiveTableName(UUID.randomUUID().toString()); protected List primaryKeys = new ArrayList<>(); protected List attributes = new ArrayList<>(); diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/dataset/DatasetsFinderFilteringDecorator.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/dataset/DatasetsFinderFilteringDecorator.java index 7c7f6379905..46dbdc800fe 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/dataset/DatasetsFinderFilteringDecorator.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/dataset/DatasetsFinderFilteringDecorator.java @@ -30,6 +30,8 @@ import com.google.common.annotations.VisibleForTesting; +import lombok.extern.slf4j.Slf4j; + import org.apache.gobblin.dataset.Dataset; import org.apache.gobblin.dataset.DatasetsFinder; import org.apache.gobblin.util.PropertiesUtils; @@ -39,6 +41,7 @@ /** * A decorator for filtering datasets after a {@link DatasetsFinder} finds a {@link List} of {@link Dataset}s */ +@Slf4j public class DatasetsFinderFilteringDecorator implements DatasetsFinder { private static final String PREFIX = "filtering.datasets.finder."; public static final String DATASET_CLASS = PREFIX + "class"; @@ -69,6 +72,7 @@ public DatasetsFinderFilteringDecorator(FileSystem fs, Properties properties) th @Override public List findDatasets() throws IOException { List datasets = datasetFinder.findDatasets(); + log.info("Found {} datasets", datasets.size()); List allowedDatasets = Collections.emptyList(); try { allowedDatasets = datasets.parallelStream() @@ -83,6 +87,7 @@ public List findDatasets() throws IOException { wrappedIOException.rethrowWrapped(); } + log.info("Allowed {}/{} datasets", allowedDatasets.size() ,datasets.size()); return allowedDatasets; } diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java index f7dac0c08b1..7603dfa7eb0 100644 --- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java +++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/metastore/HiveMetaStoreUtils.java @@ -29,8 +29,6 @@ import org.apache.avro.Schema; import org.apache.avro.SchemaParseException; import org.apache.commons.lang.reflect.MethodUtils; -import org.apache.gobblin.hive.avro.HiveAvroSerDeManager; -import org.apache.gobblin.hive.spec.HiveSpec; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; @@ -73,6 +71,8 @@ import org.apache.gobblin.hive.HiveRegistrationUnit.Column; import org.apache.gobblin.hive.HiveTable; import org.apache.gobblin.hive.SharedHiveConfKey; +import org.apache.gobblin.hive.avro.HiveAvroSerDeManager; +import org.apache.gobblin.hive.spec.HiveSpec; /** @@ -151,6 +151,15 @@ public static HiveTable getHiveTable(Table table) { return hiveTable; } + /** + * Hive does not use '-' or '.' in the table name, so they are replaced with '_' + * @param topic + * @return + */ + public static String getHiveTableName(String topic) { + return topic.replaceAll("[-.]", "_"); + } + /** * Convert a {@link HivePartition} into a {@link Partition}. */ @@ -289,7 +298,8 @@ public static boolean containsNonOptionalUnionTypeColumn(HiveTable hiveTable) { .anyMatch(type -> isNonOptionalUnion(type)); } - throw new RuntimeException("Avro based Hive tables without \"" + HiveAvroSerDeManager.SCHEMA_LITERAL +"\" are not supported"); + throw new RuntimeException("Avro based Hive tables without \"" + HiveAvroSerDeManager.SCHEMA_LITERAL +"\" are not supported. " + + "hiveTable=" + hiveTable.getDbName() + "." + hiveTable.getTableName()); } /** diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnion.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnion.java index 655c764c794..c74afaeec2a 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnion.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnion.java @@ -76,7 +76,8 @@ private DbAndTable getDbAndTable(T dataset) { throw new IllegalStateException(String.format("Dataset urn [%s] doesn't follow expected pattern. " + "Expected pattern = %s", dataset.getUrn(), pattern.pattern())); } - return new DbAndTable(m.group(1), m.group(2)); + + return new DbAndTable(m.group(1), HiveMetaStoreUtils.getHiveTableName(m.group(2))); } boolean containsNonOptionalUnion(HiveTable table) { diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnionTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnionTest.java index 33cee4929dc..14041cdecb3 100644 --- a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnionTest.java +++ b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnionTest.java @@ -17,17 +17,10 @@ package org.apache.gobblin.iceberg.predicates; -import com.google.common.io.Files; import java.io.File; import java.util.Collections; -import lombok.extern.slf4j.Slf4j; + import org.apache.commons.io.FileUtils; -import org.apache.gobblin.configuration.State; -import org.apache.gobblin.dataset.Dataset; -import org.apache.gobblin.dataset.test.SimpleDatasetForTesting; -import org.apache.gobblin.hive.HiveTable; -import org.apache.gobblin.hive.metastore.HiveMetaStoreUtils; -import org.apache.gobblin.util.ConfigUtils; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Table; @@ -40,16 +33,30 @@ import org.testng.annotations.BeforeSuite; import org.testng.annotations.Test; +import com.google.common.io.Files; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.configuration.State; +import org.apache.gobblin.dataset.Dataset; +import org.apache.gobblin.dataset.test.SimpleDatasetForTesting; +import org.apache.gobblin.hive.HiveTable; +import org.apache.gobblin.hive.metastore.HiveMetaStoreUtils; +import org.apache.gobblin.util.ConfigUtils; + + @Slf4j +// depends on icebergMetadataWriterTest to avoid concurrency between other HiveMetastoreTest(s) in CI. +// You can uncomment the dependsOnGroups if you want to test this class in isolation @Test(dependsOnGroups = "icebergMetadataWriterTest") public class DatasetHiveSchemaContainsNonOptionalUnionTest extends HiveMetastoreTest { - private static String dbName = "dbname_" + - DatasetHiveSchemaContainsNonOptionalUnionTest.class.getSimpleName().toLowerCase(); + private static String dbName = "dbName"; private static File tmpDir; private static State state; private static String dbUri; - private static String testTable = "test_table"; + private static String testTable = "test_table01"; + private static String datasetUrn = String.format("/data/%s/streaming/test-Table01/hourly/2023/01/01", dbName); @AfterSuite public void clean() throws Exception { @@ -77,14 +84,14 @@ public void setup() throws Exception { metastoreClient.createTable(HiveMetaStoreUtils.getTable(testTable)); state = ConfigUtils.configToState(ConfigUtils.propertiesToConfig(hiveConf.getAllProperties())); - state.setProp(DatasetHiveSchemaContainsNonOptionalUnion.PATTERN, "/data/(\\w+)/(\\w+)"); + state.setProp(DatasetHiveSchemaContainsNonOptionalUnion.PATTERN, "/data/(\\w+)/.*/([\\w\\d_-]+)/hourly.*"); Assert.assertNotNull(metastoreClient.getTable(dbName, DatasetHiveSchemaContainsNonOptionalUnionTest.testTable)); } @Test public void testContainsNonOptionalUnion() throws Exception { DatasetHiveSchemaContainsNonOptionalUnion predicate = new DatasetHiveSchemaContainsNonOptionalUnion(state.getProperties()); - Dataset dataset = new SimpleDatasetForTesting("/data/" + dbName + "/" + testTable); + Dataset dataset = new SimpleDatasetForTesting(datasetUrn); Assert.assertTrue(predicate.test(dataset)); } From dd9f9c454832036a4bbab9c1997f5f9da9cb0891 Mon Sep 17 00:00:00 2001 From: vikram bohra Date: Mon, 19 Jun 2023 18:56:31 -0700 Subject: [PATCH 12/30] [GOBBLIN-1845] Changes parallelstream to stream in DatasetsFinderFilteringDecorator to avoid classloader issues in spark (#3706) I think this is a quick operation so that we don't have much performance concern. Will merge it for now to unblock one urgent fix, we can open another PR if we want to improve the performance. --- .../management/dataset/DatasetsFinderFilteringDecorator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/dataset/DatasetsFinderFilteringDecorator.java b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/dataset/DatasetsFinderFilteringDecorator.java index 46dbdc800fe..ab6743f6b33 100644 --- a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/dataset/DatasetsFinderFilteringDecorator.java +++ b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/dataset/DatasetsFinderFilteringDecorator.java @@ -75,7 +75,7 @@ public List findDatasets() throws IOException { log.info("Found {} datasets", datasets.size()); List allowedDatasets = Collections.emptyList(); try { - allowedDatasets = datasets.parallelStream() + allowedDatasets = datasets.stream() .filter(dataset -> allowDatasetPredicates.stream() .map(CheckedExceptionPredicate::wrapToTunneled) .allMatch(p -> p.test(dataset))) From 5af6bca57df909e44b995e5b2d667c70e0399877 Mon Sep 17 00:00:00 2001 From: umustafi Date: Thu, 22 Jun 2023 14:04:05 -0700 Subject: [PATCH 13/30] [GOBBLIN-1846] Validate Multi-active Scheduler with Logs (#3707) * [GOBBLIN-1846] Validate Multi-active Scheduler with Logs * Adds logging at 3 critical points to ensure no scheduled events are missed - when jobs are first rescheduled (or updated) - each trigger of a new job - when jobs are unscheduled * move flow related logic to GobblinServiceJobScheduler * rename and reuse util * fix typo to pass tests --------- Co-authored-by: Urmi Mustafi --- .../configuration/ConfigurationKeys.java | 4 +- .../gobblin/scheduler/JobScheduler.java | 13 +++---- .../scheduler/GobblinServiceJobScheduler.java | 37 ++++++++++++++++++- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java index b155e8089bb..2d60fd5c83d 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java @@ -98,9 +98,9 @@ public class ConfigurationKeys { // Scheduler lease determination store configuration public static final String MYSQL_LEASE_ARBITER_PREFIX = "MysqlMultiActiveLeaseArbiter"; public static final String MULTI_ACTIVE_SCHEDULER_CONSTANTS_DB_TABLE_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".constantsTable"; - public static final String DEFAULT_MULTI_ACTIVE_SCHEDULER_CONSTANTS_DB_TABLE = MYSQL_LEASE_ARBITER_PREFIX + ".gobblin_multi_active_scheduler_constants_store"; + public static final String DEFAULT_MULTI_ACTIVE_SCHEDULER_CONSTANTS_DB_TABLE = "gobblin_multi_active_scheduler_constants_store"; public static final String SCHEDULER_LEASE_DETERMINATION_STORE_DB_TABLE_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".schedulerLeaseArbiterTable"; - public static final String DEFAULT_SCHEDULER_LEASE_DETERMINATION_STORE_DB_TABLE = MYSQL_LEASE_ARBITER_PREFIX + ".gobblin_scheduler_lease_determination_store"; + public static final String DEFAULT_SCHEDULER_LEASE_DETERMINATION_STORE_DB_TABLE = "gobblin_scheduler_lease_determination_store"; public static final String SCHEDULER_EVENT_TO_REVISIT_TIMESTAMP_MILLIS_KEY = "eventToRevisitTimestampMillis"; public static final String SCHEDULER_EVENT_TO_TRIGGER_TIMESTAMP_MILLIS_KEY = "triggerEventTimestampMillis"; public static final String SCHEDULER_EVENT_EPSILON_MILLIS_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".epsilonMillis"; diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/JobScheduler.java b/gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/JobScheduler.java index 56b1ac8c045..c49de3c6aec 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/JobScheduler.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/scheduler/JobScheduler.java @@ -398,7 +398,7 @@ public void scheduleJob(Properties jobProps, JobListener jobListener, Map - long triggerTimestampMillis = trigger.getPreviousFireTime().getTime(); - jobProps.setProperty(ConfigurationKeys.SCHEDULER_EVENT_TO_TRIGGER_TIMESTAMP_MILLIS_KEY, - String.valueOf(triggerTimestampMillis)); - try { jobScheduler.runJob(jobProps, jobListener); } catch (Throwable t) { diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobScheduler.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobScheduler.java index b62a869ba96..5e0ac398840 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobScheduler.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobScheduler.java @@ -30,14 +30,17 @@ import java.util.TimeZone; import org.apache.commons.lang.StringUtils; +import org.apache.gobblin.runtime.api.SpecNotFoundException; import org.apache.helix.HelixManager; import org.quartz.CronExpression; import org.quartz.DisallowConcurrentExecution; import org.quartz.InterruptableJob; import org.quartz.JobDataMap; +import org.quartz.JobDetail; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.quartz.SchedulerException; +import org.quartz.Trigger; import org.quartz.UnableToInterruptJobException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -96,6 +99,7 @@ */ @Alpha @Singleton +@Slf4j public class GobblinServiceJobScheduler extends JobScheduler implements SpecCatalogListener { // Scheduler related configuration @@ -442,6 +446,19 @@ public synchronized void scheduleJob(Properties jobProps, JobListener jobListene } } + @Override + protected void logNewlyScheduledJob(JobDetail job, Trigger trigger) { + Properties jobProps = (Properties) job.getJobDataMap().get(PROPERTIES_KEY); + log.info(jobSchedulerTracePrefixBuilder(jobProps) + "nextTriggerTime: {} - Job newly scheduled", + trigger.getNextFireTime()); + } + + protected static String jobSchedulerTracePrefixBuilder(Properties jobProps) { + return String.format("Scheduler trigger tracing: [flowName: %s flowGroup: %s] - ", + jobProps.getProperty(ConfigurationKeys.FLOW_NAME_KEY, "<>"), + jobProps.getProperty(ConfigurationKeys.FLOW_GROUP_KEY, "<>")); + } + @Override public void runJob(Properties jobProps, JobListener jobListener) throws JobException { try { @@ -576,6 +593,13 @@ private void unscheduleSpec(URI specURI, String specVersion) throws JobException this.scheduledFlowSpecs.remove(specURI.toString()); this.lastUpdatedTimeForFlowSpec.remove(specURI.toString()); unscheduleJob(specURI.toString()); + try { + FlowSpec spec = (FlowSpec) this.flowCatalog.get().getSpecs(specURI); + Properties properties = spec.getConfigAsProperties(); + _log.info(jobSchedulerTracePrefixBuilder(properties) + "Unscheduled Spec"); + } catch (SpecNotFoundException e) { + _log.warn("Unable to retrieve spec for URI {}", specURI); + } } else { throw new JobException(String.format( "Spec with URI: %s was not found in cache. May be it was cleaned, if not please clean it manually", @@ -666,13 +690,22 @@ public static class GobblinServiceJob extends BaseGobblinJob implements Interrup @Override public void executeImpl(JobExecutionContext context) throws JobExecutionException { - _log.info("Starting FlowSpec " + context.getJobDetail().getKey()); + JobDetail jobDetail = context.getJobDetail(); + _log.info("Starting FlowSpec " + jobDetail.getKey()); - JobDataMap dataMap = context.getJobDetail().getJobDataMap(); + JobDataMap dataMap = jobDetail.getJobDataMap(); JobScheduler jobScheduler = (JobScheduler) dataMap.get(JOB_SCHEDULER_KEY); Properties jobProps = (Properties) dataMap.get(PROPERTIES_KEY); JobListener jobListener = (JobListener) dataMap.get(JOB_LISTENER_KEY); + // Obtain trigger timestamp from trigger to pass to jobProps + Trigger trigger = context.getTrigger(); + // THIS current event has already fired if this method is called, so it now exists in + long triggerTimestampMillis = trigger.getPreviousFireTime().getTime(); + jobProps.setProperty(ConfigurationKeys.SCHEDULER_EVENT_TO_TRIGGER_TIMESTAMP_MILLIS_KEY, + String.valueOf(triggerTimestampMillis)); + _log.info(jobSchedulerTracePrefixBuilder(jobProps) + "triggerTime: {} nextTriggerTime: {} - Job triggered by " + + "scheduler", triggerTimestampMillis, trigger.getNextFireTime().getTime()); try { jobScheduler.runJob(jobProps, jobListener); } catch (Throwable t) { From 702cadf48f910c79b129032aa673f08ce4397c03 Mon Sep 17 00:00:00 2001 From: Matthew Ho Date: Mon, 26 Jun 2023 14:30:55 -0700 Subject: [PATCH 14/30] [GOBBLIN-1844] Ignore workflows marked for deletion when calculating container count (#3709) * [GOBBLIN-1844] Ignore workflows marked for deletion when calculating container count * Add comment --- .../gobblin/yarn/YarnAutoScalingManager.java | 11 +- .../yarn/YarnAutoScalingManagerTest.java | 501 ++++++------------ 2 files changed, 165 insertions(+), 347 deletions(-) diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java index e6683cfd383..a2f4d8372fc 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java @@ -30,7 +30,6 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - import org.apache.commons.compress.utils.Sets; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.helix.HelixDataAccessor; @@ -39,6 +38,7 @@ import org.apache.helix.task.JobConfig; import org.apache.helix.task.JobContext; import org.apache.helix.task.JobDag; +import org.apache.helix.task.TargetState; import org.apache.helix.task.TaskDriver; import org.apache.helix.task.TaskPartitionState; import org.apache.helix.task.TaskState; @@ -220,16 +220,19 @@ void runInternal() { YarnContainerRequestBundle yarnContainerRequestBundle = new YarnContainerRequestBundle(); for (Map.Entry workFlowEntry : taskDriver.getWorkflows().entrySet()) { WorkflowContext workflowContext = taskDriver.getWorkflowContext(workFlowEntry.getKey()); + WorkflowConfig workflowConfig = workFlowEntry.getValue(); - // Only allocate for active workflows - if (workflowContext == null || !workflowContext.getWorkflowState().equals(TaskState.IN_PROGRESS)) { + // Only allocate for active workflows. Those marked for deletion are ignored but the existing containers won't + // be released until maxIdleTimeInMinutesBeforeScalingDown + if (workflowContext == null || + TargetState.DELETE.equals(workflowConfig.getTargetState()) || + !workflowContext.getWorkflowState().equals(TaskState.IN_PROGRESS)) { continue; } log.debug("Workflow name {} config {} context {}", workFlowEntry.getKey(), workFlowEntry.getValue(), workflowContext); - WorkflowConfig workflowConfig = workFlowEntry.getValue(); JobDag jobDag = workflowConfig.getJobDag(); Set jobs = jobDag.getAllNodes(); diff --git a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/YarnAutoScalingManagerTest.java b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/YarnAutoScalingManagerTest.java index 687af96fd83..10563c2bbef 100644 --- a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/YarnAutoScalingManagerTest.java +++ b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/YarnAutoScalingManagerTest.java @@ -17,10 +17,11 @@ package org.apache.gobblin.yarn; -import java.io.IOException; -import java.util.HashMap; +import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.helix.HelixDataAccessor; @@ -29,6 +30,7 @@ import org.apache.helix.task.JobConfig; import org.apache.helix.task.JobContext; import org.apache.helix.task.JobDag; +import org.apache.helix.task.TargetState; import org.apache.helix.task.TaskDriver; import org.apache.helix.task.TaskState; import org.apache.helix.task.WorkflowConfig; @@ -63,34 +65,16 @@ public class YarnAutoScalingManagerTest { * Test for one workflow with one job */ @Test - public void testOneJob() throws IOException { + public void testOneJob() { YarnService mockYarnService = mock(YarnService.class); TaskDriver mockTaskDriver = mock(TaskDriver.class); - WorkflowConfig mockWorkflowConfig = mock(WorkflowConfig.class); - JobDag mockJobDag = mock(JobDag.class); + WorkflowConfig mockWorkflowConfig = + getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job1"), TaskState.IN_PROGRESS, TargetState.START, "workflow1"); + Mockito.when(mockTaskDriver.getWorkflows()).thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig)); - Mockito.when(mockJobDag.getAllNodes()).thenReturn(ImmutableSet.of("job1")); - Mockito.when(mockWorkflowConfig.getJobDag()).thenReturn(mockJobDag); + getJobContext(mockTaskDriver, ImmutableMap.of(2, "GobblinYarnTaskRunner-1"), "job1", ImmutableSet.of(1, 2)); - Mockito.when(mockTaskDriver.getWorkflows()) - .thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig)); - - WorkflowContext mockWorkflowContext = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext.getWorkflowState()).thenReturn(TaskState.IN_PROGRESS); - - Mockito.when(mockTaskDriver.getWorkflowContext("workflow1")).thenReturn(mockWorkflowContext); - - JobContext mockJobContext = mock(JobContext.class); - Mockito.when(mockJobContext.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(1), Integer.valueOf(2))); - Mockito.when(mockJobContext.getAssignedParticipant(2)).thenReturn("GobblinYarnTaskRunner-1"); - - Mockito.when(mockTaskDriver.getJobContext("job1")).thenReturn(mockJobContext); - - HelixDataAccessor helixDataAccessor = mock(HelixDataAccessor.class); - Mockito.when(helixDataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder("cluster")); - Mockito.when(helixDataAccessor.getChildValuesMap(Mockito.any())) - .thenReturn(ImmutableMap.of("GobblinYarnTaskRunner-1", new HelixProperty(""))); + HelixDataAccessor helixDataAccessor = getHelixDataAccessor(Arrays.asList("GobblinClusterManager", "GobblinYarnTaskRunner-1")); YarnAutoScalingManager.YarnAutoScalingRunnable runnable = new YarnAutoScalingManager.YarnAutoScalingRunnable(mockTaskDriver, mockYarnService, 1, @@ -109,40 +93,18 @@ public void testOneJob() throws IOException { * Test for one workflow with two jobs */ @Test - public void testTwoJobs() throws IOException { + public void testTwoJobs() { YarnService mockYarnService = mock(YarnService.class); TaskDriver mockTaskDriver = mock(TaskDriver.class); - WorkflowConfig mockWorkflowConfig = mock(WorkflowConfig.class); - JobDag mockJobDag = mock(JobDag.class); - - Mockito.when(mockJobDag.getAllNodes()).thenReturn(ImmutableSet.of("job1", "job2")); - Mockito.when(mockWorkflowConfig.getJobDag()).thenReturn(mockJobDag); - + WorkflowConfig mockWorkflowConfig = + getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job1", "job2"), TaskState.IN_PROGRESS, TargetState.START, "workflow1"); Mockito.when(mockTaskDriver.getWorkflows()) .thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig)); - WorkflowContext mockWorkflowContext = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext.getWorkflowState()).thenReturn(TaskState.IN_PROGRESS); + getJobContext(mockTaskDriver, ImmutableMap.of(2, "GobblinYarnTaskRunner-1"), "job1", ImmutableSet.of(1, 2)); + getJobContext(mockTaskDriver, ImmutableMap.of(3, "GobblinYarnTaskRunner-2"), "job2"); - Mockito.when(mockTaskDriver.getWorkflowContext("workflow1")).thenReturn(mockWorkflowContext); - - JobContext mockJobContext1 = mock(JobContext.class); - Mockito.when(mockJobContext1.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(1), Integer.valueOf(2))); - Mockito.when(mockJobContext1.getAssignedParticipant(2)).thenReturn("GobblinYarnTaskRunner-1"); - Mockito.when(mockTaskDriver.getJobContext("job1")).thenReturn(mockJobContext1); - - JobContext mockJobContext2 = mock(JobContext.class); - Mockito.when(mockJobContext2.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(3))); - Mockito.when(mockJobContext2.getAssignedParticipant(3)).thenReturn("GobblinYarnTaskRunner-2"); - Mockito.when(mockTaskDriver.getJobContext("job2")).thenReturn(mockJobContext2); - - HelixDataAccessor helixDataAccessor = mock(HelixDataAccessor.class); - Mockito.when(helixDataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder("cluster")); - Mockito.when(helixDataAccessor.getChildValuesMap(Mockito.any())) - .thenReturn(ImmutableMap.of("GobblinYarnTaskRunner-1", new HelixProperty(""), - "GobblinYarnTaskRunner-2", new HelixProperty(""))); + HelixDataAccessor helixDataAccessor = getHelixDataAccessor(Arrays.asList("GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2")); YarnAutoScalingManager.YarnAutoScalingRunnable runnable = new YarnAutoScalingManager.YarnAutoScalingRunnable(mockTaskDriver, mockYarnService, 1, @@ -162,59 +124,24 @@ public void testTwoJobs() throws IOException { * Test for two workflows */ @Test - public void testTwoWorkflows() throws IOException { + public void testTwoWorkflows() { YarnService mockYarnService = mock(YarnService.class); TaskDriver mockTaskDriver = mock(TaskDriver.class); - WorkflowConfig mockWorkflowConfig1 = mock(WorkflowConfig.class); - JobDag mockJobDag1 = mock(JobDag.class); - - Mockito.when(mockJobDag1.getAllNodes()).thenReturn(ImmutableSet.of("job1", "job2")); - Mockito.when(mockWorkflowConfig1.getJobDag()).thenReturn(mockJobDag1); - - WorkflowContext mockWorkflowContext1 = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext1.getWorkflowState()).thenReturn(TaskState.IN_PROGRESS); - - Mockito.when(mockTaskDriver.getWorkflowContext("workflow1")).thenReturn(mockWorkflowContext1); - - JobContext mockJobContext1 = mock(JobContext.class); - Mockito.when(mockJobContext1.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(1), Integer.valueOf(2))); - Mockito.when(mockJobContext1.getAssignedParticipant(2)).thenReturn("GobblinYarnTaskRunner-1"); - Mockito.when(mockTaskDriver.getJobContext("job1")).thenReturn(mockJobContext1); - - JobContext mockJobContext2 = mock(JobContext.class); - Mockito.when(mockJobContext2.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(3))); - Mockito.when(mockJobContext2.getAssignedParticipant(3)).thenReturn("GobblinYarnTaskRunner-2"); - Mockito.when(mockTaskDriver.getJobContext("job2")).thenReturn(mockJobContext2); - - WorkflowConfig mockWorkflowConfig2 = mock(WorkflowConfig.class); - JobDag mockJobDag2 = mock(JobDag.class); - - Mockito.when(mockJobDag2.getAllNodes()).thenReturn(ImmutableSet.of("job3")); - Mockito.when(mockWorkflowConfig2.getJobDag()).thenReturn(mockJobDag2); - - WorkflowContext mockWorkflowContext2 = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext2.getWorkflowState()).thenReturn(TaskState.IN_PROGRESS); - - Mockito.when(mockTaskDriver.getWorkflowContext("workflow2")).thenReturn(mockWorkflowContext2); - - JobContext mockJobContext3 = mock(JobContext.class); - Mockito.when(mockJobContext3.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(4), Integer.valueOf(5))); - Mockito.when(mockJobContext3.getAssignedParticipant(4)).thenReturn("GobblinYarnTaskRunner-3"); - Mockito.when(mockTaskDriver.getJobContext("job3")).thenReturn(mockJobContext3); + WorkflowConfig mockWorkflowConfig1 = + getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job1", "job2"), TaskState.IN_PROGRESS, TargetState.START, "workflow1"); + WorkflowConfig mockWorkflowConfig2 = + getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job3"), TaskState.IN_PROGRESS, TargetState.START, "workflow2"); Mockito.when(mockTaskDriver.getWorkflows()) .thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig1, "workflow2", mockWorkflowConfig2)); - HelixDataAccessor helixDataAccessor = mock(HelixDataAccessor.class); - Mockito.when(helixDataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder("cluster")); - Mockito.when(helixDataAccessor.getChildValuesMap(Mockito.any())) - .thenReturn(ImmutableMap.of("GobblinYarnTaskRunner-1", new HelixProperty(""), - "GobblinYarnTaskRunner-2", new HelixProperty(""), - "GobblinYarnTaskRunner-3", new HelixProperty(""))); + getJobContext(mockTaskDriver, ImmutableMap.of(2, "GobblinYarnTaskRunner-1"), "job1", ImmutableSet.of(1, 2)); + getJobContext(mockTaskDriver, ImmutableMap.of(3, "GobblinYarnTaskRunner-2"), "job2"); + getJobContext(mockTaskDriver, ImmutableMap.of(4, "GobblinYarnTaskRunner-3"), "job3", ImmutableSet.of(4,5)); + + HelixDataAccessor helixDataAccessor = getHelixDataAccessor(Arrays.asList( + "GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2","GobblinYarnTaskRunner-3")); YarnAutoScalingManager.YarnAutoScalingRunnable runnable = new YarnAutoScalingManager.YarnAutoScalingRunnable(mockTaskDriver, mockYarnService, 1, @@ -223,70 +150,37 @@ public void testTwoWorkflows() throws IOException { runnable.run(); // 5 containers requested and 3 workers in use - ArgumentCaptor argument = ArgumentCaptor.forClass(YarnContainerRequestBundle.class); - Mockito.verify(mockYarnService, times(1)). - requestTargetNumberOfContainers(argument.capture(), - eq(ImmutableSet.of("GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2", "GobblinYarnTaskRunner-3"))); - Assert.assertEquals(argument.getValue().getTotalContainers(), 5); + assertContainerRequest(mockYarnService, 5, + ImmutableSet.of("GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2", "GobblinYarnTaskRunner-3")); } /** - * Test for two workflows with one not in progress. - * The partitions for the workflow that is not in progress should not be counted. + * Test for three workflows with one not in progress and one marked for delete. + * The partitions for the workflow that is not in progress or is marked for delete should not be counted. */ @Test - public void testNotInProgress() throws IOException { + public void testNotInProgressOrBeingDeleted() { YarnService mockYarnService = mock(YarnService.class); TaskDriver mockTaskDriver = mock(TaskDriver.class); - WorkflowConfig mockWorkflowConfig1 = mock(WorkflowConfig.class); - JobDag mockJobDag1 = mock(JobDag.class); - - Mockito.when(mockJobDag1.getAllNodes()).thenReturn(ImmutableSet.of("job1", "job2")); - Mockito.when(mockWorkflowConfig1.getJobDag()).thenReturn(mockJobDag1); - - WorkflowContext mockWorkflowContext1 = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext1.getWorkflowState()).thenReturn(TaskState.IN_PROGRESS); - - Mockito.when(mockTaskDriver.getWorkflowContext("workflow1")).thenReturn(mockWorkflowContext1); - - JobContext mockJobContext1 = mock(JobContext.class); - Mockito.when(mockJobContext1.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(1), Integer.valueOf(2))); - Mockito.when(mockJobContext1.getAssignedParticipant(2)).thenReturn("GobblinYarnTaskRunner-1"); - Mockito.when(mockTaskDriver.getJobContext("job1")).thenReturn(mockJobContext1); - - JobContext mockJobContext2 = mock(JobContext.class); - Mockito.when(mockJobContext2.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(3))); - Mockito.when(mockJobContext2.getAssignedParticipant(3)).thenReturn("GobblinYarnTaskRunner-2"); - Mockito.when(mockTaskDriver.getJobContext("job2")).thenReturn(mockJobContext2); - - WorkflowConfig mockWorkflowConfig2 = mock(WorkflowConfig.class); - JobDag mockJobDag2 = mock(JobDag.class); - - Mockito.when(mockJobDag2.getAllNodes()).thenReturn(ImmutableSet.of("job3")); - Mockito.when(mockWorkflowConfig2.getJobDag()).thenReturn(mockJobDag2); - - WorkflowContext mockWorkflowContext2 = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext2.getWorkflowState()).thenReturn(TaskState.COMPLETED); - - Mockito.when(mockTaskDriver.getWorkflowContext("workflow2")).thenReturn(mockWorkflowContext2); - - JobContext mockJobContext3 = mock(JobContext.class); - Mockito.when(mockJobContext3.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(4), Integer.valueOf(5))); - Mockito.when(mockJobContext3.getAssignedParticipant(4)).thenReturn("GobblinYarnTaskRunner-3"); - Mockito.when(mockTaskDriver.getJobContext("job3")).thenReturn(mockJobContext3); - - Mockito.when(mockTaskDriver.getWorkflows()) - .thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig1, "workflow2", mockWorkflowConfig2)); - - HelixDataAccessor helixDataAccessor = mock(HelixDataAccessor.class); - Mockito.when(helixDataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder("cluster")); - Mockito.when(helixDataAccessor.getChildValuesMap(Mockito.any())) - .thenReturn(ImmutableMap.of("GobblinYarnTaskRunner-1", new HelixProperty(""), - "GobblinYarnTaskRunner-2", new HelixProperty(""))); + WorkflowConfig workflowInProgress = getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job-inProgress-1", "job-inProgress-2"), TaskState.IN_PROGRESS, TargetState.START, "workflowInProgress"); + WorkflowConfig workflowCompleted = getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job-complete-1"), TaskState.COMPLETED, TargetState.STOP, "workflowCompleted"); + WorkflowConfig workflowSetToBeDeleted = getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job-setToDelete-1"), TaskState.IN_PROGRESS, TargetState.DELETE, "workflowSetToBeDeleted"); + Mockito.when(mockTaskDriver.getWorkflows()).thenReturn(ImmutableMap.of( + "workflowInProgress", workflowInProgress, + "workflowCompleted", workflowCompleted, + "workflowSetToBeDeleted", workflowSetToBeDeleted)); + + getJobContext(mockTaskDriver, ImmutableMap.of(1, "GobblinYarnTaskRunner-1"), "job-inProgress-1", + ImmutableSet.of(1,2)); + getJobContext(mockTaskDriver, ImmutableMap.of(2, "GobblinYarnTaskRunner-2"), "job-inProgress-2"); + getJobContext(mockTaskDriver, ImmutableMap.of(1, "GobblinYarnTaskRunner-3"), "job-setToDelete-1"); + getJobContext(mockTaskDriver, ImmutableMap.of(1, "GobblinYarnTaskRunner-4"), "job-complete-1", + ImmutableSet.of(1, 5)); + + HelixDataAccessor helixDataAccessor = getHelixDataAccessor( + Arrays.asList("GobblinClusterManager", + "GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2", "GobblinYarnTaskRunner-3", "GobblinYarnTaskRunner-4")); YarnAutoScalingManager.YarnAutoScalingRunnable runnable = new YarnAutoScalingManager.YarnAutoScalingRunnable(mockTaskDriver, mockYarnService, 1, @@ -294,46 +188,26 @@ public void testNotInProgress() throws IOException { runnable.run(); - // 3 containers requested and 2 workers in use - ArgumentCaptor argument = ArgumentCaptor.forClass(YarnContainerRequestBundle.class); - Mockito.verify(mockYarnService, times(1)). - requestTargetNumberOfContainers(argument.capture(), - eq(ImmutableSet.of("GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2"))); - Assert.assertEquals(argument.getValue().getTotalContainers(), 3); + // 3 containers requested and 4 workers in use + assertContainerRequest(mockYarnService, 3, + ImmutableSet.of("GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2", "GobblinYarnTaskRunner-3", "GobblinYarnTaskRunner-4")); } /** * Test multiple partitions to one container */ @Test - public void testMultiplePartitionsPerContainer() throws IOException { + public void testMultiplePartitionsPerContainer() { YarnService mockYarnService = mock(YarnService.class); TaskDriver mockTaskDriver = mock(TaskDriver.class); - WorkflowConfig mockWorkflowConfig = mock(WorkflowConfig.class); - JobDag mockJobDag = mock(JobDag.class); - - Mockito.when(mockJobDag.getAllNodes()).thenReturn(ImmutableSet.of("job1")); - Mockito.when(mockWorkflowConfig.getJobDag()).thenReturn(mockJobDag); + WorkflowConfig mockWorkflowConfig = + getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job1"), TaskState.IN_PROGRESS, TargetState.START, "workflow1"); Mockito.when(mockTaskDriver.getWorkflows()) .thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig)); - WorkflowContext mockWorkflowContext = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext.getWorkflowState()).thenReturn(TaskState.IN_PROGRESS); - - Mockito.when(mockTaskDriver.getWorkflowContext("workflow1")).thenReturn(mockWorkflowContext); - - JobContext mockJobContext = mock(JobContext.class); - Mockito.when(mockJobContext.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(1), Integer.valueOf(2))); - Mockito.when(mockJobContext.getAssignedParticipant(2)).thenReturn("GobblinYarnTaskRunner-1"); - - Mockito.when(mockTaskDriver.getJobContext("job1")).thenReturn(mockJobContext); - - HelixDataAccessor helixDataAccessor = mock(HelixDataAccessor.class); - Mockito.when(helixDataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder("cluster")); - Mockito.when(helixDataAccessor.getChildValuesMap(Mockito.any())) - .thenReturn(ImmutableMap.of("GobblinYarnTaskRunner-1", new HelixProperty(""))); + getJobContext(mockTaskDriver, ImmutableMap.of(2, "GobblinYarnTaskRunner-1"), "job1"); + HelixDataAccessor helixDataAccessor = getHelixDataAccessor(Arrays.asList("GobblinYarnTaskRunner-1")); YarnAutoScalingManager.YarnAutoScalingRunnable runnable = new YarnAutoScalingManager.YarnAutoScalingRunnable(mockTaskDriver, mockYarnService, 2, @@ -342,42 +216,21 @@ public void testMultiplePartitionsPerContainer() throws IOException { runnable.run(); // 1 container requested since 2 partitions and limit is 2 partitions per container. One worker in use. - ArgumentCaptor argument = ArgumentCaptor.forClass(YarnContainerRequestBundle.class); - Mockito.verify(mockYarnService, times(1)). - requestTargetNumberOfContainers(argument.capture(), - eq(ImmutableSet.of("GobblinYarnTaskRunner-1"))); - Assert.assertEquals(argument.getValue().getTotalContainers(), 1); + assertContainerRequest(mockYarnService, 1, ImmutableSet.of("GobblinYarnTaskRunner-1")); } @Test public void testOverprovision() { YarnService mockYarnService = mock(YarnService.class); TaskDriver mockTaskDriver = mock(TaskDriver.class); - WorkflowConfig mockWorkflowConfig = mock(WorkflowConfig.class); - JobDag mockJobDag = mock(JobDag.class); - - Mockito.when(mockJobDag.getAllNodes()).thenReturn(ImmutableSet.of("job1")); - Mockito.when(mockWorkflowConfig.getJobDag()).thenReturn(mockJobDag); - + WorkflowConfig mockWorkflowConfig = + getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job1"), TaskState.IN_PROGRESS, TargetState.START, "workflow1"); Mockito.when(mockTaskDriver.getWorkflows()) .thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig)); - WorkflowContext mockWorkflowContext = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext.getWorkflowState()).thenReturn(TaskState.IN_PROGRESS); - - Mockito.when(mockTaskDriver.getWorkflowContext("workflow1")).thenReturn(mockWorkflowContext); - - JobContext mockJobContext = mock(JobContext.class); - Mockito.when(mockJobContext.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(1), Integer.valueOf(2))); - Mockito.when(mockJobContext.getAssignedParticipant(2)).thenReturn("GobblinYarnTaskRunner-1"); - - Mockito.when(mockTaskDriver.getJobContext("job1")).thenReturn(mockJobContext); + getJobContext(mockTaskDriver, ImmutableMap.of(2, "GobblinYarnTaskRunner-1"), "job1", ImmutableSet.of(1, 2)); - HelixDataAccessor helixDataAccessor = mock(HelixDataAccessor.class); - Mockito.when(helixDataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder("cluster")); - Mockito.when(helixDataAccessor.getChildValuesMap(Mockito.any())) - .thenReturn(ImmutableMap.of("GobblinYarnTaskRunner-1", new HelixProperty(""))); + HelixDataAccessor helixDataAccessor = getHelixDataAccessor(Arrays.asList("GobblinYarnTaskRunner-1")); YarnAutoScalingManager.YarnAutoScalingRunnable runnable1 = new YarnAutoScalingManager.YarnAutoScalingRunnable(mockTaskDriver, mockYarnService, 1, @@ -388,13 +241,9 @@ public void testOverprovision() { // 3 containers requested to max and one worker in use // NumPartitions = 2, Partitions per container = 1 and overprovision = 1.2 // so targetNumContainers = Ceil((2/1) * 1.2)) = 3. - ArgumentCaptor argument = ArgumentCaptor.forClass(YarnContainerRequestBundle.class); - Mockito.verify(mockYarnService, times(1)). - requestTargetNumberOfContainers(argument.capture(), - eq(ImmutableSet.of("GobblinYarnTaskRunner-1"))); - Assert.assertEquals(argument.getValue().getTotalContainers(), 3); - + assertContainerRequest(mockYarnService, 3, ImmutableSet.of("GobblinYarnTaskRunner-1")); Mockito.reset(mockYarnService); + YarnAutoScalingManager.YarnAutoScalingRunnable runnable2 = new YarnAutoScalingManager.YarnAutoScalingRunnable(mockTaskDriver, mockYarnService, 1, 0.1, noopQueue, helixDataAccessor, defaultHelixTag, defaultContainerMemory, defaultContainerCores); @@ -404,10 +253,7 @@ public void testOverprovision() { // 3 containers requested to max and one worker in use // NumPartitions = 2, Partitions per container = 1 and overprovision = 1.2 // so targetNumContainers = Ceil((2/1) * 0.1)) = 1. - Mockito.verify(mockYarnService, times(1)). - requestTargetNumberOfContainers(argument.capture(), - eq(ImmutableSet.of("GobblinYarnTaskRunner-1"))); - Assert.assertEquals(argument.getValue().getTotalContainers(), 1); + assertContainerRequest(mockYarnService, 1, ImmutableSet.of("GobblinYarnTaskRunner-1")); Mockito.reset(mockYarnService); YarnAutoScalingManager.YarnAutoScalingRunnable runnable3 = @@ -419,44 +265,22 @@ public void testOverprovision() { // 3 containers requested to max and one worker in use // NumPartitions = 2, Partitions per container = 1 and overprovision = 6.0, // so targetNumContainers = Ceil((2/1) * 6.0)) = 12. - Mockito.verify(mockYarnService, times(1)). - requestTargetNumberOfContainers(argument.capture(), - eq(ImmutableSet.of("GobblinYarnTaskRunner-1"))); - Assert.assertEquals(argument.getValue().getTotalContainers(), 12); + assertContainerRequest(mockYarnService, 12, ImmutableSet.of("GobblinYarnTaskRunner-1")); } /** * Test suppressed exception */ @Test - public void testSuppressedException() throws IOException { + public void testSuppressedException() { YarnService mockYarnService = mock(YarnService.class); TaskDriver mockTaskDriver = mock(TaskDriver.class); - WorkflowConfig mockWorkflowConfig = mock(WorkflowConfig.class); - JobDag mockJobDag = mock(JobDag.class); + WorkflowConfig mockWorkflowConfig = getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job1"), TaskState.IN_PROGRESS, TargetState.START, "workflow1"); + Mockito.when(mockTaskDriver.getWorkflows()).thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig)); - Mockito.when(mockJobDag.getAllNodes()).thenReturn(ImmutableSet.of("job1")); - Mockito.when(mockWorkflowConfig.getJobDag()).thenReturn(mockJobDag); + getJobContext(mockTaskDriver, ImmutableMap.of(2, "GobblinYarnTaskRunner-1"), "job1", ImmutableSet.of(1, 2)); - Mockito.when(mockTaskDriver.getWorkflows()) - .thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig)); - - WorkflowContext mockWorkflowContext = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext.getWorkflowState()).thenReturn(TaskState.IN_PROGRESS); - - Mockito.when(mockTaskDriver.getWorkflowContext("workflow1")).thenReturn(mockWorkflowContext); - - JobContext mockJobContext = mock(JobContext.class); - Mockito.when(mockJobContext.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(1), Integer.valueOf(2))); - Mockito.when(mockJobContext.getAssignedParticipant(2)).thenReturn("GobblinYarnTaskRunner-1"); - - Mockito.when(mockTaskDriver.getJobContext("job1")).thenReturn(mockJobContext); - - HelixDataAccessor helixDataAccessor = mock(HelixDataAccessor.class); - Mockito.when(helixDataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder("cluster")); - Mockito.when(helixDataAccessor.getChildValuesMap(Mockito.any())) - .thenReturn(ImmutableMap.of("GobblinYarnTaskRunner-1", new HelixProperty(""))); + HelixDataAccessor helixDataAccessor = getHelixDataAccessor(Arrays.asList("GobblinYarnTaskRunner-1")); TestYarnAutoScalingRunnable runnable = new TestYarnAutoScalingRunnable(mockTaskDriver, mockYarnService, 1, helixDataAccessor); @@ -471,14 +295,12 @@ public void testSuppressedException() throws IOException { Mockito.reset(mockYarnService); runnable.setRaiseException(false); runnable.run(); + // 2 container requested - Mockito.verify(mockYarnService, times(1)). - requestTargetNumberOfContainers(argument.capture(), - eq(ImmutableSet.of("GobblinYarnTaskRunner-1"))); - Assert.assertEquals(argument.getValue().getTotalContainers(), 2); + assertContainerRequest(mockYarnService, 2, ImmutableSet.of("GobblinYarnTaskRunner-1")); } - public void testMaxValueEvictingQueue() throws Exception { + public void testMaxValueEvictingQueue() { Resource resource = Resource.newInstance(16, 1); YarnAutoScalingManager.SlidingWindowReservoir window = new YarnAutoScalingManager.SlidingWindowReservoir(3, 10); // Normal insertion with eviction of originally largest value @@ -503,36 +325,16 @@ public void testMaxValueEvictingQueue() throws Exception { * candidate for scaling-down. */ @Test - public void testInstanceIdleBeyondTolerance() throws IOException { + public void testInstanceIdleBeyondTolerance() { YarnService mockYarnService = mock(YarnService.class); TaskDriver mockTaskDriver = mock(TaskDriver.class); - WorkflowConfig mockWorkflowConfig = mock(WorkflowConfig.class); - JobDag mockJobDag = mock(JobDag.class); - Mockito.when(mockJobDag.getAllNodes()).thenReturn(ImmutableSet.of("job1")); - Mockito.when(mockWorkflowConfig.getJobDag()).thenReturn(mockJobDag); - - Mockito.when(mockTaskDriver.getWorkflows()) - .thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig)); - - WorkflowContext mockWorkflowContext = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext.getWorkflowState()).thenReturn(TaskState.IN_PROGRESS); - - Mockito.when(mockTaskDriver.getWorkflowContext("workflow1")).thenReturn(mockWorkflowContext); + WorkflowConfig mockWorkflowConfig = getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job1"), TaskState.IN_PROGRESS, TargetState.START, "workflow1"); + Mockito.when(mockTaskDriver.getWorkflows()).thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig)); // Having both partition assigned to single instance initially, in this case, GobblinYarnTaskRunner-2 - JobContext mockJobContext = mock(JobContext.class); - Mockito.when(mockJobContext.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(1), Integer.valueOf(2))); - Mockito.when(mockJobContext.getAssignedParticipant(1)).thenReturn("GobblinYarnTaskRunner-2"); - Mockito.when(mockJobContext.getAssignedParticipant(2)).thenReturn("GobblinYarnTaskRunner-2"); + getJobContext(mockTaskDriver, ImmutableMap.of(1,"GobblinYarnTaskRunner-2", 2, "GobblinYarnTaskRunner-2"), "job1"); - Mockito.when(mockTaskDriver.getJobContext("job1")).thenReturn(mockJobContext); - - HelixDataAccessor helixDataAccessor = mock(HelixDataAccessor.class); - Mockito.when(helixDataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder("cluster")); - Mockito.when(helixDataAccessor.getChildValuesMap(Mockito.any())) - .thenReturn(ImmutableMap.of("GobblinYarnTaskRunner-1", new HelixProperty(""), - "GobblinYarnTaskRunner-2", new HelixProperty(""))); + HelixDataAccessor helixDataAccessor = getHelixDataAccessor(Arrays.asList("GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2")); TestYarnAutoScalingRunnable runnable = new TestYarnAutoScalingRunnable(mockTaskDriver, mockYarnService, 1, helixDataAccessor); @@ -541,21 +343,14 @@ public void testInstanceIdleBeyondTolerance() throws IOException { // 2 containers requested and one worker in use, while the evaluation will hold for true if not set externally, // still tell YarnService there are two instances being used. - ArgumentCaptor argument = ArgumentCaptor.forClass(YarnContainerRequestBundle.class); - Mockito.verify(mockYarnService, times(1)). - requestTargetNumberOfContainers(argument.capture(), - eq(ImmutableSet.of("GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2"))); - Assert.assertEquals(argument.getValue().getTotalContainers(), 2); + assertContainerRequest(mockYarnService, 2, ImmutableSet.of("GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2")); // Set failEvaluation which simulates the "beyond tolerance" case. Mockito.reset(mockYarnService); runnable.setAlwaysTagUnused(true); runnable.run(); - Mockito.verify(mockYarnService, times(1)). - requestTargetNumberOfContainers(argument.capture(), - eq(ImmutableSet.of("GobblinYarnTaskRunner-2"))); - Assert.assertEquals(argument.getValue().getTotalContainers(), 2); + assertContainerRequest(mockYarnService, 2, ImmutableSet.of("GobblinYarnTaskRunner-2")); } @Test @@ -563,63 +358,29 @@ public void testFlowsWithHelixTags() { YarnService mockYarnService = mock(YarnService.class); TaskDriver mockTaskDriver = mock(TaskDriver.class); - WorkflowConfig mockWorkflowConfig1 = mock(WorkflowConfig.class); - JobDag mockJobDag1 = mock(JobDag.class); - - Mockito.when(mockJobDag1.getAllNodes()).thenReturn(ImmutableSet.of("job1", "job2")); - Mockito.when(mockWorkflowConfig1.getJobDag()).thenReturn(mockJobDag1); - - WorkflowContext mockWorkflowContext1 = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext1.getWorkflowState()).thenReturn(TaskState.IN_PROGRESS); - - Mockito.when(mockTaskDriver.getWorkflowContext("workflow1")).thenReturn(mockWorkflowContext1); - - JobContext mockJobContext1 = mock(JobContext.class); - Mockito.when(mockJobContext1.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(1), Integer.valueOf(2))); - Mockito.when(mockJobContext1.getAssignedParticipant(2)).thenReturn("GobblinYarnTaskRunner-1"); - Mockito.when(mockTaskDriver.getJobContext("job1")).thenReturn(mockJobContext1); - - JobContext mockJobContext2 = mock(JobContext.class); - Mockito.when(mockJobContext2.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(3))); - Mockito.when(mockJobContext2.getAssignedParticipant(3)).thenReturn("GobblinYarnTaskRunner-2"); - Mockito.when(mockTaskDriver.getJobContext("job2")).thenReturn(mockJobContext2); - - WorkflowConfig mockWorkflowConfig2 = mock(WorkflowConfig.class); - JobDag mockJobDag2 = mock(JobDag.class); - - Mockito.when(mockJobDag2.getAllNodes()).thenReturn(ImmutableSet.of("job3")); - Mockito.when(mockWorkflowConfig2.getJobDag()).thenReturn(mockJobDag2); - - WorkflowContext mockWorkflowContext2 = mock(WorkflowContext.class); - Mockito.when(mockWorkflowContext2.getWorkflowState()).thenReturn(TaskState.IN_PROGRESS); + WorkflowConfig mockWorkflowConfig1 = + getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job1", "job2"), TaskState.IN_PROGRESS, TargetState.START, "workflow1"); + WorkflowConfig mockWorkflowConfig2 = + getWorkflowConfig(mockTaskDriver, ImmutableSet.of("job3"), TaskState.IN_PROGRESS, TargetState.START, "workflow2"); + Mockito.when(mockTaskDriver.getWorkflows()) + .thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig1, "workflow2", mockWorkflowConfig2)); - Mockito.when(mockTaskDriver.getWorkflowContext("workflow2")).thenReturn(mockWorkflowContext2); + getJobContext(mockTaskDriver, ImmutableMap.of(2, "GobblinYarnTaskRunner-1"), "job1", ImmutableSet.of(1, 2)); + getJobContext(mockTaskDriver, ImmutableMap.of(3, "GobblinYarnTaskRunner-2"), "job2"); + getJobContext(mockTaskDriver, ImmutableMap.of(4, "GobblinYarnTaskRunner-3"), "job3", ImmutableSet.of(4, 5)); - JobContext mockJobContext3 = mock(JobContext.class); - Mockito.when(mockJobContext3.getPartitionSet()) - .thenReturn(ImmutableSet.of(Integer.valueOf(4), Integer.valueOf(5))); - Mockito.when(mockJobContext3.getAssignedParticipant(4)).thenReturn("GobblinYarnTaskRunner-3"); - Mockito.when(mockTaskDriver.getJobContext("job3")).thenReturn(mockJobContext3); JobConfig mockJobConfig3 = mock(JobConfig.class); + Mockito.when(mockTaskDriver.getJobConfig("job3")).thenReturn(mockJobConfig3); String helixTag = "test-Tag1"; - Map resourceMap = new HashMap<>(); - resourceMap.put(GobblinClusterConfigurationKeys.HELIX_JOB_CONTAINER_MEMORY_MBS, "512"); - resourceMap.put(GobblinClusterConfigurationKeys.HELIX_JOB_CONTAINER_CORES, "8"); + Map resourceMap = ImmutableMap.of( + GobblinClusterConfigurationKeys.HELIX_JOB_CONTAINER_MEMORY_MBS, "512", + GobblinClusterConfigurationKeys.HELIX_JOB_CONTAINER_CORES, "8" + ); Mockito.when(mockJobConfig3.getInstanceGroupTag()).thenReturn(helixTag); Mockito.when(mockJobConfig3.getJobCommandConfigMap()).thenReturn(resourceMap); - Mockito.when(mockTaskDriver.getJobContext("job3")).thenReturn(mockJobContext3); - Mockito.when(mockTaskDriver.getJobConfig("job3")).thenReturn(mockJobConfig3); - Mockito.when(mockTaskDriver.getWorkflows()) - .thenReturn(ImmutableMap.of("workflow1", mockWorkflowConfig1, "workflow2", mockWorkflowConfig2)); - HelixDataAccessor helixDataAccessor = mock(HelixDataAccessor.class); - Mockito.when(helixDataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder("cluster")); - Mockito.when(helixDataAccessor.getChildValuesMap(Mockito.any())) - .thenReturn(ImmutableMap.of("GobblinYarnTaskRunner-1", new HelixProperty(""), - "GobblinYarnTaskRunner-2", new HelixProperty(""), - "GobblinYarnTaskRunner-3", new HelixProperty(""))); + HelixDataAccessor helixDataAccessor = getHelixDataAccessor( + Arrays.asList("GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2", "GobblinYarnTaskRunner-3")); YarnAutoScalingManager.YarnAutoScalingRunnable runnable = new YarnAutoScalingManager.YarnAutoScalingRunnable(mockTaskDriver, mockYarnService, 1, @@ -629,22 +390,76 @@ public void testFlowsWithHelixTags() { // 5 containers requested and 3 workers in use ArgumentCaptor argument = ArgumentCaptor.forClass(YarnContainerRequestBundle.class); - Mockito.verify(mockYarnService, times(1)). - requestTargetNumberOfContainers(argument.capture(), - eq(ImmutableSet.of("GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2", "GobblinYarnTaskRunner-3"))); - Assert.assertEquals(argument.getValue().getTotalContainers(), 5); + assertContainerRequest(argument, mockYarnService, 5, ImmutableSet.of("GobblinYarnTaskRunner-1", "GobblinYarnTaskRunner-2", "GobblinYarnTaskRunner-3")); + + // Verify that 3 containers requested with default tag and resource setting, + // while 2 with specific helix tag and resource requirement Map> resourceHelixTagMap = argument.getValue().getResourceHelixTagMap(); Map helixTagResourceMap = argument.getValue().getHelixTagResourceMap(); Map helixTagContainerCountMap = argument.getValue().getHelixTagContainerCountMap(); - // Verify that 3 containers requested with default tag and resource setting, - // while 2 with specific helix tag and resource requirement Assert.assertEquals(resourceHelixTagMap.size(), 2); Assert.assertEquals(helixTagResourceMap.get(helixTag), Resource.newInstance(512, 8)); Assert.assertEquals(helixTagResourceMap.get(defaultHelixTag), Resource.newInstance(defaultContainerMemory, defaultContainerCores)); Assert.assertEquals((int) helixTagContainerCountMap.get(helixTag), 2); Assert.assertEquals((int) helixTagContainerCountMap.get(defaultHelixTag), 3); + } + + private HelixDataAccessor getHelixDataAccessor(List taskRunners) { + HelixDataAccessor helixDataAccessor = mock(HelixDataAccessor.class); + Mockito.when(helixDataAccessor.keyBuilder()).thenReturn(new PropertyKey.Builder("cluster")); + + Mockito.when(helixDataAccessor.getChildValuesMap(Mockito.any())).thenReturn( + taskRunners.stream().collect(Collectors.toMap((name) -> name, (name) -> new HelixProperty("")))); + return helixDataAccessor; + } + + private WorkflowConfig getWorkflowConfig(TaskDriver mockTaskDriver, ImmutableSet jobNames, + TaskState taskState, TargetState targetState, String workflowName) { + WorkflowConfig mockWorkflowConfig1 = mock(WorkflowConfig.class); + JobDag mockJobDag1 = mock(JobDag.class); + + Mockito.when(mockJobDag1.getAllNodes()).thenReturn(jobNames); + Mockito.when(mockWorkflowConfig1.getJobDag()).thenReturn(mockJobDag1); + Mockito.when(mockWorkflowConfig1.getTargetState()).thenReturn(targetState); + + WorkflowContext mockWorkflowContext1 = mock(WorkflowContext.class); + Mockito.when(mockWorkflowContext1.getWorkflowState()).thenReturn(taskState); + + Mockito.when(mockTaskDriver.getWorkflowContext(workflowName)).thenReturn(mockWorkflowContext1); + return mockWorkflowConfig1; + } + + private JobContext getJobContext(TaskDriver mockTaskDriver, Map assignedParticipantMap, String jobName) { + return getJobContext(mockTaskDriver, assignedParticipantMap, jobName, assignedParticipantMap.keySet()); + } + + private JobContext getJobContext( + TaskDriver mockTaskDriver, + Map assignedParticipantMap, + String jobName, + Set partitionSet) { + JobContext mockJobContext = mock(JobContext.class); + Mockito.when(mockJobContext.getPartitionSet()).thenReturn(ImmutableSet.copyOf(partitionSet)); + for (Map.Entry entry : assignedParticipantMap.entrySet()) { + Mockito.when(mockJobContext.getAssignedParticipant(entry.getKey())).thenReturn(entry.getValue()); + } + Mockito.when(mockTaskDriver.getJobContext(jobName)).thenReturn(mockJobContext); + return mockJobContext; + } + + private void assertContainerRequest(ArgumentCaptor argument, YarnService mockYarnService, int expectedNumberOfContainers, + ImmutableSet expectedInUseInstances) { + ArgumentCaptor.forClass(YarnContainerRequestBundle.class); + Mockito.verify(mockYarnService, times(1)). + requestTargetNumberOfContainers(argument.capture(), + eq(expectedInUseInstances)); + Assert.assertEquals(argument.getValue().getTotalContainers(), expectedNumberOfContainers); + } + private void assertContainerRequest(YarnService mockYarnService, int expectedNumberOfContainers, + ImmutableSet expectedInUseInstances) { + assertContainerRequest(ArgumentCaptor.forClass(YarnContainerRequestBundle.class), mockYarnService, expectedNumberOfContainers, expectedInUseInstances); } private static class TestYarnAutoScalingRunnable extends YarnAutoScalingManager.YarnAutoScalingRunnable { From 48139492b3d27c8514e00c3dd2c954a0cac72cd7 Mon Sep 17 00:00:00 2001 From: Jack Moseley Date: Wed, 28 Jun 2023 10:19:43 -0700 Subject: [PATCH 15/30] [GOBBLIN-1842] Add timers to GobblinMCEWriter (#3703) * Add timers to GobblinMCEWriter * Add dataset level timers and more logs in flush * Fix unit tests --- .../hive/writer/HiveMetadataWriter.java | 10 ++- .../iceberg/writer/GobblinMCEWriter.java | 89 +++++++++++++------ .../iceberg/writer/IcebergMetadataWriter.java | 18 ++-- .../writer/IcebergMetadataWriterTest.java | 4 + 4 files changed, 89 insertions(+), 32 deletions(-) diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/writer/HiveMetadataWriter.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/writer/HiveMetadataWriter.java index 8400dd94b05..02c0c5895c8 100644 --- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/writer/HiveMetadataWriter.java +++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/writer/HiveMetadataWriter.java @@ -17,6 +17,7 @@ package org.apache.gobblin.hive.writer; +import com.codahale.metrics.Timer; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Optional; @@ -160,8 +161,9 @@ public void flush(String dbName, String tableName) throws IOException { HashMap, ListenableFuture> executionMap = this.currentExecutionMap.get(tableKey); //iterator all execution to get the result to make sure they all succeeded for (HashMap.Entry, ListenableFuture> execution : executionMap.entrySet()) { - try { + try (Timer.Context context = new Timer().time()) { execution.getValue().get(timeOutSeconds, TimeUnit.SECONDS); + log.info("Time taken to add partition to table {} is {} ms", tableKey, TimeUnit.NANOSECONDS.toMillis(context.stop())); } catch (TimeoutException e) { // Since TimeoutException should always be a transient issue, throw RuntimeException which will fail/retry container throw new RuntimeException("Timeout waiting for result of registration for table " + tableKey, e); @@ -177,7 +179,11 @@ public void flush(String dbName, String tableName) throws IOException { if (cache != null) { HiveSpec hiveSpec = cache.getIfPresent(execution.getKey()); if (hiveSpec != null) { - eventSubmitter.submit(buildCommitEvent(dbName, tableName, execution.getKey(), hiveSpec, HivePartitionOperation.ADD_OR_MODIFY)); + try (Timer.Context context = new Timer().time()) { + eventSubmitter.submit(buildCommitEvent(dbName, tableName, execution.getKey(), hiveSpec, + HivePartitionOperation.ADD_OR_MODIFY)); + log.info("Time taken to submit event for table {} is {} ms", tableKey, TimeUnit.NANOSECONDS.toMillis(context.stop())); + } } } } diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/GobblinMCEWriter.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/GobblinMCEWriter.java index d2d72a969ea..55987405670 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/GobblinMCEWriter.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/GobblinMCEWriter.java @@ -38,6 +38,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils; import org.apache.commons.collections.CollectionUtils; + +import com.codahale.metrics.Timer; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; @@ -69,6 +71,7 @@ import org.apache.gobblin.metadata.DataFile; import org.apache.gobblin.metadata.GobblinMetadataChangeEvent; import org.apache.gobblin.metadata.OperationType; +import org.apache.gobblin.metrics.ContextAwareTimer; import org.apache.gobblin.metrics.MetricContext; import org.apache.gobblin.metrics.Tag; import org.apache.gobblin.metrics.event.EventSubmitter; @@ -128,9 +131,17 @@ public class GobblinMCEWriter implements DataWriter { private final Set currentErrorDatasets = new HashSet<>(); @Setter private int maxErrorDataset; + @VisibleForTesting + public final MetricContext metricContext; protected EventSubmitter eventSubmitter; private final Set transientExceptionMessages; private final Set nonTransientExceptionMessages; + @VisibleForTesting + public final Map metadataWriterWriteTimers = new HashMap<>(); + @VisibleForTesting + public final Map metadataWriterFlushTimers = new HashMap<>(); + private final ContextAwareTimer hiveSpecComputationTimer; + private final Map datasetTimers = new HashMap<>(); @AllArgsConstructor static class TableStatus { @@ -150,19 +161,22 @@ public GobblinMCEWriter(DataWriterBuilder builder, State acceptedClusters = properties.getPropAsSet(ACCEPTED_CLUSTER_NAMES, ClustersNames.getInstance().getClusterName()); state = properties; maxErrorDataset = state.getPropAsInt(GMCE_METADATA_WRITER_MAX_ERROR_DATASET, DEFUALT_GMCE_METADATA_WRITER_MAX_ERROR_DATASET); + List> tags = Lists.newArrayList(); + String clusterIdentifier = ClustersNames.getInstance().getClusterName(); + tags.add(new Tag<>(MetadataWriterKeys.CLUSTER_IDENTIFIER_KEY_NAME, clusterIdentifier)); + metricContext = Instrumented.getMetricContext(state, this.getClass(), tags); + eventSubmitter = new EventSubmitter.Builder(metricContext, GOBBLIN_MCE_WRITER_METRIC_NAMESPACE).build(); for (String className : state.getPropAsList(GMCE_METADATA_WRITER_CLASSES, IcebergMetadataWriter.class.getName())) { metadataWriters.add(closer.register(GobblinConstructorUtils.invokeConstructor(MetadataWriter.class, className, state))); + metadataWriterWriteTimers.put(className, metricContext.contextAwareTimer(className + ".write", 1, TimeUnit.HOURS)); + metadataWriterFlushTimers.put(className, metricContext.contextAwareTimer(className + ".flush", 1, TimeUnit.HOURS)); } + hiveSpecComputationTimer = metricContext.contextAwareTimer("hiveSpec.computation", 1, TimeUnit.HOURS); tableOperationTypeMap = new HashMap<>(); parallelRunner = closer.register(new ParallelRunner(state.getPropAsInt(METADATA_REGISTRATION_THREADS, 20), FileSystem.get(HadoopUtils.getConfFromState(properties)))); parallelRunnerTimeoutMills = state.getPropAsInt(METADATA_PARALLEL_RUNNER_TIMEOUT_MILLS, DEFAULT_ICEBERG_PARALLEL_TIMEOUT_MILLS); - List> tags = Lists.newArrayList(); - String clusterIdentifier = ClustersNames.getInstance().getClusterName(); - tags.add(new Tag<>(MetadataWriterKeys.CLUSTER_IDENTIFIER_KEY_NAME, clusterIdentifier)); - MetricContext metricContext = Instrumented.getMetricContext(state, this.getClass(), tags); - eventSubmitter = new EventSubmitter.Builder(metricContext, GOBBLIN_MCE_WRITER_METRIC_NAMESPACE).build(); transientExceptionMessages = new HashSet<>(properties.getPropAsList(TRANSIENT_EXCEPTION_MESSAGES_KEY, "")); nonTransientExceptionMessages = new HashSet<>(properties.getPropAsList(NON_TRANSIENT_EXCEPTION_MESSAGES_KEY, "")); } @@ -187,26 +201,28 @@ public void write(GenericRecord record) throws IOException { */ private void computeSpecMap(List files, ConcurrentHashMap> specsMap, Cache> cache, State registerState, boolean isPrefix) throws IOException { - HiveRegistrationPolicy policy = HiveRegistrationPolicyBase.getPolicy(registerState); - for (String file : files) { - parallelRunner.submitCallable(new Callable() { - @Override - public Void call() throws Exception { - try { - Path regPath = isPrefix ? new Path(file) : new Path(file).getParent(); - //Use raw path to comply with HDFS federation setting. - Path rawPath = new Path(regPath.toUri().getRawPath()); - specsMap.put(regPath.toString(), cache.get(regPath.toString(), () -> policy.getHiveSpecs(rawPath))); - } catch (Throwable e) { - //todo: Emit failed GMCE in the future to easily track the error gmce and investigate the reason for that. - log.warn("Cannot get Hive Spec for {} using policy {} due to:", file, policy.toString()); - log.warn(e.getMessage()); + try (Timer.Context context = hiveSpecComputationTimer.time()) { + HiveRegistrationPolicy policy = HiveRegistrationPolicyBase.getPolicy(registerState); + for (String file : files) { + parallelRunner.submitCallable(new Callable() { + @Override + public Void call() throws Exception { + try { + Path regPath = isPrefix ? new Path(file) : new Path(file).getParent(); + //Use raw path to comply with HDFS federation setting. + Path rawPath = new Path(regPath.toUri().getRawPath()); + specsMap.put(regPath.toString(), cache.get(regPath.toString(), () -> policy.getHiveSpecs(rawPath))); + } catch (Throwable e) { + //todo: Emit failed GMCE in the future to easily track the error gmce and investigate the reason for that. + log.warn("Cannot get Hive Spec for {} using policy {} due to:", file, policy.toString()); + log.warn(e.getMessage()); + } + return null; } - return null; - } - }, file); + }, file); + } + parallelRunner.waitForTasks(parallelRunnerTimeoutMills); } - parallelRunner.waitForTasks(parallelRunnerTimeoutMills); } @Override @@ -341,7 +357,12 @@ void writeWithMetadataWriters( writer.reset(dbName, tableName); } else { try { - writer.writeEnvelope(recordEnvelope, newSpecsMap, oldSpecsMap, spec); + Timer writeTimer = metadataWriterWriteTimers.get(writer.getClass().getName()); + Timer datasetTimer = datasetTimers.computeIfAbsent(tableName, k -> metricContext.contextAwareTimer(k, 1, TimeUnit.HOURS)); + try (Timer.Context writeContext = writeTimer.time(); + Timer.Context datasetContext = datasetTimer.time()) { + writer.writeEnvelope(recordEnvelope, newSpecsMap, oldSpecsMap, spec); + } } catch (Exception e) { if (exceptionMatches(e, transientExceptionMessages)) { throw new RuntimeException("Failing container due to transient exception for db: " + dbName + " table: " + tableName, e); @@ -419,7 +440,12 @@ private void flush(String dbName, String tableName) throws IOException { writer.reset(dbName, tableName); } else { try { - writer.flush(dbName, tableName); + Timer flushTimer = metadataWriterFlushTimers.get(writer.getClass().getName()); + Timer datasetTimer = datasetTimers.computeIfAbsent(tableName, k -> metricContext.contextAwareTimer(k, 1, TimeUnit.HOURS)); + try (Timer.Context flushContext = flushTimer.time(); + Timer.Context datasetContext = datasetTimer.time()) { + writer.flush(dbName, tableName); + } } catch (IOException e) { if (exceptionMatches(e, transientExceptionMessages)) { throw new RuntimeException("Failing container due to transient exception for db: " + dbName + " table: " + tableName, e); @@ -480,6 +506,7 @@ public void flush() throws IOException { } entry.getValue().clear(); } + logTimers(); } @Override @@ -565,4 +592,16 @@ private List getFailedWriterList(MetadataWriter failedWriter) { List failedWriters = metadataWriters.subList(metadataWriters.indexOf(failedWriter), metadataWriters.size()); return failedWriters.stream().map(writer -> writer.getClass().getName()).collect(Collectors.toList()); } + + private void logTimers() { + logTimer(hiveSpecComputationTimer); + metadataWriterWriteTimers.values().forEach(this::logTimer); + metadataWriterFlushTimers.values().forEach(this::logTimer); + datasetTimers.values().forEach(this::logTimer); + } + + private void logTimer(ContextAwareTimer timer) { + log.info("Timer {} 1 hour mean duration: {} ms", timer.getName(), TimeUnit.NANOSECONDS.toMillis((long) timer.getSnapshot().getMean())); + log.info("Timer {} 1 hour 99th percentile duration: {} ms", timer.getName(), TimeUnit.NANOSECONDS.toMillis((long) timer.getSnapshot().get99thPercentile())); + } } diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java index ecd528323e2..57a28f77851 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java @@ -827,7 +827,10 @@ public void flush(String dbName, String tableName) throws IOException { String topicName = getTopicName(tid, tableMetadata); if (tableMetadata.appendFiles.isPresent()) { tableMetadata.appendFiles.get().commit(); - sendAuditCounts(topicName, tableMetadata.serializedAuditCountMaps); + try (Timer.Context context = new Timer().time()) { + sendAuditCounts(topicName, tableMetadata.serializedAuditCountMaps); + log.info("Sending audit counts for {} took {} ms", topicName, TimeUnit.NANOSECONDS.toMillis(context.stop())); + } if (tableMetadata.completenessEnabled) { checkAndUpdateCompletenessWatermark(tableMetadata, topicName, tableMetadata.datePartitions, props); } @@ -870,20 +873,25 @@ public void flush(String dbName, String tableName) throws IOException { UpdateProperties updateProperties = transaction.updateProperties(); props.forEach(updateProperties::set); updateProperties.commit(); - try (AutoCloseableHiveLock lock = this.locks.getTableLock(dbName, tableName)) { + try (AutoCloseableHiveLock lock = this.locks.getTableLock(dbName, tableName); + Timer.Context context = new Timer().time()) { transaction.commitTransaction(); + log.info("Committing transaction for table {} took {} ms", tid, TimeUnit.NANOSECONDS.toMillis(context.stop())); } // Emit GTE for snapshot commits Snapshot snapshot = tableMetadata.table.get().currentSnapshot(); Map currentProps = tableMetadata.table.get().properties(); - submitSnapshotCommitEvent(snapshot, tableMetadata, dbName, tableName, currentProps, highWatermark); + try (Timer.Context context = new Timer().time()) { + submitSnapshotCommitEvent(snapshot, tableMetadata, dbName, tableName, currentProps, highWatermark); + log.info("Sending snapshot commit event for {} took {} ms", topicName, TimeUnit.NANOSECONDS.toMillis(context.stop())); + } //Reset the table metadata for next accumulation period tableMetadata.reset(currentProps, highWatermark); - log.info(String.format("Finish commit of new snapshot %s for table %s", snapshot.snapshotId(), tid.toString())); + log.info(String.format("Finish commit of new snapshot %s for table %s", snapshot.snapshotId(), tid)); } else { - log.info("There's no transaction initiated for the table {}", tid.toString()); + log.info("There's no transaction initiated for the table {}", tid); } } catch (RuntimeException e) { throw new IOException(String.format("Fail to flush table %s %s", dbName, tableName), e); diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java index bf7dcb3833f..e5819d6baab 100644 --- a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java +++ b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java @@ -348,6 +348,10 @@ public void testFaultTolerant() throws Exception { MetadataWriter mockWriter = Mockito.mock(MetadataWriter.class); Mockito.doThrow(new IOException("Test failure")).when(mockWriter).writeEnvelope(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); gobblinMCEWriter.metadataWriters.add(0, mockWriter); + gobblinMCEWriter.metadataWriterWriteTimers.put(mockWriter.getClass().getName(), gobblinMCEWriter.metricContext + .contextAwareTimer(mockWriter.getClass().getName() + ".write", 1, TimeUnit.HOURS)); + gobblinMCEWriter.metadataWriterFlushTimers.put(mockWriter.getClass().getName(), gobblinMCEWriter.metricContext + .contextAwareTimer(mockWriter.getClass().getName() + ".flush", 1, TimeUnit.HOURS)); GobblinMetadataChangeEvent gmceWithMockWriter = SpecificData.get().deepCopy(gmce.getSchema(), gmce); gmceWithMockWriter.setAllowedMetadataWriters(Arrays.asList(IcebergMetadataWriter.class.getName(), mockWriter.getClass().getName())); From b6b49c7144a0f7663ae4e80b336bf44f2de07253 Mon Sep 17 00:00:00 2001 From: Matthew Ho Date: Wed, 28 Jun 2023 10:51:14 -0700 Subject: [PATCH 16/30] [GOBBLIN-1847] Exceptions in the JobLauncher should try to delete the existing workflow if it is launched (#3711) * [GOBBLIN-1847] Exceptions in the JobLauncher should try to delete the existing workflow if it is launched * Only cancel helix workflow if failure occurs during startup --- .../cluster/GobblinHelixJobLauncher.java | 6 +++- .../cluster/GobblinHelixJobLauncherTest.java | 31 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java index c0578a99703..cf324b44342 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobLauncher.java @@ -462,9 +462,13 @@ public void launchJob(@Nullable JobListener jobListener) throws JobException { } // TODO: Better error handling. The current impl swallows exceptions for jobs that were started by this method call. - // One potential way to improve the error handling is to make this error swallowing conifgurable + // One potential way to improve the error handling is to make this error swallowing configurable } catch (Throwable t) { errorInJobLaunching = t; + if (isLaunched) { + // Attempts to cancel the helix workflow if an error occurs during launch + cancelJob(jobListener); + } } finally { if (isLaunched) { if (this.runningMap.replace(this.jobContext.getJobName(), true, false)) { diff --git a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobLauncherTest.java b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobLauncherTest.java index 975dad38b79..d475688e5a0 100644 --- a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobLauncherTest.java +++ b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobLauncherTest.java @@ -38,6 +38,7 @@ import org.apache.helix.task.TaskDriver; import org.apache.helix.task.WorkflowConfig; import org.apache.helix.task.WorkflowContext; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -63,6 +64,7 @@ import org.apache.gobblin.runtime.JobException; import org.apache.gobblin.runtime.JobState; import org.apache.gobblin.runtime.listeners.AbstractJobListener; +import org.apache.gobblin.runtime.listeners.JobListener; import org.apache.gobblin.util.ClassAliasResolver; import org.apache.gobblin.util.ConfigUtils; @@ -299,6 +301,35 @@ public void testTimeout() throws Exception { Assert.assertThrows(JobException.class, () -> gobblinHelixJobLauncher.launchJobImpl(null)); } + public void testCancelJobOnFailureDuringLaunch() throws Exception { + final ConcurrentHashMap runningMap = new ConcurrentHashMap<>(); + final Properties props = generateJobProperties(this.baseConfig, "testDoesCancelOnFailure", "_12345"); + props.setProperty(GobblinClusterConfigurationKeys.HELIX_WORKFLOW_SUBMISSION_TIMEOUT_SECONDS, "0"); + + final GobblinHelixJobLauncher gobblinHelixJobLauncher = this.closer.register( + new GobblinHelixJobLauncher(props, this.helixManager, this.appWorkDir, ImmutableList.>of(), runningMap, + java.util.Optional.empty())); + + // The launchJob will throw an exception (see testTimeout test) and we expect the launcher to swallow the exception, + // then call still properly call cancel. We use the listener to confirm the cancel hook was correctly called once + JobListener mockListener = Mockito.mock(JobListener.class); + gobblinHelixJobLauncher.launchJob(mockListener); + Mockito.verify(mockListener).onJobCancellation(Mockito.any(JobContext.class)); + } + + public void testNoCancelWhenJobCompletesSuccessfully() throws Exception { + final ConcurrentHashMap runningMap = new ConcurrentHashMap<>(); + final Properties props = generateJobProperties(this.baseConfig, "testDoesNotCancelOnSuccess", "_12345"); + final GobblinHelixJobLauncher gobblinHelixJobLauncher = this.closer.register( + new GobblinHelixJobLauncher(props, this.helixManager, this.appWorkDir, ImmutableList.>of(), runningMap, + java.util.Optional.empty())); + + // When the job finishes successfully, the cancellation hook should not be invoked + JobListener mockListener = Mockito.mock(JobListener.class); + gobblinHelixJobLauncher.launchJob(mockListener); + Mockito.verify(mockListener, Mockito.never()).onJobCancellation(Mockito.any(JobContext.class)); + } + @Test(enabled = false, dependsOnMethods = {"testLaunchJob", "testLaunchMultipleJobs"}) public void testJobCleanup() throws Exception { final ConcurrentHashMap runningMap = new ConcurrentHashMap<>(); From 1ecce5bc733b047b3c2b945cf4eba2bc21bac241 Mon Sep 17 00:00:00 2001 From: Peiying Ye <112960226+Peiyingy@users.noreply.github.com> Date: Fri, 30 Jun 2023 10:41:44 -0700 Subject: [PATCH 17/30] [GOBBLIN-1840] Helix Job scheduler should not try to replace running workflow if within configured time (#3704) * [GOBBLIN-1840] Helix Job scheduler should not try to replace running workflow if within configured time * [GOBBLIN-1840] Remove unnecessary files * [GOBBLIN-1840] Add config for throttleTimeoutDuration * [GOBBLIN-1840] Clean up format and coding standard * [GOBBLIN-1840] Clean up format layout * [GOBBLIN-1840] Clean up auto format * [GOBBLIN-1840] Clear up empty space * [GOBBLIN-1840] Clarify naming standards and simplify repeated codes * [GOBBLIN-1840] Add Javadoc on GobblinHelixJobSchedulerTest for setting HelixManager as local variable * [GOBBLIN-1840] Optimize imports and fix unit test errors * [GOBBLIN-1840] Rewrite log info and add Javadoc * [GOBBLIN-1840] Remove job status check * [GOBBLIN-1840] Add log info and change config setting * [GOBBLIN-1840] Add @Slf4j in GobblinThrottlingHelixJobLauncherListener * [GOBBLIN-1840] Fix race condition of handleNewJobConfigArrival * [GOBBLIN-1840] Improve mockClock mechanism * [GOBBLIN-1840] Address comments * [GOBBLIN-1840] Only put entry in jobNameToNextSchedulableTime when throttle is enabled * [GOBBLIN-1840] Remove extra schedulable time updates * [GOBBLIN-1840] Fix checkstyle problems --------- Co-authored-by: Peiying Ye --- .../GobblinClusterConfigurationKeys.java | 9 + .../cluster/GobblinHelixJobScheduler.java | 134 ++++++++-- ...linThrottlingHelixJobLauncherListener.java | 69 +++++ .../apache/gobblin/cluster/HelixUtils.java | 6 +- .../cluster/GobblinHelixJobSchedulerTest.java | 243 ++++++++++++++---- 5 files changed, 385 insertions(+), 76 deletions(-) create mode 100644 gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinThrottlingHelixJobLauncherListener.java diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java index 31a8547aa98..ef83ab029a2 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java @@ -17,6 +17,8 @@ package org.apache.gobblin.cluster; +import java.time.Duration; + import org.apache.gobblin.annotation.Alpha; @@ -222,4 +224,11 @@ public class GobblinClusterConfigurationKeys { public static final String CONTAINER_ID_KEY = GOBBLIN_HELIX_PREFIX + "containerId"; public static final String GOBBLIN_CLUSTER_SYSTEM_PROPERTY_PREFIX = GOBBLIN_CLUSTER_PREFIX + "sysProps"; + + public static final String HELIX_JOB_SCHEDULING_THROTTLE_ENABLED_KEY = "helix.job.scheduling.throttle.enabled"; + public static final boolean DEFAULT_HELIX_JOB_SCHEDULING_THROTTLE_ENABLED_KEY = false; + + public static final String HELIX_JOB_SCHEDULING_THROTTLE_TIMEOUT_SECONDS_KEY = "helix.job.scheduling.throttle.timeout.seconds"; + public static final long DEFAULT_HELIX_JOB_SCHEDULING_THROTTLE_TIMEOUT_SECONDS_KEY = Duration.ofMinutes(40).getSeconds();; + } diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java index 20f47da3e50..d30554d2bb3 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinHelixJobScheduler.java @@ -18,6 +18,10 @@ package org.apache.gobblin.cluster; import java.io.IOException; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -111,14 +115,28 @@ public class GobblinHelixJobScheduler extends JobScheduler implements StandardMe private boolean startServicesCompleted; private final long helixJobStopTimeoutMillis; + /** + * The throttling timeout prevents helix workflows with the same job name / URI from being submitted + * more than once within the timeout period. This timeout is not reset by deletes / cancels, meaning that + * if you delete a workflow within the timeout period, you cannot reschedule until the timeout period is complete. + * However, if there is an error when launching the job, you can immediately reschedule the flow.

+ * + * NOTE: This throttle timeout period starts when the job launcher thread picks up the runnable. Meaning that the + * time it takes to submit to Helix and start running the flow is also included as part of the timeout period + */ + private final Duration jobSchedulingThrottleTimeout; + private ConcurrentHashMap jobNameToNextSchedulableTime; + private boolean isThrottleEnabled; + private Clock clock; + public GobblinHelixJobScheduler(Config sysConfig, HelixManager jobHelixManager, Optional taskDriverHelixManager, EventBus eventBus, Path appWorkDir, List> metadataTags, SchedulerService schedulerService, - MutableJobCatalog jobCatalog) throws Exception { - + MutableJobCatalog jobCatalog, + Clock clock) throws Exception { super(ConfigUtils.configToProperties(sysConfig), schedulerService); this.commonJobProperties = ConfigUtils.configToProperties(ConfigUtils.getConfigOrEmpty(sysConfig, COMMON_JOB_PROPS)); this.jobHelixManager = jobHelixManager; @@ -162,6 +180,27 @@ public GobblinHelixJobScheduler(Config sysConfig, this.helixWorkflowListingTimeoutMillis = ConfigUtils.getLong(sysConfig, GobblinClusterConfigurationKeys.HELIX_WORKFLOW_LISTING_TIMEOUT_SECONDS, GobblinClusterConfigurationKeys.DEFAULT_HELIX_WORKFLOW_LISTING_TIMEOUT_SECONDS) * 1000; + this.jobSchedulingThrottleTimeout = Duration.of(ConfigUtils.getLong(sysConfig, GobblinClusterConfigurationKeys.HELIX_JOB_SCHEDULING_THROTTLE_TIMEOUT_SECONDS_KEY, + GobblinClusterConfigurationKeys.DEFAULT_HELIX_JOB_SCHEDULING_THROTTLE_TIMEOUT_SECONDS_KEY), ChronoUnit.SECONDS); + + this.jobNameToNextSchedulableTime = new ConcurrentHashMap<>(); + + this.isThrottleEnabled = ConfigUtils.getBoolean(sysConfig, GobblinClusterConfigurationKeys.HELIX_JOB_SCHEDULING_THROTTLE_ENABLED_KEY, + GobblinClusterConfigurationKeys.DEFAULT_HELIX_JOB_SCHEDULING_THROTTLE_ENABLED_KEY); + + this.clock = clock; + } + + public GobblinHelixJobScheduler(Config sysConfig, + HelixManager jobHelixManager, + Optional taskDriverHelixManager, + EventBus eventBus, + Path appWorkDir, List> metadataTags, + SchedulerService schedulerService, + MutableJobCatalog jobCatalog) throws Exception { + + this(sysConfig, jobHelixManager, taskDriverHelixManager, eventBus, appWorkDir, metadataTags, + schedulerService, jobCatalog, Clock.systemUTC()); } @Override @@ -206,9 +245,9 @@ protected void startServices() throws Exception { if (cleanAllDistJobs) { for (org.apache.gobblin.configuration.State state : this.jobsMapping.getAllStates()) { - String jobUri = state.getId(); - LOGGER.info("Delete mapping for job " + jobUri); - this.jobsMapping.deleteMapping(jobUri); + String jobName = state.getId(); + LOGGER.info("Delete mapping for job " + jobName); + this.jobsMapping.deleteMapping(jobName); } } } @@ -303,36 +342,70 @@ public Object get(long timeout, TimeUnit unit) throws InterruptedException, Exec } @Subscribe - public void handleNewJobConfigArrival(NewJobConfigArrivalEvent newJobArrival) { - String jobUri = newJobArrival.getJobName(); - LOGGER.info("Received new job configuration of job " + jobUri); + public synchronized void handleNewJobConfigArrival(NewJobConfigArrivalEvent newJobArrival) { + String jobName = newJobArrival.getJobName(); + LOGGER.info("Received new job configuration of job " + jobName); + + Instant nextSchedulableTime = jobNameToNextSchedulableTime.getOrDefault(jobName, Instant.EPOCH); + if (this.isThrottleEnabled && clock.instant().isBefore(nextSchedulableTime)) { + LOGGER.info("Adding new job is skipped for job {}. Current time is {} and the next schedulable time would be {}", + jobName, + clock.instant(), + nextSchedulableTime + ); + return; + } + + if (isThrottleEnabled) { + nextSchedulableTime = clock.instant().plus(jobSchedulingThrottleTimeout); + jobNameToNextSchedulableTime.put(jobName, nextSchedulableTime); + } + try { Properties jobProps = new Properties(); jobProps.putAll(this.commonJobProperties); jobProps.putAll(newJobArrival.getJobConfig()); // set uri so that we can delete this job later - jobProps.setProperty(GobblinClusterConfigurationKeys.JOB_SPEC_URI, jobUri); + jobProps.setProperty(GobblinClusterConfigurationKeys.JOB_SPEC_URI, jobName); this.jobSchedulerMetrics.updateTimeBeforeJobScheduling(jobProps); - + GobblinHelixJobLauncherListener listener = isThrottleEnabled ? + new GobblinThrottlingHelixJobLauncherListener(this.launcherMetrics, jobNameToNextSchedulableTime) + : new GobblinHelixJobLauncherListener(this.launcherMetrics); if (jobProps.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY)) { - LOGGER.info("Scheduling job " + jobUri); - scheduleJob(jobProps, - new GobblinHelixJobLauncherListener(this.launcherMetrics)); + LOGGER.info("Scheduling job " + jobName); + scheduleJob(jobProps, listener); } else { - LOGGER.info("No job schedule found, so running job " + jobUri); - this.jobExecutor.execute(new NonScheduledJobRunner(jobProps, - new GobblinHelixJobLauncherListener(this.launcherMetrics))); + LOGGER.info("No job schedule found, so running job " + jobName); + this.jobExecutor.execute(new NonScheduledJobRunner(jobProps, listener)); } } catch (JobException je) { - LOGGER.error("Failed to schedule or run job " + jobUri, je); + LOGGER.error("Failed to schedule or run job {} . Reset the next scheduable time to {}", + jobName, + Instant.EPOCH, + je); + if (isThrottleEnabled) { + jobNameToNextSchedulableTime.put(jobName, Instant.EPOCH); + } } } @Subscribe - public void handleUpdateJobConfigArrival(UpdateJobConfigArrivalEvent updateJobArrival) { + public synchronized void handleUpdateJobConfigArrival(UpdateJobConfigArrivalEvent updateJobArrival) { LOGGER.info("Received update for job configuration of job " + updateJobArrival.getJobName()); + String jobName = updateJobArrival.getJobName(); + + Instant nextSchedulableTime = jobNameToNextSchedulableTime.getOrDefault(jobName, Instant.EPOCH); + if (this.isThrottleEnabled && clock.instant().isBefore(nextSchedulableTime)) { + LOGGER.info("Replanning is skipped for job {}. Current time is {} and the next schedulable time would be {}", + jobName, + clock.instant(), + nextSchedulableTime + ); + return; + } + try { handleDeleteJobConfigArrival(new DeleteJobConfigArrivalEvent(updateJobArrival.getJobName(), updateJobArrival.getJobConfig())); @@ -359,8 +432,17 @@ private void waitForJobCompletion(String jobName) { } } + /*** + * Deleting a workflow with throttling enabled means that the next + * schedulable time for the workflow will remain unchanged. + * Note: In such case, it is required to wait until the throttle + * timeout period elapses before the workflow can be rescheduled. + * + * @param deleteJobArrival + * @throws InterruptedException + */ @Subscribe - public void handleDeleteJobConfigArrival(DeleteJobConfigArrivalEvent deleteJobArrival) throws InterruptedException { + public synchronized void handleDeleteJobConfigArrival(DeleteJobConfigArrivalEvent deleteJobArrival) throws InterruptedException { LOGGER.info("Received delete for job configuration of job " + deleteJobArrival.getJobName()); try { unscheduleJob(deleteJobArrival.getJobName()); @@ -373,8 +455,8 @@ public void handleDeleteJobConfigArrival(DeleteJobConfigArrivalEvent deleteJobAr @Subscribe public void handleCancelJobConfigArrival(CancelJobConfigArrivalEvent cancelJobArrival) throws InterruptedException { - String jobUri = cancelJobArrival.getJoburi(); - LOGGER.info("Received cancel for job configuration of job " + jobUri); + String jobName = cancelJobArrival.getJoburi(); + LOGGER.info("Received cancel for job configuration of job " + jobName); Optional distributedJobMode; Optional planningJob = Optional.empty(); Optional actualJob = Optional.empty(); @@ -384,14 +466,14 @@ public void handleCancelJobConfigArrival(CancelJobConfigArrivalEvent cancelJobAr this.jobSchedulerMetrics.numCancellationStart.incrementAndGet(); try { - distributedJobMode = this.jobsMapping.getDistributedJobMode(jobUri); + distributedJobMode = this.jobsMapping.getDistributedJobMode(jobName); if (distributedJobMode.isPresent() && Boolean.parseBoolean(distributedJobMode.get())) { - planningJob = this.jobsMapping.getPlanningJobId(jobUri); + planningJob = this.jobsMapping.getPlanningJobId(jobName); } else { - actualJob = this.jobsMapping.getActualJobId(jobUri); + actualJob = this.jobsMapping.getActualJobId(jobName); } } catch (IOException e) { - LOGGER.warn("jobsMapping could not be retrieved for job {}", jobUri); + LOGGER.warn("jobsMapping could not be retrieved for job {}", jobName); return; } @@ -466,7 +548,7 @@ public void run() { GobblinHelixJobScheduler.this.jobSchedulerMetrics.updateTimeBetweenJobSchedulingAndJobLaunching(this.creationTimeInMillis, System.currentTimeMillis()); GobblinHelixJobScheduler.this.runJob(this.jobProps, this.jobListener); } catch (JobException je) { - LOGGER.error("Failed to run job " + this.jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY), je); + LOGGER.error("Failed to schedule or run job " + this.jobProps.getProperty(ConfigurationKeys.JOB_NAME_KEY), je); } } } diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinThrottlingHelixJobLauncherListener.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinThrottlingHelixJobLauncherListener.java new file mode 100644 index 00000000000..ec00be19bd1 --- /dev/null +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinThrottlingHelixJobLauncherListener.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.cluster; + +import java.time.Instant; +import java.util.concurrent.ConcurrentHashMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.runtime.JobContext; +import org.apache.gobblin.runtime.JobState; + + +/** + * A job listener used when {@link GobblinHelixJobLauncher} launches a job. + * In {@link GobblinHelixJobScheduler}, when throttling is enabled, this + * listener would record jobName to next schedulable time to decide whether + * the replanning should be executed or skipped. + */ +@Slf4j +public class GobblinThrottlingHelixJobLauncherListener extends GobblinHelixJobLauncherListener { + + public final static Logger LOG = LoggerFactory.getLogger(GobblinThrottlingHelixJobLauncherListener.class); + private ConcurrentHashMap jobNameToNextSchedulableTime; + + public GobblinThrottlingHelixJobLauncherListener(GobblinHelixJobLauncherMetrics jobLauncherMetrics, + ConcurrentHashMap jobNameToNextSchedulableTime) { + super(jobLauncherMetrics); + this.jobNameToNextSchedulableTime = jobNameToNextSchedulableTime; + } + + @Override + public void onJobCompletion(JobContext jobContext) + throws Exception { + super.onJobCompletion(jobContext); + if (jobContext.getJobState().getState() == JobState.RunningState.FAILED) { + jobNameToNextSchedulableTime.put(jobContext.getJobName(), Instant.EPOCH); + LOG.info("{} failed. The next schedulable time is {} so that any future schedule attempts will be allowed.", + jobContext.getJobName(), Instant.EPOCH); + } + } + + @Override + public void onJobCancellation(JobContext jobContext) + throws Exception { + super.onJobCancellation(jobContext); + jobNameToNextSchedulableTime.put(jobContext.getJobName(), Instant.EPOCH); + LOG.info("{} is cancelled. The next schedulable time is {} so that any future schedule attempts will be allowed.", + jobContext.getJobName(), Instant.EPOCH); + } +} diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java index d6200296717..688c8c7babb 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java @@ -417,7 +417,11 @@ public static Map getWorkflowIdsFromJobNames(TaskDriver taskDriv } Set helixJobs = workflowConfig.getJobDag().getAllNodes(); for (String helixJob : helixJobs) { - Iterator taskConfigIterator = taskDriver.getJobConfig(helixJob).getTaskConfigMap().values().iterator(); + JobConfig jobConfig = taskDriver.getJobConfig(helixJob); + if (jobConfig == null) { + throw new GobblinHelixUnexpectedStateException("Received null jobConfig from Helix. We should not see any null configs when reading all helixJobs. helixJob=%s", helixJob); + } + Iterator taskConfigIterator = jobConfig.getTaskConfigMap().values().iterator(); if (taskConfigIterator.hasNext()) { TaskConfig taskConfig = taskConfigIterator.next(); String jobName = taskConfig.getConfigMap().get(ConfigurationKeys.JOB_NAME_KEY); diff --git a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobSchedulerTest.java b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobSchedulerTest.java index 498e2530aec..e21c7e73874 100644 --- a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobSchedulerTest.java +++ b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobSchedulerTest.java @@ -20,18 +20,24 @@ import java.io.File; import java.io.IOException; import java.net.URL; +import java.nio.file.Files; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.Collections; import java.util.Map; import java.util.Properties; +import java.util.concurrent.atomic.AtomicReference; import org.apache.curator.test.TestingServer; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.helix.HelixManager; -import org.apache.helix.HelixManagerFactory; import org.apache.helix.InstanceType; import org.assertj.core.util.Lists; +import org.mockito.Mockito; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; @@ -52,16 +58,20 @@ import org.apache.gobblin.runtime.job_catalog.NonObservingFSJobCatalog; import org.apache.gobblin.scheduler.SchedulerService; +import static org.mockito.Mockito.when; + /** * Unit tests for {@link org.apache.gobblin.cluster.GobblinHelixJobScheduler}. * + * In all test cases, we use GobblinHelixManagerFactory instead of + * HelixManagerFactory, and instantiate a local HelixManager per test to + * provide isolation and prevent errors caused by the ZKClient being shared + * (e.g. ZKClient is not connected exceptions). */ -@Test(groups = {"gobblin.cluster"}) +@Test(groups = {"gobblin.cluster"}, singleThreaded = true) public class GobblinHelixJobSchedulerTest { public final static Logger LOG = LoggerFactory.getLogger(GobblinHelixJobSchedulerTest.class); - - private HelixManager helixManager; private FileSystem localFs; private Path appWorkDir; private final Closer closer = Closer.create(); @@ -70,10 +80,17 @@ public class GobblinHelixJobSchedulerTest { private GobblinTaskRunner gobblinTaskRunner; private Thread thread; - private final String workflowIdSuffix1 = "_1504201348471"; private final String workflowIdSuffix2 = "_1504201348472"; + private final Instant beginTime = Instant.EPOCH; + private final Duration withinThrottlePeriod = Duration.of(1, ChronoUnit.SECONDS); + private final Duration exceedsThrottlePeriod = Duration.of( + GobblinClusterConfigurationKeys.DEFAULT_HELIX_JOB_SCHEDULING_THROTTLE_TIMEOUT_SECONDS_KEY + 1, ChronoUnit.SECONDS); + + private String zkConnectingString; + private String helixClusterName; + @BeforeClass public void setUp() throws Exception { @@ -96,17 +113,11 @@ public void setUp() ConfigValueFactory.fromAnyRef(sourceJsonFile.getAbsolutePath())) .withValue(ConfigurationKeys.JOB_STATE_IN_STATE_STORE, ConfigValueFactory.fromAnyRef("true")).resolve(); - String zkConnectingString = baseConfig.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); - String helixClusterName = baseConfig.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY); + this.zkConnectingString = baseConfig.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY); + this.helixClusterName = baseConfig.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY); HelixUtils.createGobblinHelixCluster(zkConnectingString, helixClusterName); - this.helixManager = HelixManagerFactory - .getZKHelixManager(helixClusterName, TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, - zkConnectingString); - this.closer.register(() -> helixManager.disconnect()); - this.helixManager.connect(); - this.localFs = FileSystem.getLocal(new Configuration()); this.closer.register(() -> { @@ -129,58 +140,192 @@ public void setUp() this.thread.start(); } + /*** + * Time span exceeds throttle timeout, within same workflow, throttle is enabled + * Job will be updated + * @throws Exception + */ + @Test + public void testUpdateSameWorkflowLongPeriodThrottle() + throws Exception { + runWorkflowTest(exceedsThrottlePeriod, "UpdateSameWorkflowLongPeriodThrottle", + workflowIdSuffix1, workflowIdSuffix2, workflowIdSuffix2, + true, true); + } + + /*** + * Time span is within throttle timeout, within same workflow, throttle is enabled + * Job will not be updated + * @throws Exception + */ + @Test + public void testUpdateSameWorkflowShortPeriodThrottle() + throws Exception { + runWorkflowTest(withinThrottlePeriod, "UpdateSameWorkflowShortPeriodThrottle", + workflowIdSuffix1, workflowIdSuffix2, workflowIdSuffix1, + true, true); + } + + /*** + * Time span exceeds throttle timeout, within same workflow, throttle is not enabled + * Job will be updated + * @throws Exception + */ + @Test + public void testUpdateSameWorkflowLongPeriodNoThrottle() + throws Exception { + runWorkflowTest(exceedsThrottlePeriod, "UpdateSameWorkflowLongPeriodNoThrottle", + workflowIdSuffix1, workflowIdSuffix2, workflowIdSuffix2, + false, true); + } + + /*** + * Time span is within throttle timeout, within same workflow, throttle is not enabled + * Job will be updated + * @throws Exception + */ @Test - public void testNewJobAndUpdate() + public void testUpdateSameWorkflowShortPeriodNoThrottle() + throws Exception { + runWorkflowTest(withinThrottlePeriod, "UpdateSameWorkflowShortPeriodNoThrottle", + workflowIdSuffix1, workflowIdSuffix2, workflowIdSuffix2, + false, true); + } + + /*** + * Time span is within throttle timeout, within different workflow, throttle is enabled + * Job will be updated + * @throws Exception + */ + public void testUpdateDiffWorkflowShortPeriodThrottle() throws Exception { + runWorkflowTest(withinThrottlePeriod, "UpdateDiffWorkflowShortPeriodThrottle", + workflowIdSuffix1, workflowIdSuffix2, workflowIdSuffix2, + true, false); + } + + /*** + * Time span is within throttle timeout, within different workflow, throttle is not enabled + * Job will be updated + * @throws Exception + */ + @Test + public void testUpdateDiffWorkflowShortPeriodNoThrottle() + throws Exception { + runWorkflowTest(withinThrottlePeriod, "UpdateDiffWorkflowShortPeriodNoThrottle", + workflowIdSuffix1, workflowIdSuffix2, workflowIdSuffix2, + false, false); + } + + /*** + * Time span exceeds throttle timeout, within different workflow, throttle is enabled + * Job will be updated + * @throws Exception + */ + @Test + public void testUpdateDiffWorkflowLongPeriodThrottle() + throws Exception { + runWorkflowTest(exceedsThrottlePeriod, "UpdateDiffWorkflowLongPeriodThrottle", + workflowIdSuffix1, workflowIdSuffix2, workflowIdSuffix2, + true, false); + } + + /*** + * Time span exceeds throttle timeout, within different workflow, throttle is not enabled + * Job will be updated + * @throws Exception + */ + @Test + public void testUpdateDiffWorkflowLongPeriodNoThrottle() + throws Exception { + runWorkflowTest(exceedsThrottlePeriod, "UpdateDiffWorkflowLongPeriodNoThrottle", + workflowIdSuffix1, workflowIdSuffix2, workflowIdSuffix2, + false, false); + } + + private GobblinHelixJobScheduler createJobScheduler(HelixManager helixManager, boolean isThrottleEnabled, Clock clock) throws Exception { + java.nio.file.Path p = Files.createTempDirectory(GobblinHelixJobScheduler.class.getSimpleName()); Config config = ConfigFactory.empty().withValue(ConfigurationKeys.JOB_CONFIG_FILE_GENERAL_PATH_KEY, - ConfigValueFactory.fromAnyRef("/tmp/" + GobblinHelixJobScheduler.class.getSimpleName())); + ConfigValueFactory.fromAnyRef(p.toString())); SchedulerService schedulerService = new SchedulerService(new Properties()); NonObservingFSJobCatalog jobCatalog = new NonObservingFSJobCatalog(config); jobCatalog.startAsync(); - GobblinHelixJobScheduler jobScheduler = - new GobblinHelixJobScheduler(ConfigFactory.empty(), this.helixManager, java.util.Optional.empty(), - new EventBus(), appWorkDir, Lists.emptyList(), schedulerService, jobCatalog); - - final Properties properties1 = - GobblinHelixJobLauncherTest.generateJobProperties(this.baseConfig, "1", workflowIdSuffix1); - properties1.setProperty(GobblinClusterConfigurationKeys.CANCEL_RUNNING_JOB_ON_DELETE, "true"); + Config helixJobSchedulerConfig = ConfigFactory.empty().withValue(GobblinClusterConfigurationKeys.HELIX_JOB_SCHEDULING_THROTTLE_ENABLED_KEY, + ConfigValueFactory.fromAnyRef(isThrottleEnabled)); + GobblinHelixJobScheduler gobblinHelixJobScheduler = new GobblinHelixJobScheduler(helixJobSchedulerConfig, helixManager, java.util.Optional.empty(), + new EventBus(), appWorkDir, Lists.emptyList(), schedulerService, jobCatalog, clock); + return gobblinHelixJobScheduler; + } + private NewJobConfigArrivalEvent createJobConfigArrivalEvent(Properties properties) { + properties.setProperty(GobblinClusterConfigurationKeys.CANCEL_RUNNING_JOB_ON_DELETE, "true"); NewJobConfigArrivalEvent newJobConfigArrivalEvent = - new NewJobConfigArrivalEvent(properties1.getProperty(ConfigurationKeys.JOB_NAME_KEY), properties1); - jobScheduler.handleNewJobConfigArrival(newJobConfigArrivalEvent); - properties1.setProperty(ConfigurationKeys.JOB_ID_KEY, - "job_" + properties1.getProperty(ConfigurationKeys.JOB_NAME_KEY) + workflowIdSuffix2); - Map workflowIdMap; - this.helixManager.connect(); + new NewJobConfigArrivalEvent(properties.getProperty(ConfigurationKeys.JOB_NAME_KEY), properties); + return newJobConfigArrivalEvent; + } + + private void connectAndAssertWorkflowId(String expectedSuffix, String jobName, HelixManager helixManager) throws Exception { + helixManager.connect(); + String workFlowId = getWorkflowID(jobName, helixManager); + Assert.assertNotNull(workFlowId); + Assert.assertTrue(workFlowId.endsWith(expectedSuffix)); + } - String workFlowId = null; + private String getWorkflowID (String jobName, HelixManager helixManager) + throws Exception { + // Poll helix for up to 30 seconds to fetch until a workflow with a matching job name exists in Helix and then return that workflowID long endTime = System.currentTimeMillis() + 30000; + Map workflowIdMap; while (System.currentTimeMillis() < endTime) { - workflowIdMap = HelixUtils.getWorkflowIdsFromJobNames(this.helixManager, - Collections.singletonList(newJobConfigArrivalEvent.getJobName())); - if (workflowIdMap.containsKey(newJobConfigArrivalEvent.getJobName())) { - workFlowId = workflowIdMap.get(newJobConfigArrivalEvent.getJobName()); - break; + try{ + workflowIdMap = HelixUtils.getWorkflowIdsFromJobNames(helixManager, + Collections.singletonList(jobName)); + } catch(GobblinHelixUnexpectedStateException e){ + continue; + } + if (workflowIdMap.containsKey(jobName)) { + return workflowIdMap.get(jobName); } Thread.sleep(100); } - Assert.assertNotNull(workFlowId); - Assert.assertTrue(workFlowId.endsWith(workflowIdSuffix1)); + return null; + } - jobScheduler.handleUpdateJobConfigArrival( - new UpdateJobConfigArrivalEvent(properties1.getProperty(ConfigurationKeys.JOB_NAME_KEY), properties1)); - this.helixManager.connect(); - endTime = System.currentTimeMillis() + 30000; - while (System.currentTimeMillis() < endTime) { - workflowIdMap = HelixUtils.getWorkflowIdsFromJobNames(this.helixManager, - Collections.singletonList(newJobConfigArrivalEvent.getJobName())); - if (workflowIdMap.containsKey(newJobConfigArrivalEvent.getJobName())) { - workFlowId = workflowIdMap.get(newJobConfigArrivalEvent.getJobName()); - break; - } - Thread.sleep(100); + private void runWorkflowTest(Duration mockStepAmountTime, String jobSuffix, + String newJobWorkflowIdSuffix, String updateWorkflowIdSuffix, + String assertUpdateWorkflowIdSuffix, boolean isThrottleEnabled, boolean isSameWorkflow) throws Exception { + Clock mockClock = Mockito.mock(Clock.class); + AtomicReference nextInstant = new AtomicReference<>(beginTime); + when(mockClock.instant()).thenAnswer(invocation -> nextInstant.getAndAccumulate(null, (currentInstant, x) -> currentInstant.plus(mockStepAmountTime))); + + // Use GobblinHelixManagerFactory instead of HelixManagerFactory to avoid the connection error + // helixManager is set to local variable to avoid the HelixManager (ZkClient) is not connected error across tests + HelixManager helixManager = GobblinHelixManagerFactory + .getZKHelixManager(helixClusterName, TestHelper.TEST_HELIX_INSTANCE_NAME, InstanceType.CONTROLLER, + zkConnectingString); + GobblinHelixJobScheduler jobScheduler = createJobScheduler(helixManager, isThrottleEnabled, mockClock); + final Properties properties = GobblinHelixJobLauncherTest.generateJobProperties(this.baseConfig, jobSuffix, newJobWorkflowIdSuffix); + NewJobConfigArrivalEvent newJobConfigArrivalEvent = createJobConfigArrivalEvent(properties); + jobScheduler.handleNewJobConfigArrival(newJobConfigArrivalEvent); + connectAndAssertWorkflowId(newJobWorkflowIdSuffix, newJobConfigArrivalEvent.getJobName(), helixManager); + + if (isSameWorkflow) { + properties.setProperty(ConfigurationKeys.JOB_ID_KEY, + "job_" + properties.getProperty(ConfigurationKeys.JOB_NAME_KEY) + updateWorkflowIdSuffix); + jobScheduler.handleUpdateJobConfigArrival( + new UpdateJobConfigArrivalEvent(properties.getProperty(ConfigurationKeys.JOB_NAME_KEY), properties)); + connectAndAssertWorkflowId(assertUpdateWorkflowIdSuffix, newJobConfigArrivalEvent.getJobName(), helixManager); + } + else { + final Properties properties2 = + GobblinHelixJobLauncherTest.generateJobProperties( + this.baseConfig, jobSuffix + '2', updateWorkflowIdSuffix); + NewJobConfigArrivalEvent newJobConfigArrivalEvent2 = + new NewJobConfigArrivalEvent(properties2.getProperty(ConfigurationKeys.JOB_NAME_KEY), properties2); + jobScheduler.handleUpdateJobConfigArrival( + new UpdateJobConfigArrivalEvent(properties2.getProperty(ConfigurationKeys.JOB_NAME_KEY), properties2)); + connectAndAssertWorkflowId(assertUpdateWorkflowIdSuffix, newJobConfigArrivalEvent2.getJobName(), helixManager); } - Assert.assertTrue(workFlowId.endsWith(workflowIdSuffix2)); } @AfterClass From 3d14b86a5bdb1b99d7fc13a5864a831ff9221b1c Mon Sep 17 00:00:00 2001 From: Peiying Ye <112960226+Peiyingy@users.noreply.github.com> Date: Fri, 30 Jun 2023 14:41:04 -0700 Subject: [PATCH 18/30] [GOBBLIN-1841] Move disabling of current live instances to the GobblinClusterManager startup (#3708) * [GOBBLIN-1841] Implement disableLiveHelixInstances and unit test * [GOBBLIN-1841] clear commit history * [GOBBLIN-1841] remove disableLiveHelixInstances from Yarn * [GOBBLIN-1841] remove extra comments * [GOBBLIN-1841] Implement TestGobblinClusterManager class * [GOBBLIN-1841] Remove unnecessary imports * [GOBBLIN-1841] Optimize imports * [GOBBLIN-1841] Move disableTaskRunnersFromPreviousExecutions to GobblinApplicationMaster * [GOBBLIN-1841] Add edge cases check * [GOBBLIN-1840] Fix checkstyle error * [GOBBLIN-1841] Fix GobblinYarnAppLauncherTest testJobCleanup * [GOBBLIN-1841] Add back javadoc to disableTaskRunnersFromPreviousExecutions implementation --- .../cluster/GobblinClusterManager.java | 33 ++++--- .../apache/gobblin/cluster/HelixUtils.java | 14 +++ .../yarn/GobblinApplicationMaster.java | 35 ++++++++ .../gobblin/yarn/GobblinYarnAppLauncher.java | 27 +----- .../gobblin/yarn/YarnAutoScalingManager.java | 15 +--- .../yarn/GobblinApplicationMasterTest.java | 85 +++++++++++++++++++ .../yarn/GobblinYarnAppLauncherTest.java | 25 ++++++ 7 files changed, 185 insertions(+), 49 deletions(-) create mode 100644 gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinApplicationMasterTest.java diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java index c7b74fe9265..05ec790e373 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterManager.java @@ -279,15 +279,7 @@ public synchronized void start() { LOGGER.info("Starting the Gobblin Cluster Manager"); this.eventBus.register(this); - this.multiManager.connect(); - - // Standalone mode registers a handler to clean up on manager leadership change, so only clean up for non-standalone - // mode, such as YARN mode - if (!this.isStandaloneMode) { - this.multiManager.cleanUpJobs(); - } - - configureHelixQuotaBasedTaskScheduling(); + setupHelix(); if (this.isStandaloneMode) { // standalone mode starts non-daemon threads later, so need to have this thread to keep process up @@ -316,6 +308,18 @@ public void run() { this.started = true; } + public synchronized void setupHelix() { + this.multiManager.connect(); + + // Standalone mode registers a handler to clean up on manager leadership change, so only clean up for non-standalone + // mode, such as YARN mode + if (!this.isStandaloneMode) { + this.multiManager.cleanUpJobs(); + } + + configureHelixQuotaBasedTaskScheduling(); + } + /** * Stop the Gobblin Cluster Manager. */ @@ -427,11 +431,18 @@ boolean isHelixManagerConnected() { */ @VisibleForTesting void initializeHelixManager() { - this.multiManager = new GobblinHelixMultiManager( - this.config, aVoid -> GobblinClusterManager.this.getUserDefinedMessageHandlerFactory(), this.eventBus, stopStatus) ; + this.multiManager = createMultiManager(); this.multiManager.addLeadershipChangeAwareComponent(this); } + /*** + * Can be overriden to inject mock GobblinHelixMultiManager + * @return a new GobblinHelixMultiManager + */ + public GobblinHelixMultiManager createMultiManager() { + return new GobblinHelixMultiManager(this.config, aVoid -> GobblinClusterManager.this.getUserDefinedMessageHandlerFactory(), this.eventBus, stopStatus); + } + @VisibleForTesting void sendShutdownRequest() { Criteria criteria = new Criteria(); diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java index 688c8c7babb..45c3685d17a 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixUtils.java @@ -28,11 +28,13 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; import org.apache.helix.HelixAdmin; import org.apache.helix.HelixDataAccessor; import org.apache.helix.HelixException; import org.apache.helix.HelixManager; +import org.apache.helix.HelixProperty; import org.apache.helix.PropertyKey; import org.apache.helix.manager.zk.ZKHelixManager; import org.apache.helix.model.HelixConfigScope; @@ -452,6 +454,18 @@ public static List getLiveInstances(HelixManager helixManager) { return accessor.getChildNames(liveInstancesKey); } + /** + * Getting all instances (Helix Participants) in cluster at this moment. + * Note that the raw result could contain AppMaster node and replanner node. + * @param filterString Helix instances whose name containing fitlerString will pass filtering. + */ + public static Set getParticipants(HelixDataAccessor helixDataAccessor, String filterString) { + PropertyKey.Builder keyBuilder = helixDataAccessor.keyBuilder(); + PropertyKey liveInstance = keyBuilder.liveInstances(); + Map childValuesMap = helixDataAccessor.getChildValuesMap(liveInstance); + return childValuesMap.keySet().stream().filter(x -> filterString.isEmpty() || x.contains(filterString)).collect(Collectors.toSet()); + } + public static boolean isInstanceLive(HelixManager helixManager, String instanceName) { HelixDataAccessor accessor = helixManager.getHelixDataAccessor(); PropertyKey liveInstanceKey = accessor.keyBuilder().liveInstance(instanceName); diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java index 16cb954802e..ff2cb0e1a63 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinApplicationMaster.java @@ -19,6 +19,7 @@ import java.util.Collections; import java.util.List; +import java.util.Set; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.DefaultParser; @@ -32,6 +33,9 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.helix.HelixAdmin; +import org.apache.helix.HelixDataAccessor; +import org.apache.helix.HelixManager; import org.apache.helix.NotificationContext; import org.apache.helix.messaging.handling.HelixTaskResult; import org.apache.helix.messaging.handling.MessageHandler; @@ -52,6 +56,8 @@ import org.apache.gobblin.cluster.GobblinClusterConfigurationKeys; import org.apache.gobblin.cluster.GobblinClusterManager; import org.apache.gobblin.cluster.GobblinClusterUtils; +import org.apache.gobblin.cluster.GobblinHelixMultiManager; +import org.apache.gobblin.cluster.HelixUtils; import org.apache.gobblin.util.ConfigUtils; import org.apache.gobblin.util.JvmUtils; import org.apache.gobblin.util.PathUtils; @@ -135,6 +141,35 @@ protected MultiTypeMessageHandlerFactory getUserDefinedMessageHandlerFactory() { return new ControllerUserDefinedMessageHandlerFactory(); } + @Override + public synchronized void setupHelix() { + super.setupHelix(); + this.disableTaskRunnersFromPreviousExecutions(this.multiManager); + } + + /** + * A method to disable pre-existing live instances in a Helix cluster. This can happen when a previous Yarn application + * leaves behind orphaned Yarn worker processes. Since Helix does not provide an API to drop a live instance, we use + * the disable instance API to fence off these orphaned instances and prevent them from becoming participants in the + * new cluster. + * + * NOTE: this is a workaround for an existing YARN bug. Once YARN has a fix to guarantee container kills on application + * completion, this method should be removed. + */ + public static void disableTaskRunnersFromPreviousExecutions(GobblinHelixMultiManager multiManager) { + HelixManager helixManager = multiManager.getJobClusterHelixManager(); + HelixDataAccessor helixDataAccessor = helixManager.getHelixDataAccessor(); + String clusterName = helixManager.getClusterName(); + HelixAdmin helixAdmin = helixManager.getClusterManagmentTool(); + Set taskRunners = HelixUtils.getParticipants(helixDataAccessor, + GobblinYarnTaskRunner.HELIX_YARN_INSTANCE_NAME_PREFIX); + LOGGER.warn("Found {} task runners in the cluster.", taskRunners.size()); + for (String taskRunner : taskRunners) { + LOGGER.warn("Disabling instance: {}", taskRunner); + helixAdmin.enableInstance(clusterName, taskRunner, false); + } + } + /** * A custom {@link MultiTypeMessageHandlerFactory} for {@link ControllerUserDefinedMessageHandler}s that * handle messages of type {@link org.apache.helix.model.Message.MessageType#USER_DEFINE_MSG}. diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java index 99c4094df50..48ac8947972 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/GobblinYarnAppLauncher.java @@ -38,8 +38,6 @@ import org.apache.avro.Schema; import org.apache.commons.io.FileUtils; import org.apache.commons.mail.EmailException; -import org.apache.gobblin.util.hadoop.TokenUtils; -import org.apache.gobblin.util.reflection.GobblinConstructorUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -71,7 +69,6 @@ import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.hadoop.yarn.util.Records; import org.apache.helix.Criteria; -import org.apache.helix.HelixAdmin; import org.apache.helix.HelixManager; import org.apache.helix.HelixManagerFactory; import org.apache.helix.InstanceType; @@ -117,8 +114,10 @@ import org.apache.gobblin.util.EmailUtils; import org.apache.gobblin.util.ExecutorsUtils; import org.apache.gobblin.util.JvmUtils; +import org.apache.gobblin.util.hadoop.TokenUtils; import org.apache.gobblin.util.io.StreamUtils; import org.apache.gobblin.util.logs.LogCopier; +import org.apache.gobblin.util.reflection.GobblinConstructorUtils; import org.apache.gobblin.yarn.event.ApplicationReportArrivalEvent; import org.apache.gobblin.yarn.event.GetApplicationReportFailureEvent; @@ -371,7 +370,6 @@ public void launch() throws IOException, YarnException, InterruptedException { this.applicationId = getReconnectableApplicationId(); if (!this.applicationId.isPresent()) { - disableLiveHelixInstances(); LOGGER.info("No reconnectable application found so submitting a new application"); this.yarnClient = potentialYarnClients.get(this.originalYarnRMAddress); this.applicationId = Optional.of(setupAndSubmitApplication()); @@ -454,7 +452,6 @@ public synchronized void stop() throws IOException, TimeoutException { if (!this.detachOnExitEnabled) { LOGGER.info("Disabling all live Helix instances.."); - disableLiveHelixInstances(); } disconnectHelixManager(); @@ -540,26 +537,6 @@ void connectHelixManager() { } } - /** - * A method to disable pre-existing live instances in a Helix cluster. This can happen when a previous Yarn application - * leaves behind orphaned Yarn worker processes. Since Helix does not provide an API to drop a live instance, we use - * the disable instance API to fence off these orphaned instances and prevent them from becoming participants in the - * new cluster. - * - * NOTE: this is a workaround for an existing YARN bug. Once YARN has a fix to guarantee container kills on application - * completion, this method should be removed. - */ - void disableLiveHelixInstances() { - String clusterName = this.helixManager.getClusterName(); - HelixAdmin helixAdmin = this.helixManager.getClusterManagmentTool(); - List liveInstances = HelixUtils.getLiveInstances(this.helixManager); - LOGGER.warn("Found {} live instances in the cluster.", liveInstances.size()); - for (String instanceName: liveInstances) { - LOGGER.warn("Disabling instance: {}", instanceName); - helixAdmin.enableInstance(clusterName, instanceName, false); - } - } - @VisibleForTesting void disconnectHelixManager() { if (this.helixManager.isConnected()) { diff --git a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java index a2f4d8372fc..c447af99ecc 100644 --- a/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java +++ b/gobblin-yarn/src/main/java/org/apache/gobblin/yarn/YarnAutoScalingManager.java @@ -34,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.helix.HelixDataAccessor; import org.apache.helix.HelixManager; -import org.apache.helix.PropertyKey; import org.apache.helix.task.JobConfig; import org.apache.helix.task.JobContext; import org.apache.helix.task.JobDag; @@ -56,6 +55,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.cluster.GobblinClusterConfigurationKeys; +import org.apache.gobblin.cluster.HelixUtils; import org.apache.gobblin.util.ConfigUtils; import org.apache.gobblin.util.ExecutorsUtils; @@ -184,17 +184,6 @@ public void run() { } } - /** - * Getting all instances (Helix Participants) in cluster at this moment. - * Note that the raw result could contains AppMaster node and replanner node. - * @param filterString Helix instances whose name containing fitlerString will pass filtering. - */ - private Set getParticipants(String filterString) { - PropertyKey.Builder keyBuilder = helixDataAccessor.keyBuilder(); - return helixDataAccessor.getChildValuesMap(keyBuilder.liveInstances()) - .keySet().stream().filter(x -> filterString.isEmpty() || x.contains(filterString)).collect(Collectors.toSet()); - } - private String getInuseParticipantForHelixPartition(JobContext jobContext, int partition) { if (jobContext.getPartitionNumAttempts(partition) > THRESHOLD_NUMBER_OF_ATTEMPTS_FOR_LOGGING) { log.warn("Helix task {} has been retried for {} times, please check the config to see how we can handle this task better", @@ -275,7 +264,7 @@ void runInternal() { } // Find all participants appearing in this cluster. Note that Helix instances can contain cluster-manager // and potentially replanner-instance. - Set allParticipants = getParticipants(HELIX_YARN_INSTANCE_NAME_PREFIX); + Set allParticipants = HelixUtils.getParticipants(helixDataAccessor, HELIX_YARN_INSTANCE_NAME_PREFIX); // Find all joined participants not in-use for this round of inspection. // If idle time is beyond tolerance, mark the instance as unused by assigning timestamp as -1. diff --git a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinApplicationMasterTest.java b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinApplicationMasterTest.java new file mode 100644 index 00000000000..947021b04d1 --- /dev/null +++ b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinApplicationMasterTest.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.yarn; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.apache.helix.HelixAdmin; +import org.apache.helix.HelixDataAccessor; +import org.apache.helix.HelixManager; +import org.apache.helix.HelixProperty; +import org.apache.helix.PropertyKey; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import junit.framework.TestCase; + +import org.apache.gobblin.cluster.GobblinHelixMultiManager; + +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; + + +public class GobblinApplicationMasterTest extends TestCase { + @Test + public void testDisableTaskRunnersFromPreviousExecutions() { + GobblinHelixMultiManager mockMultiManager = Mockito.mock(GobblinHelixMultiManager.class); + + HelixManager mockHelixManager = Mockito.mock(HelixManager.class); + when(mockMultiManager.getJobClusterHelixManager()).thenReturn(mockHelixManager); + + HelixAdmin mockHelixAdmin = Mockito.mock(HelixAdmin.class); + when(mockHelixManager.getClusterManagmentTool()).thenReturn(mockHelixAdmin); + when(mockHelixManager.getClusterName()).thenReturn("mockCluster"); + + HelixDataAccessor mockAccessor = Mockito.mock(HelixDataAccessor.class); + when(mockHelixManager.getHelixDataAccessor()).thenReturn(mockAccessor); + + PropertyKey.Builder mockBuilder = Mockito.mock(PropertyKey.Builder.class); + when(mockAccessor.keyBuilder()).thenReturn(mockBuilder); + + PropertyKey mockLiveInstancesKey = Mockito.mock(PropertyKey.class); + when(mockBuilder.liveInstances()).thenReturn(mockLiveInstancesKey); + + int instanceCount = 3; + + // GobblinYarnTaskRunner prefix would be disabled, while GobblinClusterManager prefix will not + ArrayList gobblinYarnTaskRunnerPrefix = new ArrayList(); + ArrayList gobblinClusterManagerPrefix = new ArrayList(); + for (int i = 0; i < instanceCount; i++) { + gobblinYarnTaskRunnerPrefix.add("GobblinYarnTaskRunner_TestInstance_" + i); + gobblinClusterManagerPrefix.add("GobblinClusterManager_TestInstance_" + i); + } + + Map mockChildValues = new HashMap<>(); + for (int i = 0; i < instanceCount; i++) { + mockChildValues.put(gobblinYarnTaskRunnerPrefix.get(i), Mockito.mock(HelixProperty.class)); + mockChildValues.put(gobblinClusterManagerPrefix.get(i), Mockito.mock(HelixProperty.class)); + } + when(mockAccessor.getChildValuesMap(mockLiveInstancesKey)).thenReturn(mockChildValues); + + GobblinApplicationMaster.disableTaskRunnersFromPreviousExecutions(mockMultiManager); + + for (int i = 0; i < instanceCount; i++) { + Mockito.verify(mockHelixAdmin).enableInstance("mockCluster", gobblinYarnTaskRunnerPrefix.get(i), false); + Mockito.verify(mockHelixAdmin, times(0)).enableInstance("mockCluster", gobblinClusterManagerPrefix.get(i), false); + } + } +} \ No newline at end of file diff --git a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java index 2eb8c4f32f5..65e9dc26946 100644 --- a/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java +++ b/gobblin-yarn/src/test/java/org/apache/gobblin/yarn/GobblinYarnAppLauncherTest.java @@ -27,6 +27,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -44,9 +45,13 @@ import org.apache.hadoop.yarn.client.api.YarnClient; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.MiniYARNCluster; +import org.apache.helix.HelixAdmin; +import org.apache.helix.HelixDataAccessor; import org.apache.helix.HelixManager; import org.apache.helix.HelixManagerFactory; +import org.apache.helix.HelixProperty; import org.apache.helix.InstanceType; +import org.apache.helix.PropertyKey; import org.apache.helix.model.Message; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; @@ -84,6 +89,7 @@ import org.apache.gobblin.testing.AssertWithBackoff; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.when; /** @@ -431,6 +437,25 @@ public void testJobCleanup() throws Exception { GobblinHelixMultiManager mockMultiManager = Mockito.mock(GobblinHelixMultiManager.class); appMaster.setMultiManager(mockMultiManager); + + HelixManager mockHelixManager = Mockito.mock(HelixManager.class); + when(mockMultiManager.getJobClusterHelixManager()).thenReturn(mockHelixManager); + + HelixAdmin mockHelixAdmin = Mockito.mock(HelixAdmin.class); + when(mockHelixManager.getClusterManagmentTool()).thenReturn(mockHelixAdmin); + + HelixDataAccessor mockAccessor = Mockito.mock(HelixDataAccessor.class); + when(mockHelixManager.getHelixDataAccessor()).thenReturn(mockAccessor); + + PropertyKey.Builder mockBuilder = Mockito.mock(PropertyKey.Builder.class); + when(mockAccessor.keyBuilder()).thenReturn(mockBuilder); + + PropertyKey mockLiveInstancesKey = Mockito.mock(PropertyKey.class); + when(mockBuilder.liveInstances()).thenReturn(mockLiveInstancesKey); + + Map mockChildValues = new HashMap<>(); + when(mockAccessor.getChildValuesMap(mockLiveInstancesKey)).thenReturn(mockChildValues); + appMaster.start(); Mockito.verify(mockMultiManager, times(1)).cleanUpJobs(); From 2c684767bd9402bbc55d7c2091f71faf8ef3fd77 Mon Sep 17 00:00:00 2001 From: umustafi Date: Mon, 10 Jul 2023 14:32:28 -0700 Subject: [PATCH 19/30] [GOBBLIN-1849] Add Flow Group & Name to Job Config for Job Scheduler (#3713) Co-authored-by: Urmi Mustafi --- .../modules/scheduler/GobblinServiceJobScheduler.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobScheduler.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobScheduler.java index 5e0ac398840..d860ed48587 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobScheduler.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/scheduler/GobblinServiceJobScheduler.java @@ -677,6 +677,13 @@ private Properties createJobConfig(FlowSpec flowSpec) { flowSpecProperties.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY)); } + // Note: the default values for missing flow name/group are different than the ones above to easily identify where + // the values are not present initially + jobConfig.setProperty(ConfigurationKeys.FLOW_NAME_KEY, + flowSpecProperties.getProperty(ConfigurationKeys.FLOW_NAME_KEY, "<>")); + jobConfig.setProperty(ConfigurationKeys.FLOW_GROUP_KEY, + flowSpecProperties.getProperty(ConfigurationKeys.FLOW_GROUP_KEY, "<>")); + return jobConfig; } From 2c2ffac77fed07bcc6486f9dee8cd60e41650ef7 Mon Sep 17 00:00:00 2001 From: William Lo Date: Mon, 10 Jul 2023 17:44:53 -0400 Subject: [PATCH 20/30] [GOBBLIN-1848] Add tags to dagmanager metrics for extensibility (#3712) * Add tags to dagmanager metrics for extensibility * Fix concurrency bug in test * Add job level metrics in dagmanager test * Test not cleaning dm threads * Only cleanup metrics if threads started by the dagmanager --- .../gobblin/metrics/MetricTagNames.java | 22 +++++++++++++++++++ .../modules/orchestration/DagManager.java | 2 +- .../orchestration/DagManagerMetrics.java | 15 ++++++++++++- .../modules/orchestration/DagManagerTest.java | 14 +++++++----- 4 files changed, 46 insertions(+), 7 deletions(-) create mode 100644 gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/MetricTagNames.java diff --git a/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/MetricTagNames.java b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/MetricTagNames.java new file mode 100644 index 00000000000..bce085d2b5e --- /dev/null +++ b/gobblin-metrics-libs/gobblin-metrics-base/src/main/java/org/apache/gobblin/metrics/MetricTagNames.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.metrics; + +public class MetricTagNames { + public static final String METRIC_BACKEND_REPRESENTATION = "metricBackendRepresentation"; +} diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java index 80da8a9e99d..acbf9f71da8 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManager.java @@ -230,7 +230,7 @@ public DagManager(Config config, JobStatusRetriever jobStatusRetriever, boolean } else { this.eventSubmitter = Optional.absent(); } - this.dagManagerMetrics = new DagManagerMetrics(metricContext); + this.dagManagerMetrics = new DagManagerMetrics(); TimeUnit jobStartTimeUnit = TimeUnit.valueOf(ConfigUtils.getString(config, JOB_START_SLA_UNITS, ConfigurationKeys.FALLBACK_GOBBLIN_JOB_START_SLA_TIME_UNIT)); this.defaultJobStartSlaTimeMillis = jobStartTimeUnit.toMillis(ConfigUtils.getLong(config, JOB_START_SLA_TIME, ConfigurationKeys.FALLBACK_GOBBLIN_JOB_START_SLA_TIME)); this.jobStatusRetriever = jobStatusRetriever; diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManagerMetrics.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManagerMetrics.java index e9e605b0661..a5f34cff7f2 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManagerMetrics.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/DagManagerMetrics.java @@ -28,16 +28,22 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.instrumented.GobblinMetricsKeys; +import org.apache.gobblin.instrumented.Instrumented; import org.apache.gobblin.metrics.ContextAwareCounter; import org.apache.gobblin.metrics.ContextAwareGauge; import org.apache.gobblin.metrics.ContextAwareMeter; +import org.apache.gobblin.metrics.GobblinMetrics; import org.apache.gobblin.metrics.MetricContext; +import org.apache.gobblin.metrics.MetricTagNames; import org.apache.gobblin.metrics.RootMetricContext; import org.apache.gobblin.metrics.ServiceMetricNames; +import org.apache.gobblin.metrics.Tag; import org.apache.gobblin.metrics.metric.filter.MetricNameRegexFilter; import org.apache.gobblin.service.FlowId; import org.apache.gobblin.service.RequesterService; @@ -75,6 +81,13 @@ public DagManagerMetrics(MetricContext metricContext) { this.metricContext = metricContext; } + public DagManagerMetrics() { + // Create a new metric context for the DagManagerMetrics tagged appropriately + List> tags = new ArrayList<>(); + tags.add(new Tag<>(MetricTagNames.METRIC_BACKEND_REPRESENTATION, GobblinMetrics.MetricType.COUNTER)); + this.metricContext = Instrumented.getMetricContext(ConfigUtils.configToState(ConfigFactory.empty()), this.getClass(), tags); + } + public void activate() { if (this.metricContext != null) { allSuccessfulMeter = metricContext.contextAwareMeter(MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX, @@ -249,7 +262,7 @@ protected static MetricNameRegexFilter getMetricsFilterForDagManager() { public void cleanup() { // Add null check so that unit test will not affect each other when we de-active non-instrumented DagManager - if(this.metricContext != null) { + if(this.metricContext != null && this.metricContext.getTagMap().get(GobblinMetricsKeys.CLASS_META).equals(DagManager.class.getSimpleName())) { // The DMThread's metrics mappings follow the lifecycle of the DMThread itself and so are lost by DM deactivation-reactivation but the RootMetricContext is a (persistent) singleton. // To avoid IllegalArgumentException by the RMC preventing (re-)add of a metric already known, remove all metrics that a new DMThread thread would attempt to add (in DagManagerThread::initialize) whenever running post-re-enablement RootMetricContext.get().removeMatching(getMetricsFilterForDagManager()); diff --git a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/DagManagerTest.java b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/DagManagerTest.java index 0a572cf26f3..2babd068311 100644 --- a/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/DagManagerTest.java +++ b/gobblin-service/src/test/java/org/apache/gobblin/service/modules/orchestration/DagManagerTest.java @@ -709,6 +709,9 @@ public void testResumeCancelledDag() throws URISyntaxException, IOException { @Test (dependsOnMethods = "testResumeCancelledDag") public void testJobStartSLAKilledDag() throws URISyntaxException, IOException { + String slakilledMeterName = MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX, "job0", ServiceMetricNames.START_SLA_EXCEEDED_FLOWS_METER); + long slaKilledMeterCount = metricContext.getParent().get().getMeters().get(slakilledMeterName) == null? 0 : + metricContext.getParent().get().getMeters().get(slakilledMeterName).getCount(); long flowExecutionId = System.currentTimeMillis(); String flowGroupId = "0"; String flowGroup = "group" + flowGroupId; @@ -780,8 +783,7 @@ public void testJobStartSLAKilledDag() throws URISyntaxException, IOException { Assert.assertEquals(this.dagToJobs.size(), 1); Assert.assertTrue(this.dags.containsKey(dagId1)); - String slakilledMeterName = MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX, "job0", ServiceMetricNames.START_SLA_EXCEEDED_FLOWS_METER); - Assert.assertEquals(metricContext.getParent().get().getMeters().get(slakilledMeterName).getCount(), 1); + Assert.assertEquals(metricContext.getParent().get().getMeters().get(slakilledMeterName).getCount(), slaKilledMeterCount + 1); // Cleanup this._dagManagerThread.run(); @@ -1162,10 +1164,13 @@ public void testJobSlaKilledMetrics() throws URISyntaxException, IOException { Config executorOneConfig = ConfigFactory.empty() .withValue(ConfigurationKeys.SPECEXECUTOR_INSTANCE_URI_KEY, ConfigValueFactory.fromAnyRef("executorOne")) .withValue(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, ConfigValueFactory.fromAnyRef(flowExecutionId)) - .withValue(ConfigurationKeys.GOBBLIN_FLOW_SLA_TIME, ConfigValueFactory.fromAnyRef(10)); + .withValue(ConfigurationKeys.GOBBLIN_FLOW_SLA_TIME, ConfigValueFactory.fromAnyRef(10)) + .withValue(ConfigurationKeys.GOBBLIN_OUTPUT_JOB_LEVEL_METRICS, ConfigValueFactory.fromAnyRef(true)); Config executorTwoConfig = ConfigFactory.empty() .withValue(ConfigurationKeys.SPECEXECUTOR_INSTANCE_URI_KEY, ConfigValueFactory.fromAnyRef("executorTwo")) - .withValue(ConfigurationKeys.GOBBLIN_FLOW_SLA_TIME, ConfigValueFactory.fromAnyRef(10)); + .withValue(ConfigurationKeys.GOBBLIN_FLOW_SLA_TIME, ConfigValueFactory.fromAnyRef(10)) + .withValue(ConfigurationKeys.GOBBLIN_OUTPUT_JOB_LEVEL_METRICS, ConfigValueFactory.fromAnyRef(true)); + List> dagList = buildDagList(2, "newUser", executorOneConfig); dagList.add(buildDag("2", flowExecutionId, "FINISH_RUNNING", 1, "newUser", executorTwoConfig)); @@ -1229,7 +1234,6 @@ public void testJobSlaKilledMetrics() throws URISyntaxException, IOException { String slakilledMeterName2 = MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX, "executorTwo", ServiceMetricNames.SLA_EXCEEDED_FLOWS_METER); String failedFlowGauge = MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX, "group1","flow1", ServiceMetricNames.RUNNING_STATUS); - String slakilledGroupName = MetricRegistry.name(ServiceMetricNames.GOBBLIN_SERVICE_PREFIX, "group0", ServiceMetricNames.SLA_EXCEEDED_FLOWS_METER); Assert.assertEquals(metricContext.getParent().get().getMeters().get(slakilledMeterName1).getCount(), 2); Assert.assertEquals(metricContext.getParent().get().getMeters().get(slakilledMeterName2).getCount(), 1); // Cleanup From ca48bcd0aee7bb4ecd7277b25944bff9f4fc378f Mon Sep 17 00:00:00 2001 From: umustafi Date: Tue, 18 Jul 2023 14:03:35 -0700 Subject: [PATCH 21/30] [GOBBLIN-1851] Unit tests for MysqlMultiActiveLeaseArbiter with Single Participant (#3715) * Unit tests and corresponding fixes for 6 lease acquisition cases * remove logs * address review comments with small changes * clean up naming, logs, java doc * close resultSets * check if test passes on github * fix unit test * insert or update constants into table correctly * rename prop * address comments --------- Co-authored-by: Urmi Mustafi Co-authored-by: Urmi Mustafi --- .../configuration/ConfigurationKeys.java | 2 +- .../api/MysqlMultiActiveLeaseArbiter.java | 326 ++++++++++++------ .../api/MysqlMultiActiveLeaseArbiterTest.java | 146 ++++++++ .../core/GobblinServiceGuiceModule.java | 2 +- .../orchestration/FlowTriggerHandler.java | 75 ++-- 5 files changed, 413 insertions(+), 138 deletions(-) create mode 100644 gobblin-runtime/src/test/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiterTest.java diff --git a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java index 2d60fd5c83d..50bd2a03d33 100644 --- a/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java +++ b/gobblin-api/src/main/java/org/apache/gobblin/configuration/ConfigurationKeys.java @@ -99,7 +99,7 @@ public class ConfigurationKeys { public static final String MYSQL_LEASE_ARBITER_PREFIX = "MysqlMultiActiveLeaseArbiter"; public static final String MULTI_ACTIVE_SCHEDULER_CONSTANTS_DB_TABLE_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".constantsTable"; public static final String DEFAULT_MULTI_ACTIVE_SCHEDULER_CONSTANTS_DB_TABLE = "gobblin_multi_active_scheduler_constants_store"; - public static final String SCHEDULER_LEASE_DETERMINATION_STORE_DB_TABLE_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".schedulerLeaseArbiterTable"; + public static final String SCHEDULER_LEASE_DETERMINATION_STORE_DB_TABLE_KEY = MYSQL_LEASE_ARBITER_PREFIX + ".schedulerLeaseArbiter.store.db.table"; public static final String DEFAULT_SCHEDULER_LEASE_DETERMINATION_STORE_DB_TABLE = "gobblin_scheduler_lease_determination_store"; public static final String SCHEDULER_EVENT_TO_REVISIT_TIMESTAMP_MILLIS_KEY = "eventToRevisitTimestampMillis"; public static final String SCHEDULER_EVENT_TO_TRIGGER_TIMESTAMP_MILLIS_KEY = "triggerEventTimestampMillis"; diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java index 8a40c71b2e5..9318d8b56f6 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java @@ -17,6 +17,7 @@ package org.apache.gobblin.runtime.api; +import com.google.common.base.Optional; import java.io.IOException; import java.sql.Connection; import java.sql.PreparedStatement; @@ -29,6 +30,7 @@ import com.zaxxer.hikari.HikariDataSource; import javax.sql.DataSource; +import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.gobblin.broker.SharedResourcesBrokerFactory; @@ -54,6 +56,11 @@ * than epsilon and encapsulate executor communication latency including retry attempts * * The `event_timestamp` is the time of the flow_action event request. + * --- Note --- + * We only use the participant's local event_timestamp internally to identify the particular flow_action event, but + * after interacting with the database utilize the CURRENT_TIMESTAMP of the database to insert or keep + * track of our event. This is to avoid any discrepancies due to clock drift between participants as well as + * variation in local time and database time for future comparisons. * ---Event consolidation--- * Note that for the sake of simplification, we only allow one event associated with a particular flow's flow_action * (ie: only one LAUNCH for example of flow FOO, but there can be a LAUNCH, KILL, & RESUME for flow FOO at once) during @@ -80,41 +87,48 @@ protected interface CheckedFunction { private final String constantsTableName; private final int epsilon; private final int linger; + private String thisTableGetInfoStatement; + private String thisTableSelectAfterInsertStatement; // TODO: define retention on this table - private static final String CREATE_LEASE_ARBITER_TABLE_STATEMENT = "CREATE TABLE IF NOT EXISTS %S (" + private static final String CREATE_LEASE_ARBITER_TABLE_STATEMENT = "CREATE TABLE IF NOT EXISTS %s (" + "flow_group varchar(" + ServiceConfigKeys.MAX_FLOW_GROUP_LENGTH + ") NOT NULL, flow_name varchar(" + ServiceConfigKeys.MAX_FLOW_GROUP_LENGTH + ") NOT NULL, " + "flow_execution_id varchar(" + ServiceConfigKeys.MAX_FLOW_EXECUTION_ID_LENGTH + ") NOT NULL, flow_action varchar(100) NOT NULL, " + "event_timestamp TIMESTAMP, " - + "lease_acquisition_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP," + + "lease_acquisition_timestamp TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, " + "PRIMARY KEY (flow_group,flow_name,flow_execution_id,flow_action))"; private static final String CREATE_CONSTANTS_TABLE_STATEMENT = "CREATE TABLE IF NOT EXISTS %s " - + "(epsilon INT, linger INT), PRIMARY KEY (epsilon, linger); INSERT INTO %s (epsilon, linger) VALUES (?,?)"; + + "(primary_key INT, epsilon INT, linger INT, PRIMARY KEY (primary_key))"; + // Only insert epsilon and linger values from config if this table does not contain a pre-existing values already. + private static final String UPSERT_CONSTANTS_TABLE_STATEMENT = "INSERT INTO %s (primary_key, epsilon, linger) " + + "VALUES(1, ?, ?) ON DUPLICATE KEY UPDATE epsilon=VALUES(epsilon), linger=VALUES(linger)"; protected static final String WHERE_CLAUSE_TO_MATCH_KEY = "WHERE flow_group=? AND flow_name=? AND flow_execution_id=?" + " AND flow_action=?"; protected static final String WHERE_CLAUSE_TO_MATCH_ROW = WHERE_CLAUSE_TO_MATCH_KEY + " AND event_timestamp=? AND lease_acquisition_timestamp=?"; - protected static final String SELECT_AFTER_INSERT_STATEMENT = "SELECT ROW_COUNT() AS rows_inserted_count, " - + "lease_acquisition_timestamp, linger FROM %s, %s " + WHERE_CLAUSE_TO_MATCH_KEY; + protected static final String SELECT_AFTER_INSERT_STATEMENT = "SELECT event_timestamp, lease_acquisition_timestamp, " + + "linger FROM %s, %s " + WHERE_CLAUSE_TO_MATCH_KEY; // Does a cross join between the two tables to have epsilon and linger values available. Returns the following values: - // event_timestamp, lease_acquisition_timestamp, isWithinEpsilon (boolean if event_timestamp in table is within - // epsilon), leaseValidityStatus (1 if lease has not expired, 2 if expired, 3 if column is NULL or no longer leasing) + // event_timestamp, lease_acquisition_timestamp, isWithinEpsilon (boolean if current time in db is within epsilon of + // event_timestamp), leaseValidityStatus (1 if lease has not expired, 2 if expired, 3 if column is NULL or no longer + // leasing) protected static final String GET_EVENT_INFO_STATEMENT = "SELECT event_timestamp, lease_acquisition_timestamp, " - + "abs(event_timestamp - ?) <= epsilon as isWithinEpsilon, CASE " - + "WHEN CURRENT_TIMESTAMP < (lease_acquisition_timestamp + linger) then 1" - + "WHEN CURRENT_TIMESTAMP >= (lease_acquisition_timestamp + linger) then 2" - + "ELSE 3 END as leaseValidityStatus, linger FROM %s, %s " + WHERE_CLAUSE_TO_MATCH_KEY; + + "TIMESTAMPDIFF(microsecond, event_timestamp, CURRENT_TIMESTAMP) / 1000 <= epsilon as is_within_epsilon, CASE " + + "WHEN CURRENT_TIMESTAMP < DATE_ADD(lease_acquisition_timestamp, INTERVAL linger*1000 MICROSECOND) then 1 " + + "WHEN CURRENT_TIMESTAMP >= DATE_ADD(lease_acquisition_timestamp, INTERVAL linger*1000 MICROSECOND) then 2 " + + "ELSE 3 END as lease_validity_status, linger, CURRENT_TIMESTAMP FROM %s, %s " + WHERE_CLAUSE_TO_MATCH_KEY; // Insert or update row to acquire lease if values have not changed since the previous read // Need to define three separate statements to handle cases where row does not exist or has null values to check - protected static final String CONDITIONALLY_ACQUIRE_LEASE_IF_NEW_ROW_STATEMENT = "INSERT INTO %s " - + "(flow_group, flow_name, flow_execution_id, flow_action, event_timestamp) VALUES (?, ?, ?, ?, ?)"; + protected static final String CONDITIONALLY_ACQUIRE_LEASE_IF_NEW_ROW_STATEMENT = "INSERT INTO %s (flow_group, " + + "flow_name, flow_execution_id, flow_action, event_timestamp, lease_acquisition_timestamp) " + + "VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)"; protected static final String CONDITIONALLY_ACQUIRE_LEASE_IF_FINISHED_LEASING_STATEMENT = "UPDATE %s " - + "SET event_timestamp=?" + WHERE_CLAUSE_TO_MATCH_KEY - + " AND event_timestamp=? AND lease_acquisition_timestamp is NULL"; + + "SET event_timestamp=CURRENT_TIMESTAMP, lease_acquisition_timestamp=CURRENT_TIMESTAMP " + + WHERE_CLAUSE_TO_MATCH_KEY + " AND event_timestamp=? AND lease_acquisition_timestamp is NULL"; protected static final String CONDITIONALLY_ACQUIRE_LEASE_IF_MATCHING_ALL_COLS_STATEMENT = "UPDATE %s " - + "SET event_timestamp=?" + WHERE_CLAUSE_TO_MATCH_ROW - + " AND event_timestamp=? AND lease_acquisition_timestamp=?"; + + "SET event_timestamp=CURRENT_TIMESTAMP, lease_acquisition_timestamp=CURRENT_TIMESTAMP " + + WHERE_CLAUSE_TO_MATCH_ROW; // Complete lease acquisition if values have not changed since lease was acquired protected static final String CONDITIONALLY_COMPLETE_LEASE_STATEMENT = "UPDATE %s SET " + "lease_acquisition_timestamp = NULL " + WHERE_CLAUSE_TO_MATCH_ROW; @@ -136,81 +150,107 @@ public MysqlMultiActiveLeaseArbiter(Config config) throws IOException { ConfigurationKeys.DEFAULT_SCHEDULER_EVENT_EPSILON_MILLIS); this.linger = ConfigUtils.getInt(config, ConfigurationKeys.SCHEDULER_EVENT_LINGER_MILLIS_KEY, ConfigurationKeys.DEFAULT_SCHEDULER_EVENT_LINGER_MILLIS); + this.thisTableGetInfoStatement = String.format(GET_EVENT_INFO_STATEMENT, this.leaseArbiterTableName, + this.constantsTableName); + this.thisTableSelectAfterInsertStatement = String.format(SELECT_AFTER_INSERT_STATEMENT, this.leaseArbiterTableName, + this.constantsTableName); this.dataSource = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); + String createArbiterStatement = String.format( + CREATE_LEASE_ARBITER_TABLE_STATEMENT, leaseArbiterTableName); try (Connection connection = dataSource.getConnection(); - PreparedStatement createStatement = connection.prepareStatement(String.format( - CREATE_LEASE_ARBITER_TABLE_STATEMENT, leaseArbiterTableName))) { + PreparedStatement createStatement = connection.prepareStatement(createArbiterStatement)) { createStatement.executeUpdate(); connection.commit(); } catch (SQLException e) { throw new IOException("Table creation failure for " + leaseArbiterTableName, e); } - withPreparedStatement(String.format(CREATE_CONSTANTS_TABLE_STATEMENT, this.constantsTableName, this.constantsTableName), - createStatement -> { + initializeConstantsTable(); + + log.info("MysqlMultiActiveLeaseArbiter initialized"); + } + + // Initialize Constants table if needed and insert row into it if one does not exist + private void initializeConstantsTable() throws IOException { + String createConstantsStatement = String.format(CREATE_CONSTANTS_TABLE_STATEMENT, this.constantsTableName); + withPreparedStatement(createConstantsStatement, createStatement -> createStatement.executeUpdate(), true); + + String insertConstantsStatement = String.format(UPSERT_CONSTANTS_TABLE_STATEMENT, this.constantsTableName); + withPreparedStatement(insertConstantsStatement, insertStatement -> { int i = 0; - createStatement.setInt(++i, epsilon); - createStatement.setInt(++i, linger); - return createStatement.executeUpdate();}, true); + insertStatement.setInt(++i, epsilon); + insertStatement.setInt(++i, linger); + return insertStatement.executeUpdate(); + }, true); } @Override public LeaseAttemptStatus tryAcquireLease(DagActionStore.DagAction flowAction, long eventTimeMillis) throws IOException { // Check table for an existing entry for this flow action and event time - ResultSet resultSet = withPreparedStatement( - String.format(GET_EVENT_INFO_STATEMENT, this.leaseArbiterTableName, this.constantsTableName), + Optional getResult = withPreparedStatement(thisTableGetInfoStatement, getInfoStatement -> { int i = 0; - getInfoStatement.setTimestamp(++i, new Timestamp(eventTimeMillis)); getInfoStatement.setString(++i, flowAction.getFlowGroup()); getInfoStatement.setString(++i, flowAction.getFlowName()); getInfoStatement.setString(++i, flowAction.getFlowExecutionId()); getInfoStatement.setString(++i, flowAction.getFlowActionType().toString()); - return getInfoStatement.executeQuery(); + ResultSet resultSet = getInfoStatement.executeQuery(); + try { + if (!resultSet.next()) { + return Optional.absent(); + } + return Optional.of(createGetInfoResult(resultSet)); + } finally { + if (resultSet != null) { + resultSet.close(); + } + } }, true); - String formattedSelectAfterInsertStatement = - String.format(SELECT_AFTER_INSERT_STATEMENT, this.leaseArbiterTableName, this.constantsTableName); try { - // CASE 1: If no existing row for this flow action, then go ahead and insert - if (!resultSet.next()) { + if (!getResult.isPresent()) { + log.debug("tryAcquireLease for [{}, eventTimestamp: {}] - CASE 1: no existing row for this flow action, then go" + + " ahead and insert", flowAction, eventTimeMillis); String formattedAcquireLeaseNewRowStatement = String.format(CONDITIONALLY_ACQUIRE_LEASE_IF_NEW_ROW_STATEMENT, this.leaseArbiterTableName); - ResultSet rs = withPreparedStatement( - formattedAcquireLeaseNewRowStatement + "; " + formattedSelectAfterInsertStatement, + int numRowsUpdated = withPreparedStatement(formattedAcquireLeaseNewRowStatement, insertStatement -> { - completeInsertPreparedStatement(insertStatement, flowAction, eventTimeMillis); - return insertStatement.executeQuery(); + completeInsertPreparedStatement(insertStatement, flowAction); + return insertStatement.executeUpdate(); }, true); - return handleResultFromAttemptedLeaseObtainment(rs, flowAction, eventTimeMillis); + return evaluateStatusAfterLeaseAttempt(numRowsUpdated, flowAction, Optional.absent()); } // Extract values from result set - Timestamp dbEventTimestamp = resultSet.getTimestamp("event_timestamp"); - Timestamp dbLeaseAcquisitionTimestamp = resultSet.getTimestamp("lease_acquisition_timestamp"); - boolean isWithinEpsilon = resultSet.getBoolean("isWithinEpsilon"); - int leaseValidityStatus = resultSet.getInt("leaseValidityStatus"); - int dbLinger = resultSet.getInt("linger"); + Timestamp dbEventTimestamp = getResult.get().getDbEventTimestamp(); + Timestamp dbLeaseAcquisitionTimestamp = getResult.get().getDbLeaseAcquisitionTimestamp(); + boolean isWithinEpsilon = getResult.get().isWithinEpsilon(); + int leaseValidityStatus = getResult.get().getLeaseValidityStatus(); + // Used to calculate minimum amount of time until a participant should check whether a lease expired + int dbLinger = getResult.get().getDbLinger(); + Timestamp dbCurrentTimestamp = getResult.get().getDbCurrentTimestamp(); + + log.info("Multi-active arbiter replacing local trigger event timestamp [{}, triggerEventTimestamp: {}] with " + + "database eventTimestamp {}", flowAction, eventTimeMillis, dbCurrentTimestamp.getTime()); - // CASE 2: If our event timestamp is older than the last event in db, then skip this trigger - if (eventTimeMillis < dbEventTimestamp.getTime()) { - return new NoLongerLeasingStatus(); - } // Lease is valid if (leaseValidityStatus == 1) { - // CASE 3: Same event, lease is valid if (isWithinEpsilon) { + log.debug("tryAcquireLease for [{}, eventTimestamp: {}] - CASE 2: Same event, lease is valid", flowAction, + dbCurrentTimestamp.getTime()); // Utilize db timestamp for reminder return new LeasedToAnotherStatus(flowAction, dbEventTimestamp.getTime(), - dbLeaseAcquisitionTimestamp.getTime() + dbLinger - System.currentTimeMillis()); + dbLeaseAcquisitionTimestamp.getTime() + dbLinger - dbCurrentTimestamp.getTime()); } - // CASE 4: Distinct event, lease is valid - // Utilize db timestamp for wait time, but be reminded of own event timestamp - return new LeasedToAnotherStatus(flowAction, eventTimeMillis, - dbLeaseAcquisitionTimestamp.getTime() + dbLinger - System.currentTimeMillis()); + log.debug("tryAcquireLease for [{}, eventTimestamp: {}] - CASE 3: Distinct event, lease is valid", flowAction, + dbCurrentTimestamp.getTime()); + // Utilize db lease acquisition timestamp for wait time + return new LeasedToAnotherStatus(flowAction, dbCurrentTimestamp.getTime(), + dbLeaseAcquisitionTimestamp.getTime() + dbLinger - dbCurrentTimestamp.getTime()); } - // CASE 5: Lease is out of date (regardless of whether same or distinct event) else if (leaseValidityStatus == 2) { + log.debug("tryAcquireLease for [{}, eventTimestamp: {}] - CASE 4: Lease is out of date (regardless of whether " + + "same or distinct event)", flowAction, dbCurrentTimestamp.getTime()); if (isWithinEpsilon) { log.warn("Lease should not be out of date for the same trigger event since epsilon << linger for flowAction" + " {}, db eventTimestamp {}, db leaseAcquisitionTimestamp {}, linger {}", flowAction, @@ -219,84 +259,143 @@ else if (leaseValidityStatus == 2) { // Use our event to acquire lease, check for previous db eventTimestamp and leaseAcquisitionTimestamp String formattedAcquireLeaseIfMatchingAllStatement = String.format(CONDITIONALLY_ACQUIRE_LEASE_IF_MATCHING_ALL_COLS_STATEMENT, this.leaseArbiterTableName); - ResultSet rs = withPreparedStatement( - formattedAcquireLeaseIfMatchingAllStatement + "; " + formattedSelectAfterInsertStatement, - updateStatement -> { - completeUpdatePreparedStatement(updateStatement, flowAction, eventTimeMillis, true, + int numRowsUpdated = withPreparedStatement(formattedAcquireLeaseIfMatchingAllStatement, + insertStatement -> { + completeUpdatePreparedStatement(insertStatement, flowAction, true, true, dbEventTimestamp, dbLeaseAcquisitionTimestamp); - return updateStatement.executeQuery(); + return insertStatement.executeUpdate(); }, true); - return handleResultFromAttemptedLeaseObtainment(rs, flowAction, eventTimeMillis); + return evaluateStatusAfterLeaseAttempt(numRowsUpdated, flowAction, Optional.of(dbCurrentTimestamp)); } // No longer leasing this event - // CASE 6: Same event, no longer leasing event in db: terminate if (isWithinEpsilon) { + log.debug("tryAcquireLease for [{}, eventTimestamp: {}] - CASE 5: Same event, no longer leasing event in db: " + + "terminate", flowAction, dbCurrentTimestamp.getTime()); return new NoLongerLeasingStatus(); } - // CASE 7: Distinct event, no longer leasing event in db + log.debug("tryAcquireLease for [{}, eventTimestamp: {}] - CASE 6: Distinct event, no longer leasing event in " + + "db", flowAction, dbCurrentTimestamp.getTime()); // Use our event to acquire lease, check for previous db eventTimestamp and NULL leaseAcquisitionTimestamp String formattedAcquireLeaseIfFinishedStatement = String.format(CONDITIONALLY_ACQUIRE_LEASE_IF_FINISHED_LEASING_STATEMENT, this.leaseArbiterTableName); - ResultSet rs = withPreparedStatement( - formattedAcquireLeaseIfFinishedStatement + "; " + formattedSelectAfterInsertStatement, - updateStatement -> { - completeUpdatePreparedStatement(updateStatement, flowAction, eventTimeMillis, true, + int numRowsUpdated = withPreparedStatement(formattedAcquireLeaseIfFinishedStatement, + insertStatement -> { + completeUpdatePreparedStatement(insertStatement, flowAction, true, false, dbEventTimestamp, null); - return updateStatement.executeQuery(); + return insertStatement.executeUpdate(); }, true); - return handleResultFromAttemptedLeaseObtainment(rs, flowAction, eventTimeMillis); + return evaluateStatusAfterLeaseAttempt(numRowsUpdated, flowAction, Optional.of(dbCurrentTimestamp)); } catch (SQLException e) { throw new RuntimeException(e); } } + protected GetEventInfoResult createGetInfoResult(ResultSet resultSet) throws IOException { + try { + // Extract values from result set + Timestamp dbEventTimestamp = resultSet.getTimestamp("event_timestamp"); + Timestamp dbLeaseAcquisitionTimestamp = resultSet.getTimestamp("lease_acquisition_timestamp"); + boolean withinEpsilon = resultSet.getBoolean("is_within_epsilon"); + int leaseValidityStatus = resultSet.getInt("lease_validity_status"); + int dbLinger = resultSet.getInt("linger"); + Timestamp dbCurrentTimestamp = resultSet.getTimestamp("CURRENT_TIMESTAMP"); + return new GetEventInfoResult(dbEventTimestamp, dbLeaseAcquisitionTimestamp, withinEpsilon, leaseValidityStatus, + dbLinger, dbCurrentTimestamp); + } catch (SQLException e) { + throw new IOException(e); + } finally { + if (resultSet != null) { + try { + resultSet.close(); + } catch (SQLException e) { + throw new IOException(e); + } + } + } + } + + protected SelectInfoResult createSelectInfoResult(ResultSet resultSet) throws IOException { + try { + if (!resultSet.next()) { + throw new IOException("Expected num rows and lease_acquisition_timestamp returned from query but received nothing, so " + + "providing empty result to lease evaluation code"); + } + long eventTimeMillis = resultSet.getTimestamp(1).getTime(); + long leaseAcquisitionTimeMillis = resultSet.getTimestamp(2).getTime(); + int dbLinger = resultSet.getInt(3); + return new SelectInfoResult(eventTimeMillis, leaseAcquisitionTimeMillis, dbLinger); + } catch (SQLException e) { + throw new IOException(e); + } finally { + if (resultSet != null) { + try { + resultSet.close(); + } catch (SQLException e) { + throw new IOException(e); + } + } + } + } + /** - * Attempt lease by insert or update following a read based on the condition the state of the table has not changed - * since the read. Parse the result to return the corresponding status based on successful insert/update or not. - * @param resultSet - * @param eventTimeMillis - * @return LeaseAttemptStatus + * Parse result of attempted insert/update to obtain a lease for a + * {@link org.apache.gobblin.runtime.api.DagActionStore.DagAction} event by selecting values corresponding to that + * event from the table to return the corresponding status based on successful insert/update or not. * @throws SQLException * @throws IOException */ - protected LeaseAttemptStatus handleResultFromAttemptedLeaseObtainment(ResultSet resultSet, - DagActionStore.DagAction flowAction, long eventTimeMillis) + protected LeaseAttemptStatus evaluateStatusAfterLeaseAttempt(int numRowsUpdated, + DagActionStore.DagAction flowAction, Optional dbCurrentTimestamp) throws SQLException, IOException { - if (!resultSet.next()) { - throw new IOException("Expected num rows and lease_acquisition_timestamp returned from query but received nothing"); - } - int numRowsUpdated = resultSet.getInt(1); - long leaseAcquisitionTimeMillis = resultSet.getTimestamp(2).getTime(); - int dbLinger = resultSet.getInt(3); + // Fetch values in row after attempted insert + SelectInfoResult selectInfoResult = withPreparedStatement(thisTableSelectAfterInsertStatement, + selectStatement -> { + completeWhereClauseMatchingKeyPreparedStatement(selectStatement, flowAction); + ResultSet resultSet = selectStatement.executeQuery(); + try { + return createSelectInfoResult(resultSet); + } finally { + if (resultSet != null) { + resultSet.close(); + } + } + }, true); if (numRowsUpdated == 1) { - return new LeaseObtainedStatus(flowAction, eventTimeMillis, leaseAcquisitionTimeMillis); + log.debug("Obtained lease for [{}, eventTimestamp: {}] successfully!", flowAction, + selectInfoResult.eventTimeMillis); + return new LeaseObtainedStatus(flowAction, selectInfoResult.eventTimeMillis, + selectInfoResult.getLeaseAcquisitionTimeMillis()); } // Another participant acquired lease in between - return new LeasedToAnotherStatus(flowAction, eventTimeMillis, - leaseAcquisitionTimeMillis + dbLinger - System.currentTimeMillis()); + return new LeasedToAnotherStatus(flowAction, selectInfoResult.getEventTimeMillis(), + selectInfoResult.getLeaseAcquisitionTimeMillis() + selectInfoResult.getDbLinger() + - (dbCurrentTimestamp.isPresent() ? dbCurrentTimestamp.get().getTime() : System.currentTimeMillis())); } /** * Complete the INSERT statement for a new flow action lease where the flow action is not present in the table * @param statement * @param flowAction - * @param eventTimeMillis * @throws SQLException */ - protected void completeInsertPreparedStatement(PreparedStatement statement, DagActionStore.DagAction flowAction, - long eventTimeMillis) throws SQLException { + protected void completeInsertPreparedStatement(PreparedStatement statement, DagActionStore.DagAction flowAction) + throws SQLException { int i = 0; // Values to set in new row statement.setString(++i, flowAction.getFlowGroup()); statement.setString(++i, flowAction.getFlowName()); statement.setString(++i, flowAction.getFlowExecutionId()); statement.setString(++i, flowAction.getFlowActionType().toString()); - statement.setTimestamp(++i, new Timestamp(eventTimeMillis)); - // Values to check if existing row matches previous read - statement.setString(++i, flowAction.getFlowGroup()); - statement.setString(++i, flowAction.getFlowName()); - statement.setString(++i, flowAction.getFlowExecutionId()); - statement.setString(++i, flowAction.getFlowActionType().toString()); - // Values to select for return + } + + /** + * Complete the WHERE clause to match a flow action in a select statement + * @param statement + * @param flowAction + * @throws SQLException + */ + protected void completeWhereClauseMatchingKeyPreparedStatement(PreparedStatement statement, DagActionStore.DagAction flowAction) + throws SQLException { + int i = 0; statement.setString(++i, flowAction.getFlowGroup()); statement.setString(++i, flowAction.getFlowName()); statement.setString(++i, flowAction.getFlowExecutionId()); @@ -308,7 +407,6 @@ protected void completeInsertPreparedStatement(PreparedStatement statement, DagA * updated. * @param statement * @param flowAction - * @param eventTimeMillis * @param needEventTimeCheck true if need to compare `originalEventTimestamp` with db event_timestamp * @param needLeaseAcquisitionTimeCheck true if need to compare `originalLeaseAcquisitionTimestamp` with db one * @param originalEventTimestamp value to compare to db one, null if not needed @@ -316,11 +414,9 @@ protected void completeInsertPreparedStatement(PreparedStatement statement, DagA * @throws SQLException */ protected void completeUpdatePreparedStatement(PreparedStatement statement, DagActionStore.DagAction flowAction, - long eventTimeMillis, boolean needEventTimeCheck, boolean needLeaseAcquisitionTimeCheck, + boolean needEventTimeCheck, boolean needLeaseAcquisitionTimeCheck, Timestamp originalEventTimestamp, Timestamp originalLeaseAcquisitionTimestamp) throws SQLException { int i = 0; - // Value to update - statement.setTimestamp(++i, new Timestamp(eventTimeMillis)); // Values to check if existing row matches previous read statement.setString(++i, flowAction.getFlowGroup()); statement.setString(++i, flowAction.getFlowName()); @@ -333,11 +429,6 @@ protected void completeUpdatePreparedStatement(PreparedStatement statement, DagA if (needLeaseAcquisitionTimeCheck) { statement.setTimestamp(++i, originalLeaseAcquisitionTimestamp); } - // Values to select for return - statement.setString(++i, flowAction.getFlowGroup()); - statement.setString(++i, flowAction.getFlowName()); - statement.setString(++i, flowAction.getFlowExecutionId()); - statement.setString(++i, flowAction.getFlowActionType().toString()); } @Override @@ -375,17 +466,44 @@ public boolean recordLeaseSuccess(LeaseObtainedStatus status) } /** Abstracts recurring pattern around resource management and exception re-mapping. */ - protected T withPreparedStatement(String sql, CheckedFunction f, boolean shouldCommit) throws IOException { + protected T withPreparedStatement(String sql, CheckedFunction f, boolean shouldCommit) + throws IOException { try (Connection connection = this.dataSource.getConnection(); PreparedStatement statement = connection.prepareStatement(sql)) { T result = f.apply(statement); if (shouldCommit) { connection.commit(); } + statement.close(); return result; } catch (SQLException e) { - log.warn("Received SQL exception that can result from invalid connection. Checking if validation query is set {} Exception is {}", ((HikariDataSource) this.dataSource).getConnectionTestQuery(), e); + log.warn("Received SQL exception that can result from invalid connection. Checking if validation query is set {} " + + "Exception is {}", ((HikariDataSource) this.dataSource).getConnectionTestQuery(), e); throw new IOException(e); } } + + + /** + * DTO for arbiter's current lease state for a FlowActionEvent + */ + @Data + static class GetEventInfoResult { + private final Timestamp dbEventTimestamp; + private final Timestamp dbLeaseAcquisitionTimestamp; + private final boolean withinEpsilon; + private final int leaseValidityStatus; + private final int dbLinger; + private final Timestamp dbCurrentTimestamp; + } + + /** + DTO for result of SELECT query used to determine status of lease acquisition attempt + */ + @Data + static class SelectInfoResult { + private final long eventTimeMillis; + private final long leaseAcquisitionTimeMillis; + private final int dbLinger; + } } diff --git a/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiterTest.java b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiterTest.java new file mode 100644 index 00000000000..3ede1ce835c --- /dev/null +++ b/gobblin-runtime/src/test/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiterTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.runtime.api; + +import com.typesafe.config.Config; +import lombok.extern.slf4j.Slf4j; +import org.apache.gobblin.config.ConfigBuilder; +import org.apache.gobblin.configuration.ConfigurationKeys; +import org.apache.gobblin.metastore.testing.ITestMetastoreDatabase; +import org.apache.gobblin.metastore.testing.TestMetastoreDatabaseFactory; +import org.testng.Assert; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +@Slf4j +public class MysqlMultiActiveLeaseArbiterTest { + private static final int EPSILON = 30000; + private static final int LINGER = 80000; + private static final String USER = "testUser"; + private static final String PASSWORD = "testPassword"; + private static final String TABLE = "mysql_multi_active_lease_arbiter_store"; + private static final String flowGroup = "testFlowGroup"; + private static final String flowName = "testFlowName"; + private static final String flowExecutionId = "12345677"; + private static DagActionStore.DagAction launchDagAction = + new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.LAUNCH); + + private static final long eventTimeMillis = System.currentTimeMillis(); + private MysqlMultiActiveLeaseArbiter mysqlMultiActiveLeaseArbiter; + + // The setup functionality verifies that the initialization of the tables is done correctly and verifies any SQL + // syntax errors. + @BeforeClass + public void setUp() throws Exception { + ITestMetastoreDatabase testDb = TestMetastoreDatabaseFactory.get(); + + Config config = ConfigBuilder.create() + .addPrimitive(ConfigurationKeys.MYSQL_LEASE_ARBITER_PREFIX + "." + ConfigurationKeys.SCHEDULER_EVENT_EPSILON_MILLIS_KEY, EPSILON) + .addPrimitive(ConfigurationKeys.MYSQL_LEASE_ARBITER_PREFIX + "." + ConfigurationKeys.SCHEDULER_EVENT_LINGER_MILLIS_KEY, LINGER) + .addPrimitive(ConfigurationKeys.MYSQL_LEASE_ARBITER_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_URL_KEY, testDb.getJdbcUrl()) + .addPrimitive(ConfigurationKeys.MYSQL_LEASE_ARBITER_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_USER_KEY, USER) + .addPrimitive(ConfigurationKeys.MYSQL_LEASE_ARBITER_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, PASSWORD) + .addPrimitive(ConfigurationKeys.MYSQL_LEASE_ARBITER_PREFIX + "." + ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, TABLE) + .build(); + + this.mysqlMultiActiveLeaseArbiter = new MysqlMultiActiveLeaseArbiter(config); + } + + /* + Tests all cases of trying to acquire a lease (CASES 1-6 detailed below) for a flow action event with one + participant involved. + */ + // TODO: refactor this to break it into separate test cases as much is possible + @Test + public void testAcquireLeaseSingleParticipant() throws Exception { + // Tests CASE 1 of acquire lease for a flow action event not present in DB + MultiActiveLeaseArbiter.LeaseAttemptStatus firstLaunchStatus = + mysqlMultiActiveLeaseArbiter.tryAcquireLease(launchDagAction, eventTimeMillis); + Assert.assertTrue(firstLaunchStatus instanceof MultiActiveLeaseArbiter.LeaseObtainedStatus); + MultiActiveLeaseArbiter.LeaseObtainedStatus firstObtainedStatus = + (MultiActiveLeaseArbiter.LeaseObtainedStatus) firstLaunchStatus; + Assert.assertTrue(firstObtainedStatus.getEventTimestamp() <= + firstObtainedStatus.getLeaseAcquisitionTimestamp()); + Assert.assertTrue(firstObtainedStatus.getFlowAction().equals( + new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.LAUNCH))); + + // Verify that different DagAction types for the same flow can have leases at the same time + DagActionStore.DagAction killDagAction = new + DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.KILL); + MultiActiveLeaseArbiter.LeaseAttemptStatus killStatus = + mysqlMultiActiveLeaseArbiter.tryAcquireLease(killDagAction, eventTimeMillis); + Assert.assertTrue(killStatus instanceof MultiActiveLeaseArbiter.LeaseObtainedStatus); + MultiActiveLeaseArbiter.LeaseObtainedStatus killObtainedStatus = + (MultiActiveLeaseArbiter.LeaseObtainedStatus) killStatus; + Assert.assertTrue( + killObtainedStatus.getLeaseAcquisitionTimestamp() >= killObtainedStatus.getEventTimestamp()); + + // Tests CASE 2 of acquire lease for a flow action event that already has a valid lease for the same event in db + // Very little time should have passed if this test directly follows the one above so this call will be considered + // the same as the previous event + MultiActiveLeaseArbiter.LeaseAttemptStatus secondLaunchStatus = + mysqlMultiActiveLeaseArbiter.tryAcquireLease(launchDagAction, eventTimeMillis); + Assert.assertTrue(secondLaunchStatus instanceof MultiActiveLeaseArbiter.LeasedToAnotherStatus); + MultiActiveLeaseArbiter.LeasedToAnotherStatus secondLeasedToAnotherStatus = + (MultiActiveLeaseArbiter.LeasedToAnotherStatus) secondLaunchStatus; + Assert.assertTrue(secondLeasedToAnotherStatus.getEventTimeMillis() == firstObtainedStatus.getEventTimestamp()); + Assert.assertTrue(secondLeasedToAnotherStatus.getMinimumLingerDurationMillis() >= LINGER); + + // Tests CASE 3 of trying to acquire a lease for a distinct flow action event, while the previous event's lease is + // valid + // Allow enough time to pass for this trigger to be considered distinct, but not enough time so the lease expires + Thread.sleep(EPSILON * 3/2); + MultiActiveLeaseArbiter.LeaseAttemptStatus thirdLaunchStatus = + mysqlMultiActiveLeaseArbiter.tryAcquireLease(launchDagAction, eventTimeMillis); + Assert.assertTrue(thirdLaunchStatus instanceof MultiActiveLeaseArbiter.LeasedToAnotherStatus); + MultiActiveLeaseArbiter.LeasedToAnotherStatus thirdLeasedToAnotherStatus = + (MultiActiveLeaseArbiter.LeasedToAnotherStatus) thirdLaunchStatus; + Assert.assertTrue(thirdLeasedToAnotherStatus.getEventTimeMillis() > firstObtainedStatus.getEventTimestamp()); + Assert.assertTrue(thirdLeasedToAnotherStatus.getMinimumLingerDurationMillis() < LINGER); + + // Tests CASE 4 of lease out of date + Thread.sleep(LINGER); + MultiActiveLeaseArbiter.LeaseAttemptStatus fourthLaunchStatus = + mysqlMultiActiveLeaseArbiter.tryAcquireLease(launchDagAction, eventTimeMillis); + Assert.assertTrue(fourthLaunchStatus instanceof MultiActiveLeaseArbiter.LeaseObtainedStatus); + MultiActiveLeaseArbiter.LeaseObtainedStatus fourthObtainedStatus = + (MultiActiveLeaseArbiter.LeaseObtainedStatus) fourthLaunchStatus; + Assert.assertTrue(fourthObtainedStatus.getEventTimestamp() > eventTimeMillis + LINGER); + Assert.assertTrue(fourthObtainedStatus.getEventTimestamp() + <= fourthObtainedStatus.getLeaseAcquisitionTimestamp()); + + // Tests CASE 5 of no longer leasing the same event in DB + // done immediately after previous lease obtainment so should be marked as the same event + Assert.assertTrue(mysqlMultiActiveLeaseArbiter.recordLeaseSuccess(fourthObtainedStatus)); + Assert.assertTrue(System.currentTimeMillis() - fourthObtainedStatus.getEventTimestamp() < EPSILON); + MultiActiveLeaseArbiter.LeaseAttemptStatus fifthLaunchStatus = + mysqlMultiActiveLeaseArbiter.tryAcquireLease(launchDagAction, eventTimeMillis); + Assert.assertTrue(fifthLaunchStatus instanceof MultiActiveLeaseArbiter.NoLongerLeasingStatus); + + // Tests CASE 6 of no longer leasing a distinct event in DB + // Wait so this event is considered distinct and a new lease will be acquired + Thread.sleep(EPSILON * 3/2); + MultiActiveLeaseArbiter.LeaseAttemptStatus sixthLaunchStatus = + mysqlMultiActiveLeaseArbiter.tryAcquireLease(launchDagAction, eventTimeMillis); + Assert.assertTrue(sixthLaunchStatus instanceof MultiActiveLeaseArbiter.LeaseObtainedStatus); + MultiActiveLeaseArbiter.LeaseObtainedStatus sixthObtainedStatus = + (MultiActiveLeaseArbiter.LeaseObtainedStatus) sixthLaunchStatus; + Assert.assertTrue(sixthObtainedStatus.getEventTimestamp() + <= sixthObtainedStatus.getLeaseAcquisitionTimestamp()); + } +} diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java index c0f140a9fe4..1423e2d8c14 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceGuiceModule.java @@ -168,7 +168,7 @@ public void configure(Binder binder) { OptionalBinder.newOptionalBinder(binder, MultiActiveLeaseArbiter.class); OptionalBinder.newOptionalBinder(binder, FlowTriggerHandler.class); if (serviceConfig.isMultiActiveSchedulerEnabled()) { - binder.bind(MysqlMultiActiveLeaseArbiter.class); + binder.bind(MultiActiveLeaseArbiter.class).to(MysqlMultiActiveLeaseArbiter.class); binder.bind(FlowTriggerHandler.class); } diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java index 42ab0af96fa..ec63276c4c5 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java @@ -17,6 +17,7 @@ package org.apache.gobblin.service.modules.orchestration; +import com.google.common.base.Optional; import java.io.IOException; import java.time.LocalDateTime; import java.time.ZoneId; @@ -62,15 +63,15 @@ public class FlowTriggerHandler { private final int schedulerMaxBackoffMillis; private static Random random = new Random(); - protected MultiActiveLeaseArbiter multiActiveLeaseArbiter; + protected Optional multiActiveLeaseArbiter; protected SchedulerService schedulerService; - protected DagActionStore dagActionStore; + protected Optional dagActionStore; private MetricContext metricContext; private ContextAwareMeter numFlowsSubmitted; @Inject - public FlowTriggerHandler(Config config, MultiActiveLeaseArbiter leaseDeterminationStore, - SchedulerService schedulerService, DagActionStore dagActionStore) { + public FlowTriggerHandler(Config config, Optional leaseDeterminationStore, + SchedulerService schedulerService, Optional dagActionStore) { this.schedulerMaxBackoffMillis = ConfigUtils.getInt(config, ConfigurationKeys.SCHEDULER_MAX_BACKOFF_MILLIS_KEY, ConfigurationKeys.DEFAULT_SCHEDULER_MAX_BACKOFF_MILLIS); this.multiActiveLeaseArbiter = leaseDeterminationStore; @@ -91,41 +92,51 @@ public FlowTriggerHandler(Config config, MultiActiveLeaseArbiter leaseDeterminat */ public void handleTriggerEvent(Properties jobProps, DagActionStore.DagAction flowAction, long eventTimeMillis) throws IOException { - MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = - multiActiveLeaseArbiter.tryAcquireLease(flowAction, eventTimeMillis); - // TODO: add a log event or metric for each of these cases - if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeaseObtainedStatus) { - MultiActiveLeaseArbiter.LeaseObtainedStatus leaseObtainedStatus = (MultiActiveLeaseArbiter.LeaseObtainedStatus) leaseAttemptStatus; - if (persistFlowAction(leaseObtainedStatus)) { + if (multiActiveLeaseArbiter.isPresent()) { + MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = multiActiveLeaseArbiter.get().tryAcquireLease(flowAction, eventTimeMillis); + // TODO: add a log event or metric for each of these cases + if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeaseObtainedStatus) { + MultiActiveLeaseArbiter.LeaseObtainedStatus leaseObtainedStatus = (MultiActiveLeaseArbiter.LeaseObtainedStatus) leaseAttemptStatus; + if (persistFlowAction(leaseObtainedStatus)) { + return; + } + // If persisting the flow action failed, then we set another trigger for this event to occur immediately to + // re-attempt handling the event + scheduleReminderForEvent(jobProps, + new MultiActiveLeaseArbiter.LeasedToAnotherStatus(flowAction, leaseObtainedStatus.getEventTimestamp(), 0L), + eventTimeMillis); + return; + } else if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeasedToAnotherStatus) { + scheduleReminderForEvent(jobProps, (MultiActiveLeaseArbiter.LeasedToAnotherStatus) leaseAttemptStatus, + eventTimeMillis); + return; + } else if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.NoLongerLeasingStatus) { return; } - // If persisting the flow action failed, then we set another trigger for this event to occur immediately to - // re-attempt handling the event - scheduleReminderForEvent(jobProps, new MultiActiveLeaseArbiter.LeasedToAnotherStatus(flowAction, - leaseObtainedStatus.getEventTimestamp(), 0L), eventTimeMillis); - return; - } else if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeasedToAnotherStatus) { - scheduleReminderForEvent(jobProps, (MultiActiveLeaseArbiter.LeasedToAnotherStatus) leaseAttemptStatus, - eventTimeMillis); - return; - } else if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.NoLongerLeasingStatus) { - return; + throw new RuntimeException(String.format("Received type of leaseAttemptStatus: %s not handled by this method", + leaseAttemptStatus.getClass().getName())); + } else { + throw new RuntimeException(String.format("Multi-active scheduler is not enabled so trigger event should not be " + + "handled with this method.")); } - throw new RuntimeException(String.format("Received type of leaseAttemptStatus: %s not handled by this method", - leaseAttemptStatus.getClass().getName())); } // Called after obtaining a lease to persist the flow action to {@link DagActionStore} and mark the lease as done private boolean persistFlowAction(MultiActiveLeaseArbiter.LeaseObtainedStatus leaseStatus) { - try { - DagActionStore.DagAction flowAction = leaseStatus.getFlowAction(); - this.dagActionStore.addDagAction(flowAction.getFlowGroup(), flowAction.getFlowName(), - flowAction.getFlowExecutionId(), flowAction.getFlowActionType()); - // If the flow action has been persisted to the {@link DagActionStore} we can close the lease - this.numFlowsSubmitted.mark(); - return this.multiActiveLeaseArbiter.recordLeaseSuccess(leaseStatus); - } catch (IOException e) { - throw new RuntimeException(e); + if (this.dagActionStore.isPresent() && this.multiActiveLeaseArbiter.isPresent()) { + try { + DagActionStore.DagAction flowAction = leaseStatus.getFlowAction(); + this.dagActionStore.get().addDagAction(flowAction.getFlowGroup(), flowAction.getFlowName(), flowAction.getFlowExecutionId(), flowAction.getFlowActionType()); + // If the flow action has been persisted to the {@link DagActionStore} we can close the lease + this.numFlowsSubmitted.mark(); + return this.multiActiveLeaseArbiter.get().recordLeaseSuccess(leaseStatus); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + throw new RuntimeException("DagActionStore is " + (this.dagActionStore.isPresent() ? "" : "NOT") + " present. " + + "Multi-Active scheduler is " + (this.multiActiveLeaseArbiter.isPresent() ? "" : "NOT") + " present. Both " + + "should be enabled if this method is called."); } } From c4a466b414c43a59ddead75de25ae87e6732b716 Mon Sep 17 00:00:00 2001 From: Matthew Ho Date: Tue, 18 Jul 2023 15:08:07 -0700 Subject: [PATCH 22/30] [GOBBLIN-1853] Reduce # of Hive calls during schema related updates (#3716) --- .../hive/writer/HiveMetadataWriter.java | 63 +++++++++++-------- .../iceberg/writer/IcebergMetadataWriter.java | 15 +++-- .../writer/HiveMetadataWriterTest.java | 6 ++ 3 files changed, 55 insertions(+), 29 deletions(-) diff --git a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/writer/HiveMetadataWriter.java b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/writer/HiveMetadataWriter.java index 02c0c5895c8..138ac3d9417 100644 --- a/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/writer/HiveMetadataWriter.java +++ b/gobblin-hive-registration/src/main/java/org/apache/gobblin/hive/writer/HiveMetadataWriter.java @@ -17,16 +17,6 @@ package org.apache.gobblin.hive.writer; -import com.codahale.metrics.Timer; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; -import com.google.common.base.Optional; -import com.google.common.base.Throwables; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.collect.Lists; -import com.google.common.io.Closer; -import com.google.common.util.concurrent.ListenableFuture; import java.io.IOException; import java.util.Collection; import java.util.Collections; @@ -38,20 +28,37 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; -import javax.annotation.Nullable; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericRecord; +import org.apache.avro.specific.SpecificData; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils; + +import com.codahale.metrics.Timer; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Joiner; +import com.google.common.base.Optional; +import com.google.common.base.Throwables; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.Lists; +import com.google.common.io.Closer; +import com.google.common.util.concurrent.ListenableFuture; + +import javax.annotation.Nullable; import lombok.AccessLevel; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.avro.Schema; -import org.apache.avro.generic.GenericRecord; -import org.apache.avro.specific.SpecificData; + import org.apache.gobblin.configuration.State; import org.apache.gobblin.data.management.copy.hive.WhitelistBlacklist; import org.apache.gobblin.hive.HiveRegister; import org.apache.gobblin.hive.HiveRegistrationUnit; import org.apache.gobblin.hive.HiveTable; +import org.apache.gobblin.hive.avro.HiveAvroSerDeManager; import org.apache.gobblin.hive.metastore.HiveMetaStoreBasedRegister; import org.apache.gobblin.hive.metastore.HiveMetaStoreUtils; import org.apache.gobblin.hive.spec.HiveSpec; @@ -68,10 +75,6 @@ import org.apache.gobblin.util.AvroUtils; import org.apache.gobblin.util.ClustersNames; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; -import org.apache.hadoop.hive.serde2.avro.AvroSerdeUtils; - /** * This writer is used to register the hiveSpec into hive metaStore @@ -313,9 +316,15 @@ protected static boolean updateLatestSchemaMapWithExistingSchema(String dbName, return false; } - HiveTable existingTable = hiveRegister.getTable(dbName, tableName).get(); - latestSchemaMap.put(tableKey, - existingTable.getSerDeProps().getProp(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName())); + HiveTable table = hiveRegister.getTable(dbName, tableName).get(); + String latestSchema = table.getSerDeProps().getProp(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName()); + if (latestSchema == null) { + throw new IllegalStateException(String.format("The %s in the table %s.%s is null. This implies the DB is " + + "misconfigured and was not correctly created through Gobblin, since all Gobblin managed tables should " + + "have %s", HiveAvroSerDeManager.SCHEMA_LITERAL, dbName, tableName, HiveAvroSerDeManager.SCHEMA_LITERAL)); + } + + latestSchemaMap.put(tableKey, latestSchema); return true; } @@ -445,10 +454,14 @@ private void schemaUpdateHelper(GobblinMetadataChangeEvent gmce, HiveSpec spec, return; } //Force to set the schema even there is no schema literal defined in the spec - if (latestSchemaMap.containsKey(tableKey)) { - spec.getTable().getSerDeProps() - .setProp(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName(), latestSchemaMap.get(tableKey)); - HiveMetaStoreUtils.updateColumnsInfoIfNeeded(spec); + String latestSchema = latestSchemaMap.get(tableKey); + if (latestSchema != null) { + String tableSchema = spec.getTable().getSerDeProps().getProp(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName()); + if (tableSchema == null || !tableSchema.equals(latestSchema)) { + spec.getTable().getSerDeProps() + .setProp(AvroSerdeUtils.AvroTableProperties.SCHEMA_LITERAL.getPropName(), latestSchemaMap.get(tableKey)); + HiveMetaStoreUtils.updateColumnsInfoIfNeeded(spec); + } } } diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java index 57a28f77851..b6c9e15dc11 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java @@ -48,10 +48,7 @@ import org.apache.avro.generic.GenericRecord; import org.apache.avro.specific.SpecificData; - import org.apache.commons.lang3.tuple.Pair; -import org.apache.gobblin.hive.writer.MetadataWriterKeys; -import org.apache.gobblin.source.extractor.extract.LongWatermark; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -112,6 +109,7 @@ import org.apache.gobblin.hive.metastore.HiveMetaStoreUtils; import org.apache.gobblin.hive.spec.HiveSpec; import org.apache.gobblin.hive.writer.MetadataWriter; +import org.apache.gobblin.hive.writer.MetadataWriterKeys; import org.apache.gobblin.iceberg.Utils.IcebergUtils; import org.apache.gobblin.metadata.GobblinMetadataChangeEvent; import org.apache.gobblin.metadata.OperationType; @@ -122,6 +120,7 @@ import org.apache.gobblin.metrics.event.GobblinEventBuilder; import org.apache.gobblin.metrics.kafka.KafkaSchemaRegistry; import org.apache.gobblin.metrics.kafka.SchemaRegistryException; +import org.apache.gobblin.source.extractor.extract.LongWatermark; import org.apache.gobblin.stream.RecordEnvelope; import org.apache.gobblin.time.TimeIterator; import org.apache.gobblin.util.AvroUtils; @@ -129,6 +128,7 @@ import org.apache.gobblin.util.HadoopUtils; import org.apache.gobblin.util.ParallelRunner; import org.apache.gobblin.util.WriterUtils; + import static org.apache.gobblin.iceberg.writer.IcebergMetadataWriterConfigKeys.*; /** @@ -493,13 +493,20 @@ private void computeCandidateSchema(GobblinMetadataChangeEvent gmce, TableIdenti * @return table with updated schema and partition spec */ private Table addPartitionToIcebergTable(Table table, String fieldName, String type) { + boolean isTableUpdated = false; if(!table.schema().columns().stream().anyMatch(x -> x.name().equalsIgnoreCase(fieldName))) { table.updateSchema().addColumn(fieldName, Types.fromPrimitiveString(type)).commit(); + isTableUpdated = true; } if(!table.spec().fields().stream().anyMatch(x -> x.name().equalsIgnoreCase(fieldName))) { table.updateSpec().addField(fieldName).commit(); + isTableUpdated = true; + } + + if (isTableUpdated) { + table.refresh(); } - table.refresh(); + return table; } diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java index f50bc81aa9a..aa6c73d5295 100644 --- a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java +++ b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java @@ -388,6 +388,12 @@ public void testUpdateLatestSchemaWithExistingSchema() throws IOException { )); Assert.assertTrue(updateLatestSchema.apply(tableNameAllowed)); Mockito.verify(hiveRegister, Mockito.times(2)).getTable(eq(dbName), eq(tableNameAllowed)); + + HiveTable tableThatHasNoSchemaLiteral = Mockito.mock(HiveTable.class); + String nameOfTableThatHasNoSchemaLiteral = "improperlyConfiguredTable"; + Mockito.when(hiveRegister.getTable(eq(dbName), eq(nameOfTableThatHasNoSchemaLiteral))).thenReturn(Optional.of(tableThatHasNoSchemaLiteral)); + Mockito.when(tableThatHasNoSchemaLiteral.getSerDeProps()).thenReturn(new State()); + Assert.assertThrows(IllegalStateException.class, () -> updateLatestSchema.apply(nameOfTableThatHasNoSchemaLiteral)); } private String writeRecord(File file) throws IOException { From a5274cdfff73702d5b9a4afed7ce18fbe3fc93de Mon Sep 17 00:00:00 2001 From: Ratandeep Ratti Date: Tue, 18 Jul 2023 17:01:47 -0700 Subject: [PATCH 23/30] Remove unused ORC writer code (#3710) --- .../gobblin/writer/GobblinBaseOrcWriter.java | 157 +----------------- .../gobblin/writer/GobblinOrcWriterTest.java | 71 -------- 2 files changed, 1 insertion(+), 227 deletions(-) diff --git a/gobblin-modules/gobblin-orc/src/main/java/org/apache/gobblin/writer/GobblinBaseOrcWriter.java b/gobblin-modules/gobblin-orc/src/main/java/org/apache/gobblin/writer/GobblinBaseOrcWriter.java index b05c3af2b24..528ceaaf660 100644 --- a/gobblin-modules/gobblin-orc/src/main/java/org/apache/gobblin/writer/GobblinBaseOrcWriter.java +++ b/gobblin-modules/gobblin-orc/src/main/java/org/apache/gobblin/writer/GobblinBaseOrcWriter.java @@ -27,15 +27,6 @@ import org.apache.orc.OrcFile; import org.apache.orc.TypeDescription; import org.apache.orc.Writer; -import org.apache.orc.storage.ql.exec.vector.BytesColumnVector; -import org.apache.orc.storage.ql.exec.vector.ColumnVector; -import org.apache.orc.storage.ql.exec.vector.DecimalColumnVector; -import org.apache.orc.storage.ql.exec.vector.DoubleColumnVector; -import org.apache.orc.storage.ql.exec.vector.ListColumnVector; -import org.apache.orc.storage.ql.exec.vector.LongColumnVector; -import org.apache.orc.storage.ql.exec.vector.MapColumnVector; -import org.apache.orc.storage.ql.exec.vector.StructColumnVector; -import org.apache.orc.storage.ql.exec.vector.UnionColumnVector; import org.apache.orc.storage.ql.exec.vector.VectorizedRowBatch; import com.google.common.annotations.VisibleForTesting; @@ -45,8 +36,6 @@ import org.apache.gobblin.configuration.State; import org.apache.gobblin.state.ConstructState; -import static org.apache.gobblin.configuration.ConfigurationKeys.AVG_RECORD_SIZE; - /** * A wrapper for ORC-core writer without dependency on Hive SerDe library. */ @@ -56,39 +45,6 @@ public abstract class GobblinBaseOrcWriter extends FsDataWriter { public static final String ORC_WRITER_BATCH_SIZE = ORC_WRITER_PREFIX + "batchSize"; public static final int DEFAULT_ORC_WRITER_BATCH_SIZE = 1000; - private static final String CONTAINER_MEMORY_MBS = ORC_WRITER_PREFIX + "jvm.memory.mbs"; - private static final int DEFAULT_CONTAINER_MEMORY_MBS = 4096; - - private static final String CONTAINER_JVM_MEMORY_XMX_RATIO_KEY = ORC_WRITER_PREFIX + "container.jvmMemoryXmxRatio"; - private static final double DEFAULT_CONTAINER_JVM_MEMORY_XMX_RATIO_KEY = 1.0; - - static final String CONTAINER_JVM_MEMORY_OVERHEAD_MBS = ORC_WRITER_PREFIX + "container.jvmMemoryOverheadMbs"; - private static final int DEFAULT_CONTAINER_JVM_MEMORY_OVERHEAD_MBS = 0; - - @VisibleForTesting - static final String ORC_WRITER_AUTO_TUNE_ENABLED = ORC_WRITER_PREFIX + "autoTuneEnabled"; - private static final boolean ORC_WRITER_AUTO_TUNE_DEFAULT = false; - private static final long EXEMPLIFIED_RECORD_SIZE_IN_BYTES = 1024; - - /** - * This value gives an estimation on how many writers are buffering records at the same time in a container. - * Since time-based partition scheme is a commonly used practice, plus the chances for late-arrival data, - * usually there would be 2-3 writers running during the hourly boundary. 3 is chosen here for being conservative. - */ - private static final int ESTIMATED_PARALLELISM_WRITERS = 3; - - // The serialized record size passed from AVG_RECORD_SIZE is smaller than the actual in-memory representation - // of a record. This is just the number represents how many times that the actual buffer storing record is larger - // than the serialized size passed down from upstream constructs. - @VisibleForTesting - static final String RECORD_SIZE_SCALE_FACTOR = "recordSize.scaleFactor"; - static final int DEFAULT_RECORD_SIZE_SCALE_FACTOR = 6; - - /** - * Check comment of {@link #deepCleanRowBatch} for the usage of this configuration. - */ - private static final String ORC_WRITER_DEEP_CLEAN_EVERY_BATCH = ORC_WRITER_PREFIX + "deepCleanBatch"; - private final OrcValueWriter valueWriter; @VisibleForTesting VectorizedRowBatch rowBatch; @@ -99,61 +55,14 @@ public abstract class GobblinBaseOrcWriter extends FsDataWriter { // the close method may be invoked multiple times, but the underlying writer only supports close being called once private volatile boolean closed = false; - private final boolean deepCleanBatch; private final int batchSize; protected final S inputSchema; - /** - * There are a couple of parameters in ORC writer that requires manual tuning based on record size given that executor - * for running these ORC writers has limited heap space. This helper function wrap them and has side effect for the - * argument {@param properties}. - * - * Assumption for current implementation: - * The extractor or source class should set {@link org.apache.gobblin.configuration.ConfigurationKeys#AVG_RECORD_SIZE} - */ - protected void autoTunedOrcWriterParams(State properties) { - double writerRatio = properties.getPropAsDouble(OrcConf.MEMORY_POOL.name(), (double) OrcConf.MEMORY_POOL.getDefaultValue()); - long availableHeapPerWriter = Math.round(availableHeapSize(properties) * writerRatio / ESTIMATED_PARALLELISM_WRITERS); - - // Upstream constructs will need to set this value properly - long estimatedRecordSize = getEstimatedRecordSize(properties); - long rowsBetweenCheck = availableHeapPerWriter * 1024 / estimatedRecordSize; - properties.setProp(OrcConf.ROWS_BETWEEN_CHECKS.name(), - Math.min(rowsBetweenCheck, (int) OrcConf.ROWS_BETWEEN_CHECKS.getDefaultValue())); - // Row batch size should be smaller than row_between_check, 4 is just a magic number picked here. - long batchSize = Math.min(rowsBetweenCheck / 4, DEFAULT_ORC_WRITER_BATCH_SIZE); - properties.setProp(ORC_WRITER_BATCH_SIZE, batchSize); - log.info("Tuned the parameter " + OrcConf.ROWS_BETWEEN_CHECKS.name() + " to be:" + rowsBetweenCheck + "," - + ORC_WRITER_BATCH_SIZE + " to be:" + batchSize); - } - - /** - * Calculate the heap size in MB available for ORC writers. - */ - protected long availableHeapSize(State properties) { - // Calculate the recommended size as the threshold for memory check - long physicalMem = Math.round(properties.getPropAsLong(CONTAINER_MEMORY_MBS, DEFAULT_CONTAINER_MEMORY_MBS) - * properties.getPropAsDouble(CONTAINER_JVM_MEMORY_XMX_RATIO_KEY, DEFAULT_CONTAINER_JVM_MEMORY_XMX_RATIO_KEY)); - long nonHeap = properties.getPropAsLong(CONTAINER_JVM_MEMORY_OVERHEAD_MBS, DEFAULT_CONTAINER_JVM_MEMORY_OVERHEAD_MBS); - return physicalMem - nonHeap; - } - - /** - * Calculate the estimated record size in bytes. - */ - protected long getEstimatedRecordSize(State properties) { - long estimatedRecordSizeScale = properties.getPropAsInt(RECORD_SIZE_SCALE_FACTOR, DEFAULT_RECORD_SIZE_SCALE_FACTOR); - return (properties.contains(AVG_RECORD_SIZE) ? properties.getPropAsLong(AVG_RECORD_SIZE) - : EXEMPLIFIED_RECORD_SIZE_IN_BYTES) * estimatedRecordSizeScale; - } public GobblinBaseOrcWriter(FsDataWriterBuilder builder, State properties) throws IOException { super(builder, properties); - if (properties.getPropAsBoolean(ORC_WRITER_AUTO_TUNE_ENABLED, ORC_WRITER_AUTO_TUNE_DEFAULT)) { - autoTunedOrcWriterParams(properties); - } // Create value-writer which is essentially a record-by-record-converter with buffering in batch. this.inputSchema = builder.getSchema(); @@ -163,8 +72,6 @@ public GobblinBaseOrcWriter(FsDataWriterBuilder builder, State properties) this.rowBatchPool = RowBatchPool.instance(properties); this.enableRowBatchPool = properties.getPropAsBoolean(RowBatchPool.ENABLE_ROW_BATCH_POOL, false); this.rowBatch = enableRowBatchPool ? rowBatchPool.getRowBatch(typeDescription, batchSize) : typeDescription.createRowBatch(batchSize); - this.deepCleanBatch = properties.getPropAsBoolean(ORC_WRITER_DEEP_CLEAN_EVERY_BATCH, false); - log.info("Created ORC writer, batch size: {}, {}: {}", batchSize, OrcConf.ROWS_BETWEEN_CHECKS.getAttribute(), properties.getProp( @@ -192,7 +99,7 @@ public GobblinBaseOrcWriter(FsDataWriterBuilder builder, State properties) protected abstract TypeDescription getOrcSchema(); /** - * Get an {@OrcValueWriter} for the specified schema and configuration. + * Get an {@link OrcValueWriter} for the specified schema and configuration. */ protected abstract OrcValueWriter getOrcValueWriter(TypeDescription typeDescription, S inputSchema, State state); @@ -231,9 +138,6 @@ public void flush() if (rowBatch.size > 0) { orcFileWriter.addRowBatch(rowBatch); rowBatch.reset(); - if (deepCleanBatch) { - deepCleanRowBatch(rowBatch); - } } } @@ -285,65 +189,6 @@ public void write(D record) if (rowBatch.size == this.batchSize) { orcFileWriter.addRowBatch(rowBatch); rowBatch.reset(); - if (deepCleanBatch) { - log.info("A reset of rowBatch is triggered - releasing holding memory for large object"); - deepCleanRowBatch(rowBatch); - } - } - } - - /** - * The reset call of {@link VectorizedRowBatch} doesn't release the memory occupied by each {@link ColumnVector}'s child, - * which is usually an array of objects, while it only set those value to null. - * This method ensure the reference to the child array for {@link ColumnVector} are released and gives a hint of GC, - * so that each reset could release the memory pre-allocated by {@link ColumnVector#ensureSize(int, boolean)} method. - * - * This feature is configurable and should only be turned on if a dataset is: - * 1. Has large per-record size. - * 2. Has {@link org.apache.hadoop.hive.ql.exec.vector.MultiValuedColumnVector} as part of schema, - * like array, map and all nested structures containing these. - */ - @VisibleForTesting - void deepCleanRowBatch(VectorizedRowBatch rowBatch) { - for(int i = 0; i < rowBatch.cols.length; ++i) { - ColumnVector cv = rowBatch.cols[i]; - if (cv != null) { - removeRefOfColumnVectorChild(cv); - } - } - } - - /** - * Set the child field of {@link ColumnVector} to null, assuming input {@link ColumnVector} is nonNull. - */ - private void removeRefOfColumnVectorChild(ColumnVector cv) { - if (cv instanceof StructColumnVector) { - StructColumnVector structCv = (StructColumnVector) cv; - for (ColumnVector childCv: structCv.fields) { - removeRefOfColumnVectorChild(childCv); - } - } else if (cv instanceof ListColumnVector) { - ListColumnVector listCv = (ListColumnVector) cv; - removeRefOfColumnVectorChild(listCv.child); - } else if (cv instanceof MapColumnVector) { - MapColumnVector mapCv = (MapColumnVector) cv; - removeRefOfColumnVectorChild(mapCv.keys); - removeRefOfColumnVectorChild(mapCv.values); - } else if (cv instanceof UnionColumnVector) { - UnionColumnVector unionCv = (UnionColumnVector) cv; - for (ColumnVector unionChildCv : unionCv.fields) { - removeRefOfColumnVectorChild(unionChildCv); - } - } else if (cv instanceof LongColumnVector) { - ((LongColumnVector) cv).vector = null; - } else if (cv instanceof DoubleColumnVector) { - ((DoubleColumnVector) cv).vector = null; - } else if (cv instanceof BytesColumnVector) { - ((BytesColumnVector) cv).vector = null; - ((BytesColumnVector) cv).start = null; - ((BytesColumnVector) cv).length = null; - } else if (cv instanceof DecimalColumnVector) { - ((DecimalColumnVector) cv).vector = null; } } } diff --git a/gobblin-modules/gobblin-orc/src/test/java/org/apache/gobblin/writer/GobblinOrcWriterTest.java b/gobblin-modules/gobblin-orc/src/test/java/org/apache/gobblin/writer/GobblinOrcWriterTest.java index e6b5f3d0312..0b0912cf7a7 100644 --- a/gobblin-modules/gobblin-orc/src/test/java/org/apache/gobblin/writer/GobblinOrcWriterTest.java +++ b/gobblin-modules/gobblin-orc/src/test/java/org/apache/gobblin/writer/GobblinOrcWriterTest.java @@ -31,9 +31,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.Writable; -import org.apache.orc.OrcConf; -import org.apache.orc.storage.ql.exec.vector.BytesColumnVector; -import org.apache.orc.storage.ql.exec.vector.ListColumnVector; import org.mockito.Mockito; import org.testng.Assert; import org.testng.annotations.Test; @@ -45,12 +42,7 @@ import org.apache.gobblin.configuration.State; import org.apache.gobblin.source.workunit.WorkUnit; -import static org.apache.gobblin.configuration.ConfigurationKeys.AVG_RECORD_SIZE; import static org.apache.gobblin.writer.GenericRecordToOrcValueWriterTest.deserializeOrcRecords; -import static org.apache.gobblin.writer.GobblinOrcWriter.CONTAINER_JVM_MEMORY_OVERHEAD_MBS; -import static org.apache.gobblin.writer.GobblinOrcWriter.DEFAULT_RECORD_SIZE_SCALE_FACTOR; -import static org.apache.gobblin.writer.GobblinOrcWriter.ORC_WRITER_AUTO_TUNE_ENABLED; -import static org.apache.gobblin.writer.GobblinOrcWriter.RECORD_SIZE_SCALE_FACTOR; import static org.mockito.Mockito.*; @@ -81,69 +73,6 @@ public static final List deserializeAvroRecords(Class clazz, Sche return records; } - @Test - public void testAutoTuned() throws Exception { - Closer closer = Closer.create(); - Schema schema = - new Schema.Parser().parse(this.getClass().getClassLoader().getResourceAsStream("orc_writer_test/schema.avsc")); - - FsDataWriterBuilder mockBuilder = - (FsDataWriterBuilder) Mockito.mock(FsDataWriterBuilder.class); - when(mockBuilder.getSchema()).thenReturn(schema); - State properties = new WorkUnit(); - String stagingDir = Files.createTempDir().getAbsolutePath(); - String outputDir = Files.createTempDir().getAbsolutePath(); - properties.setProp(ConfigurationKeys.WRITER_STAGING_DIR, stagingDir); - properties.setProp(ConfigurationKeys.WRITER_FILE_PATH, "simple"); - properties.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, outputDir); - when(mockBuilder.getFileName(properties)).thenReturn("file"); - - properties.setProp(ORC_WRITER_AUTO_TUNE_ENABLED, true); - properties.setProp(CONTAINER_JVM_MEMORY_OVERHEAD_MBS, 2048); - closer.register(new GobblinOrcWriter(mockBuilder, properties)); - // Verify the side effect within the properties object. - Assert.assertEquals(properties.getPropAsInt(OrcConf.ROWS_BETWEEN_CHECKS.name()), - Math.round((4096 - 2048) * 0.5 * 1024 / 3) / (1024 * properties.getPropAsInt(RECORD_SIZE_SCALE_FACTOR, DEFAULT_RECORD_SIZE_SCALE_FACTOR))); - - // Will get to 5000 - properties.setProp(AVG_RECORD_SIZE, 10); - closer.register(new GobblinOrcWriter(mockBuilder, properties)); - Assert.assertEquals(properties.getPropAsInt(OrcConf.ROWS_BETWEEN_CHECKS.name()), 5000); - - closer.close(); - } - - @Test - public void testRowBatchDeepClean() throws Exception { - Schema schema = new Schema.Parser().parse( - this.getClass().getClassLoader().getResourceAsStream("orc_writer_list_test/schema.avsc")); - List recordList = deserializeAvroRecords(this.getClass(), schema, "orc_writer_list_test/data.json"); - // Mock WriterBuilder, bunch of mocking behaviors to work-around precondition checks in writer builder - FsDataWriterBuilder mockBuilder = - (FsDataWriterBuilder) Mockito.mock(FsDataWriterBuilder.class); - when(mockBuilder.getSchema()).thenReturn(schema); - State dummyState = new WorkUnit(); - String stagingDir = Files.createTempDir().getAbsolutePath(); - String outputDir = Files.createTempDir().getAbsolutePath(); - dummyState.setProp(ConfigurationKeys.WRITER_STAGING_DIR, stagingDir); - dummyState.setProp(ConfigurationKeys.WRITER_FILE_PATH, "simple"); - dummyState.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, outputDir); - dummyState.setProp("orcWriter.deepCleanBatch", "true"); - when(mockBuilder.getFileName(dummyState)).thenReturn("file"); - - Closer closer = Closer.create(); - - GobblinOrcWriter orcWriter = closer.register(new GobblinOrcWriter(mockBuilder, dummyState)); - for (GenericRecord genericRecord : recordList) { - orcWriter.write(genericRecord); - } - // Manual trigger flush - orcWriter.flush(); - - Assert.assertNull(((BytesColumnVector) ((ListColumnVector) orcWriter.rowBatch.cols[0]).child).vector); - Assert.assertNull(((BytesColumnVector) orcWriter.rowBatch.cols[1]).vector); - } - /** * A basic unit for trivial writer correctness. * TODO: A detailed test suite of ORC-writer for different sorts of schema: From 4a9dd53b885f0166dccac5026e4e2c632c98a419 Mon Sep 17 00:00:00 2001 From: Matthew Ho Date: Tue, 18 Jul 2023 17:29:55 -0700 Subject: [PATCH 24/30] [GOBBLIN-1855] Metadata writer tests do not work in isolation after upgrading to Iceberg 1.2.0 (#3718) --- .../DatasetHiveSchemaContainsNonOptionalUnionTest.java | 6 +++++- .../gobblin/iceberg/writer/HiveMetadataWriterTest.java | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnionTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnionTest.java index 14041cdecb3..5436b5e6d86 100644 --- a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnionTest.java +++ b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/predicates/DatasetHiveSchemaContainsNonOptionalUnionTest.java @@ -21,6 +21,7 @@ import java.util.Collections; import org.apache.commons.io.FileUtils; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Table; @@ -66,7 +67,10 @@ public void clean() throws Exception { @BeforeSuite public void setup() throws Exception { Class.forName("org.apache.derby.jdbc.EmbeddedDriver").newInstance(); - startMetastore(); + try { + startMetastore(); + } catch (AlreadyExistsException ignored) { } + tmpDir = Files.createTempDir(); dbUri = String.format("%s/%s/%s", tmpDir.getAbsolutePath(),"metastore", dbName); try { diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java index aa6c73d5295..2797bdf4960 100644 --- a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java +++ b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/HiveMetadataWriterTest.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.serde.serdeConstants; @@ -128,6 +129,10 @@ public void clean() throws Exception { @BeforeSuite public void setUp() throws Exception { Class.forName("org.apache.derby.jdbc.EmbeddedDriver").newInstance(); + try { + startMetastore(); + } catch (AlreadyExistsException ignored) { } + State state = ConfigUtils.configToState(ConfigUtils.propertiesToConfig(hiveConf.getAllProperties())); Optional metastoreUri = Optional.fromNullable(state.getProperties().getProperty(HiveRegister.HIVE_METASTORE_URI_KEY)); hc = HiveMetastoreClientPool.get(state.getProperties(), metastoreUri); From bea009e6035937643bfbed73b7b63455a02d790f Mon Sep 17 00:00:00 2001 From: William Lo Date: Thu, 20 Jul 2023 17:18:36 -0400 Subject: [PATCH 25/30] =?UTF-8?q?[GOBBLIN-1857]=20Add=20override=20flag=20?= =?UTF-8?q?to=20force=20generate=20a=20job=20execution=20id=20based=20on?= =?UTF-8?q?=20gobbl=E2=80=A6=20(#3719)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add override flag to force generate a job execution id based on gobblin cluster system time * fix typo --- .../GobblinClusterConfigurationKeys.java | 4 ++ .../gobblin/cluster/HelixJobsMapping.java | 12 +++-- .../cluster/GobblinHelixJobMappingTest.java | 53 +++++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobMappingTest.java diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java index ef83ab029a2..a2e8a4b5654 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/GobblinClusterConfigurationKeys.java @@ -180,6 +180,10 @@ public class GobblinClusterConfigurationKeys { public static final String CANCEL_RUNNING_JOB_ON_DELETE = GOBBLIN_CLUSTER_PREFIX + "job.cancelRunningJobOnDelete"; public static final String DEFAULT_CANCEL_RUNNING_JOB_ON_DELETE = "false"; + // Job Execution ID for Helix jobs is inferred from Flow Execution IDs, but there are scenarios in earlyStop jobs where + // this behavior needs to be avoided due to concurrent planning and actual jobs sharing the same execution ID + public static final String USE_GENERATED_JOBEXECUTION_IDS = GOBBLIN_CLUSTER_PREFIX + "job.useGeneratedJobExecutionIds"; + // By default we cancel job by calling helix stop API. In some cases, jobs just hang in STOPPING state and preventing // new job being launched. We provide this config to give an option to cancel jobs by calling Delete API. Directly delete // a Helix workflow should be safe in Gobblin world, as Gobblin job is stateless for Helix since we implement our own state store diff --git a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixJobsMapping.java b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixJobsMapping.java index 8128aa0aab1..b8fb436e4ce 100644 --- a/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixJobsMapping.java +++ b/gobblin-cluster/src/main/java/org/apache/gobblin/cluster/HelixJobsMapping.java @@ -96,15 +96,17 @@ public HelixJobsMapping(Config sysConfig, URI fsUri, String rootDir) { } public static String createPlanningJobId (Properties jobPlanningProps) { + long planningJobId = PropertiesUtils.getPropAsBoolean(jobPlanningProps, GobblinClusterConfigurationKeys.USE_GENERATED_JOBEXECUTION_IDS, "false") ? + System.currentTimeMillis() : PropertiesUtils.getPropAsLong(jobPlanningProps, ConfigurationKeys.FLOW_EXECUTION_ID_KEY, System.currentTimeMillis()); return JobLauncherUtils.newJobId(GobblinClusterConfigurationKeys.PLANNING_JOB_NAME_PREFIX - + JobState.getJobNameFromProps(jobPlanningProps), - PropertiesUtils.getPropAsLong(jobPlanningProps, ConfigurationKeys.FLOW_EXECUTION_ID_KEY, System.currentTimeMillis())); + + JobState.getJobNameFromProps(jobPlanningProps), planningJobId); } public static String createActualJobId (Properties jobProps) { - return JobLauncherUtils.newJobId(GobblinClusterConfigurationKeys.ACTUAL_JOB_NAME_PREFIX - + JobState.getJobNameFromProps(jobProps), - PropertiesUtils.getPropAsLong(jobProps, ConfigurationKeys.FLOW_EXECUTION_ID_KEY, System.currentTimeMillis())); + long actualJobId = PropertiesUtils.getPropAsBoolean(jobProps, GobblinClusterConfigurationKeys.USE_GENERATED_JOBEXECUTION_IDS, "false") ? + System.currentTimeMillis() : PropertiesUtils.getPropAsLong(jobProps, ConfigurationKeys.FLOW_EXECUTION_ID_KEY, System.currentTimeMillis()); + return JobLauncherUtils.newJobId(GobblinClusterConfigurationKeys.ACTUAL_JOB_NAME_PREFIX + + JobState.getJobNameFromProps(jobProps), actualJobId); } @Nullable diff --git a/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobMappingTest.java b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobMappingTest.java new file mode 100644 index 00000000000..022a9fb8a66 --- /dev/null +++ b/gobblin-cluster/src/test/java/org/apache/gobblin/cluster/GobblinHelixJobMappingTest.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.cluster; + +import java.util.Properties; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import org.apache.gobblin.configuration.ConfigurationKeys; + + +public class GobblinHelixJobMappingTest { + + @Test + void testMapJobNameWithFlowExecutionId() { + Properties props = new Properties(); + props.setProperty(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, "1234"); + props.setProperty(ConfigurationKeys.JOB_NAME_KEY, "job1"); + String planningJobId = HelixJobsMapping.createPlanningJobId(props); + String actualJobId = HelixJobsMapping.createActualJobId(props); + Assert.assertEquals(planningJobId, "job_PlanningJobjob1_1234"); + Assert.assertEquals(actualJobId, "job_ActualJobjob1_1234"); + } + + @Test + void testMapJobNameWithOverride() { + Properties props = new Properties(); + props.setProperty(GobblinClusterConfigurationKeys.USE_GENERATED_JOBEXECUTION_IDS, "true"); + props.setProperty(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, "1234"); + props.setProperty(ConfigurationKeys.JOB_NAME_KEY, "job1"); + String planningJobId = HelixJobsMapping.createPlanningJobId(props); + String actualJobId = HelixJobsMapping.createActualJobId(props); + // The jobID will be the system timestamp instead of the flow execution ID + Assert.assertNotEquals(planningJobId, "job_PlanningJobjob1_1234"); + Assert.assertNotEquals(actualJobId, "job_ActualJobjob1_1234"); + } +} From 0151890d1f5274fb86827074a755e54adf189ba0 Mon Sep 17 00:00:00 2001 From: meethngala Date: Fri, 21 Jul 2023 09:26:36 -0700 Subject: [PATCH 26/30] [GOBBLIN- 1856] Add flow trigger handler leasing metrics (#3717) * add flow trigger handler leasing metrics * remove todo and resolve merge conflicts * address PR comments --------- Co-authored-by: Meeth Gala --- .../gobblin/metrics/ServiceMetricNames.java | 5 +++++ .../orchestration/FlowTriggerHandler.java | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/gobblin-metrics-libs/gobblin-metrics/src/main/java/org/apache/gobblin/metrics/ServiceMetricNames.java b/gobblin-metrics-libs/gobblin-metrics/src/main/java/org/apache/gobblin/metrics/ServiceMetricNames.java index 94e2a82c6c0..9e9ab1e69b2 100644 --- a/gobblin-metrics-libs/gobblin-metrics/src/main/java/org/apache/gobblin/metrics/ServiceMetricNames.java +++ b/gobblin-metrics-libs/gobblin-metrics/src/main/java/org/apache/gobblin/metrics/ServiceMetricNames.java @@ -34,6 +34,11 @@ public class ServiceMetricNames { public static final String FLOW_ORCHESTRATION_TIMER = GOBBLIN_SERVICE_PREFIX + ".flowOrchestration.time"; public static final String FLOW_ORCHESTRATION_DELAY = GOBBLIN_SERVICE_PREFIX + ".flowOrchestration.delay"; + // Flow Trigger Handler Lease Status Counts + public static final String FLOW_TRIGGER_HANDLER_LEASE_OBTAINED_COUNT = GOBBLIN_SERVICE_PREFIX + ".flowTriggerHandler.leaseObtained"; + public static final String FLOW_TRIGGER_HANDLER_LEASED_TO_ANOTHER_COUNT = GOBBLIN_SERVICE_PREFIX + ".flowTriggerHandler.leasedToAnother"; + public static final String FLOW_TRIGGER_HANDLER_NO_LONGER_LEASING_COUNT = GOBBLIN_SERVICE_PREFIX + ".flowTriggerHandler.noLongerLeasing"; + //Job status poll timer public static final String JOB_STATUS_POLLED_TIMER = GOBBLIN_SERVICE_PREFIX + ".jobStatusPoll.time"; diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java index ec63276c4c5..7fdee038414 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java @@ -38,8 +38,10 @@ import org.apache.gobblin.configuration.ConfigurationKeys; import org.apache.gobblin.instrumented.Instrumented; +import org.apache.gobblin.metrics.ContextAwareCounter; import org.apache.gobblin.metrics.ContextAwareMeter; import org.apache.gobblin.metrics.MetricContext; +import org.apache.gobblin.metrics.ServiceMetricNames; import org.apache.gobblin.runtime.api.DagActionStore; import org.apache.gobblin.runtime.api.MultiActiveLeaseArbiter; import org.apache.gobblin.runtime.api.MysqlMultiActiveLeaseArbiter; @@ -69,6 +71,12 @@ public class FlowTriggerHandler { private MetricContext metricContext; private ContextAwareMeter numFlowsSubmitted; + private ContextAwareCounter leaseObtainedCount; + + private ContextAwareCounter leasedToAnotherStatusCount; + + private ContextAwareCounter noLongerLeasingStatusCount; + @Inject public FlowTriggerHandler(Config config, Optional leaseDeterminationStore, SchedulerService schedulerService, Optional dagActionStore) { @@ -80,6 +88,9 @@ public FlowTriggerHandler(Config config, Optional lease this.metricContext = Instrumented.getMetricContext(new org.apache.gobblin.configuration.State(ConfigUtils.configToProperties(config)), this.getClass()); this.numFlowsSubmitted = metricContext.contextAwareMeter(RuntimeMetrics.GOBBLIN_FLOW_TRIGGER_HANDLER_NUM_FLOWS_SUBMITTED); + this.leaseObtainedCount = this.metricContext.contextAwareCounter(ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASE_OBTAINED_COUNT); + this.leasedToAnotherStatusCount = this.metricContext.contextAwareCounter(ServiceMetricNames.FLOW_TRIGGER_HANDLER_LEASED_TO_ANOTHER_COUNT); + this.noLongerLeasingStatusCount = this.metricContext.contextAwareCounter(ServiceMetricNames.FLOW_TRIGGER_HANDLER_NO_LONGER_LEASING_COUNT); } /** @@ -94,10 +105,12 @@ public void handleTriggerEvent(Properties jobProps, DagActionStore.DagAction flo throws IOException { if (multiActiveLeaseArbiter.isPresent()) { MultiActiveLeaseArbiter.LeaseAttemptStatus leaseAttemptStatus = multiActiveLeaseArbiter.get().tryAcquireLease(flowAction, eventTimeMillis); - // TODO: add a log event or metric for each of these cases if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeaseObtainedStatus) { MultiActiveLeaseArbiter.LeaseObtainedStatus leaseObtainedStatus = (MultiActiveLeaseArbiter.LeaseObtainedStatus) leaseAttemptStatus; + this.leaseObtainedCount.inc(); if (persistFlowAction(leaseObtainedStatus)) { + log.info("Successfully persisted lease: [%s, eventTimestamp: %s] ", leaseObtainedStatus.getFlowAction(), + leaseObtainedStatus.getEventTimestamp()); return; } // If persisting the flow action failed, then we set another trigger for this event to occur immediately to @@ -107,10 +120,14 @@ public void handleTriggerEvent(Properties jobProps, DagActionStore.DagAction flo eventTimeMillis); return; } else if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.LeasedToAnotherStatus) { + this.leasedToAnotherStatusCount.inc(); scheduleReminderForEvent(jobProps, (MultiActiveLeaseArbiter.LeasedToAnotherStatus) leaseAttemptStatus, eventTimeMillis); return; } else if (leaseAttemptStatus instanceof MultiActiveLeaseArbiter.NoLongerLeasingStatus) { + this.noLongerLeasingStatusCount.inc(); + log.debug("Received type of leaseAttemptStatus: [%s, eventTimestamp: %s] ", leaseAttemptStatus.getClass().getName(), + eventTimeMillis); return; } throw new RuntimeException(String.format("Received type of leaseAttemptStatus: %s not handled by this method", From 14262c1198195fe95c86f2b7445d9d9b22c5739a Mon Sep 17 00:00:00 2001 From: Arjun Singh Bora Date: Mon, 24 Jul 2023 11:36:04 -0700 Subject: [PATCH 27/30] Correct num of failures (#3722) * set number of failed attempts correctly * set number of failed attempts correctly --- .../java/org/apache/gobblin/runtime/AbstractJobLauncher.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java index 6e47cc482b9..3f2570c68f5 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/AbstractJobLauncher.java @@ -608,6 +608,8 @@ public WorkUnit apply(@Nullable WorkUnit input) { String errMsg = "Failed to launch and run job " + jobId + " due to " + t.getMessage(); LOG.error(errMsg + ": " + t, t); this.jobContext.getJobState().setJobFailureException(t); + jobState.setProp(ConfigurationKeys.JOB_FAILURES_KEY, + Integer.parseInt(jobState.getProp(ConfigurationKeys.JOB_FAILURES_KEY, "0")) + 1); } finally { try { troubleshooter.refineIssues(); From ba408f23c89357bc36ee70415041c2cff2b36df0 Mon Sep 17 00:00:00 2001 From: Tao Qin <35046097+wsarecv@users.noreply.github.com> Date: Mon, 24 Jul 2023 12:25:20 -0700 Subject: [PATCH 28/30] [GOBBLIN-1838] Introduce total count based completion watermark (#3701) * [GOBBLIN-1838] Introduce total count based completion watermark * Refactor and address comments * fix style * combine completeness computation * Fix the logic for quiet topics * Add UT for ComopletenessWatermarkUpdater * space only change * refine --------- Co-authored-by: Tao Qin --- .../verifier/KafkaAuditCountVerifier.java | 124 ++++++-- .../verifier/KafkaAuditCountVerifierTest.java | 75 ++++- .../writer/CompletenessWatermarkUpdater.java | 284 ++++++++++++++++++ .../iceberg/writer/IcebergMetadataWriter.java | 178 +++++------ .../IcebergMetadataWriterConfigKeys.java | 8 +- .../CompletenessWatermarkUpdaterTest.java | 205 +++++++++++++ .../writer/IcebergMetadataWriterTest.java | 30 +- 7 files changed, 761 insertions(+), 143 deletions(-) create mode 100644 gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/CompletenessWatermarkUpdater.java create mode 100644 gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/CompletenessWatermarkUpdaterTest.java diff --git a/gobblin-completeness/src/main/java/org/apache/gobblin/completeness/verifier/KafkaAuditCountVerifier.java b/gobblin-completeness/src/main/java/org/apache/gobblin/completeness/verifier/KafkaAuditCountVerifier.java index d4495f9ebf7..77d89ff3176 100644 --- a/gobblin-completeness/src/main/java/org/apache/gobblin/completeness/verifier/KafkaAuditCountVerifier.java +++ b/gobblin-completeness/src/main/java/org/apache/gobblin/completeness/verifier/KafkaAuditCountVerifier.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import com.google.common.base.Preconditions; @@ -43,14 +44,22 @@ public class KafkaAuditCountVerifier { public static final String COMPLETENESS_PREFIX = "completeness."; public static final String SOURCE_TIER = COMPLETENESS_PREFIX + "source.tier"; public static final String REFERENCE_TIERS = COMPLETENESS_PREFIX + "reference.tiers"; + public static final String TOTAL_COUNT_REFERENCE_TIERS = COMPLETENESS_PREFIX + "totalCount.reference.tiers"; public static final String THRESHOLD = COMPLETENESS_PREFIX + "threshold"; private static final double DEFAULT_THRESHOLD = 0.999; public static final String COMPLETE_ON_NO_COUNTS = COMPLETENESS_PREFIX + "complete.on.no.counts"; + + public enum CompletenessType { + ClassicCompleteness, + TotalCountCompleteness + } + private final boolean returnCompleteOnNoCounts; private final AuditCountClient auditCountClient; private final String srcTier; private final Collection refTiers; + private final Collection totalCountRefTiers; private final double threshold; /** @@ -69,6 +78,9 @@ public KafkaAuditCountVerifier(State state, AuditCountClient client) { state.getPropAsDouble(THRESHOLD, DEFAULT_THRESHOLD); this.srcTier = state.getProp(SOURCE_TIER); this.refTiers = Splitter.on(",").omitEmptyStrings().trimResults().splitToList(state.getProp(REFERENCE_TIERS)); + this.totalCountRefTiers = state.contains(TOTAL_COUNT_REFERENCE_TIERS) + ? Splitter.on(",").omitEmptyStrings().trimResults().splitToList(state.getProp(TOTAL_COUNT_REFERENCE_TIERS)) + : null; this.returnCompleteOnNoCounts = state.getPropAsBoolean(COMPLETE_ON_NO_COUNTS, false); } @@ -90,23 +102,52 @@ private static AuditCountClient getAuditClient(State state) { } } + public Map calculateCompleteness(String datasetName, long beginInMillis, long endInMillis) + throws IOException { + return calculateCompleteness(datasetName, beginInMillis, endInMillis, this.threshold); + } + /** * Compare source tier against reference tiers. - * Compute completion percentage by srcCount/refCount. Return true iff the highest percentages is greater than threshold. + * Compute completion percentage which is true iff the calculated percentages is greater than threshold. * * @param datasetName A dataset short name like 'PageViewEvent' * @param beginInMillis Unix timestamp in milliseconds * @param endInMillis Unix timestamp in milliseconds * @param threshold User defined threshold + * + * @return a map of completeness result by CompletenessType */ - public boolean isComplete(String datasetName, long beginInMillis, long endInMillis, double threshold) - throws IOException { - return getCompletenessPercentage(datasetName, beginInMillis, endInMillis) > threshold; + public Map calculateCompleteness(String datasetName, long beginInMillis, long endInMillis, + double threshold) throws IOException { + Map countsByTier = getTierAndCount(datasetName, beginInMillis, endInMillis); + log.info(String.format("checkTierCounts: audit counts map for %s for range [%s,%s]", datasetName, beginInMillis, endInMillis)); + countsByTier.forEach((x,y) -> log.info(String.format(" %s : %s ", x, y))); + + Map result = new HashMap<>(); + result.put(CompletenessType.ClassicCompleteness, calculateCompleteness(datasetName, beginInMillis, endInMillis, + CompletenessType.ClassicCompleteness, countsByTier) > threshold); + result.put(CompletenessType.TotalCountCompleteness, calculateCompleteness(datasetName, beginInMillis, endInMillis, + CompletenessType.TotalCountCompleteness, countsByTier) > threshold); + return result; } - public boolean isComplete(String datasetName, long beginInMillis, long endInMillis) - throws IOException { - return isComplete(datasetName, beginInMillis, endInMillis, this.threshold); + private double calculateCompleteness(String datasetName, long beginInMillis, long endInMillis, CompletenessType type, + Map countsByTier) throws IOException { + if (countsByTier.isEmpty() && this.returnCompleteOnNoCounts) { + log.info(String.format("Found empty counts map for %s, returning complete", datasetName)); + return 1.0; + } + + switch (type) { + case ClassicCompleteness: + return calculateClassicCompleteness(datasetName, beginInMillis, endInMillis, countsByTier); + case TotalCountCompleteness: + return calculateTotalCountCompleteness(datasetName, beginInMillis, endInMillis, countsByTier); + default: + log.error("Skip unsupported completeness type {}", type); + return -1; + } } /** @@ -118,39 +159,70 @@ public boolean isComplete(String datasetName, long beginInMillis, long endInMill * * @return The highest percentage value */ - private double getCompletenessPercentage(String datasetName, long beginInMillis, long endInMillis) throws IOException { - Map countsByTier = getTierAndCount(datasetName, beginInMillis, endInMillis); - log.info(String.format("Audit counts map for %s for range [%s,%s]", datasetName, beginInMillis, endInMillis)); - countsByTier.forEach((x,y) -> log.info(String.format(" %s : %s ", x, y))); - if (countsByTier.isEmpty() && this.returnCompleteOnNoCounts) { - log.info(String.format("Found empty counts map for %s, returning complete", datasetName)); - return 1.0; - } - double percent = -1; - if (!countsByTier.containsKey(this.srcTier)) { - throw new IOException(String.format("Source tier %s audit count cannot be retrieved for dataset %s between %s and %s", this.srcTier, datasetName, beginInMillis, endInMillis)); - } + private double calculateClassicCompleteness(String datasetName, long beginInMillis, long endInMillis, + Map countsByTier) throws IOException { + validateTierCounts(datasetName, beginInMillis, endInMillis, countsByTier, this.srcTier, this.refTiers); + double percent = -1; for (String refTier: this.refTiers) { - if (!countsByTier.containsKey(refTier)) { - throw new IOException(String.format("Reference tier %s audit count cannot be retrieved for dataset %s between %s and %s", refTier, datasetName, beginInMillis, endInMillis)); - } long refCount = countsByTier.get(refTier); - if(refCount <= 0) { - throw new IOException(String.format("Reference tier %s count cannot be less than or equal to zero", refTier)); - } long srcCount = countsByTier.get(this.srcTier); - percent = Double.max(percent, (double) srcCount / (double) refCount); } if (percent < 0) { throw new IOException("Cannot calculate completion percentage"); } + return percent; + } + /** + * Check total count based completeness by comparing source tier against reference tiers, + * and calculate the completion percentage by srcCount/sum_of(refCount). + * + * @param datasetName A dataset short name like 'PageViewEvent' + * @param beginInMillis Unix timestamp in milliseconds + * @param endInMillis Unix timestamp in milliseconds + * + * @return The percentage value by srcCount/sum_of(refCount) + */ + private double calculateTotalCountCompleteness(String datasetName, long beginInMillis, long endInMillis, + Map countsByTier) throws IOException { + if (this.totalCountRefTiers == null) { + return -1; + } + validateTierCounts(datasetName, beginInMillis, endInMillis, countsByTier, this.srcTier, this.totalCountRefTiers); + + long srcCount = countsByTier.get(this.srcTier); + long totalRefCount = this.totalCountRefTiers + .stream() + .mapToLong(countsByTier::get) + .sum(); + double percent = Double.max(-1, (double) srcCount / (double) totalRefCount); + if (percent < 0) { + throw new IOException("Cannot calculate total count completion percentage"); + } return percent; } + private static void validateTierCounts(String datasetName, long beginInMillis, long endInMillis, Map countsByTier, + String sourceTier, Collection referenceTiers) + throws IOException { + if (!countsByTier.containsKey(sourceTier)) { + throw new IOException(String.format("Source tier %s audit count cannot be retrieved for dataset %s between %s and %s", sourceTier, datasetName, beginInMillis, endInMillis)); + } + + for (String refTier: referenceTiers) { + if (!countsByTier.containsKey(refTier)) { + throw new IOException(String.format("Reference tier %s audit count cannot be retrieved for dataset %s between %s and %s", refTier, datasetName, beginInMillis, endInMillis)); + } + long refCount = countsByTier.get(refTier); + if(refCount <= 0) { + throw new IOException(String.format("Reference tier %s count cannot be less than or equal to zero", refTier)); + } + } + } + /** * Fetch all pairs for a given dataset between a time range */ diff --git a/gobblin-completeness/src/test/java/org/apache/gobblin/completeness/verifier/KafkaAuditCountVerifierTest.java b/gobblin-completeness/src/test/java/org/apache/gobblin/completeness/verifier/KafkaAuditCountVerifierTest.java index 7a005c6c272..3e7140f419e 100644 --- a/gobblin-completeness/src/test/java/org/apache/gobblin/completeness/verifier/KafkaAuditCountVerifierTest.java +++ b/gobblin-completeness/src/test/java/org/apache/gobblin/completeness/verifier/KafkaAuditCountVerifierTest.java @@ -33,6 +33,11 @@ public class KafkaAuditCountVerifierTest { public static final String SOURCE_TIER = "gobblin"; public static final String REFERENCE_TIERS = "producer"; + public static final String TOTAL_COUNT_REF_TIER_0 = "producer_0"; + public static final String TOTAL_COUNT_REF_TIER_1 = "producer_1"; + public static final String TOTAL_COUNT_REFERENCE_TIERS = TOTAL_COUNT_REF_TIER_0 + "," + TOTAL_COUNT_REF_TIER_1; + + public void testFetch() throws IOException { final String topic = "testTopic"; State props = new State(); @@ -48,22 +53,86 @@ public void testFetch() throws IOException { REFERENCE_TIERS, 1000L )); // Default threshold - Assert.assertTrue(verifier.isComplete(topic, 0L, 0L)); + Assert.assertTrue(verifier.calculateCompleteness(topic, 0L, 0L) + .get(KafkaAuditCountVerifier.CompletenessType.ClassicCompleteness)); // 99.999 % complete client.setTierCounts(ImmutableMap.of( SOURCE_TIER, 999L, REFERENCE_TIERS, 1000L )); - Assert.assertTrue(verifier.isComplete(topic, 0L, 0L)); + Assert.assertTrue(verifier.calculateCompleteness(topic, 0L, 0L) + .get(KafkaAuditCountVerifier.CompletenessType.ClassicCompleteness)); // <= 99% complete client.setTierCounts(ImmutableMap.of( SOURCE_TIER, 990L, REFERENCE_TIERS, 1000L )); - Assert.assertFalse(verifier.isComplete(topic, 0L, 0L)); + Assert.assertFalse(verifier.calculateCompleteness(topic, 0L, 0L) + .get(KafkaAuditCountVerifier.CompletenessType.ClassicCompleteness)); + } + + public void testTotalCountCompleteness() throws IOException { + final String topic = "testTopic"; + State props = new State(); + props.setProp(KafkaAuditCountVerifier.SOURCE_TIER, SOURCE_TIER); + props.setProp(KafkaAuditCountVerifier.REFERENCE_TIERS, REFERENCE_TIERS); + props.setProp(KafkaAuditCountVerifier.TOTAL_COUNT_REFERENCE_TIERS, TOTAL_COUNT_REFERENCE_TIERS); + props.setProp(KafkaAuditCountVerifier.THRESHOLD, ".99"); + TestAuditClient client = new TestAuditClient(props); + KafkaAuditCountVerifier verifier = new KafkaAuditCountVerifier(props, client); + + // All complete + client.setTierCounts(ImmutableMap.of( + SOURCE_TIER, 1000L, + REFERENCE_TIERS, 1000L, + TOTAL_COUNT_REF_TIER_0, 600L, + TOTAL_COUNT_REF_TIER_1, 400L + )); + // Default threshold + Assert.assertTrue(verifier.calculateCompleteness(topic, 0L, 0L) + .get(KafkaAuditCountVerifier.CompletenessType.TotalCountCompleteness)); + + // 99.999 % complete + client.setTierCounts(ImmutableMap.of( + SOURCE_TIER, 999L, + REFERENCE_TIERS, 1000L, + TOTAL_COUNT_REF_TIER_0, 600L, + TOTAL_COUNT_REF_TIER_1, 400L + )); + Assert.assertTrue(verifier.calculateCompleteness(topic, 0L, 0L) + .get(KafkaAuditCountVerifier.CompletenessType.TotalCountCompleteness)); + + // <= 99% complete + client.setTierCounts(ImmutableMap.of( + SOURCE_TIER, 990L, + REFERENCE_TIERS, 1000L, + TOTAL_COUNT_REF_TIER_0, 600L, + TOTAL_COUNT_REF_TIER_1, 400L + )); + Assert.assertFalse(verifier.calculateCompleteness(topic, 0L, 0L) + .get(KafkaAuditCountVerifier.CompletenessType.TotalCountCompleteness)); } + public void testEmptyAuditCount() throws IOException { + final String topic = "testTopic"; + State props = new State(); + props.setProp(KafkaAuditCountVerifier.SOURCE_TIER, SOURCE_TIER); + props.setProp(KafkaAuditCountVerifier.REFERENCE_TIERS, REFERENCE_TIERS); + props.setProp(KafkaAuditCountVerifier.TOTAL_COUNT_REFERENCE_TIERS, TOTAL_COUNT_REFERENCE_TIERS); + props.setProp(KafkaAuditCountVerifier.THRESHOLD, ".99"); + props.setProp(KafkaAuditCountVerifier.COMPLETE_ON_NO_COUNTS, true); + TestAuditClient client = new TestAuditClient(props); + KafkaAuditCountVerifier verifier = new KafkaAuditCountVerifier(props, client); + // Client gets empty audit count + client.setTierCounts(ImmutableMap.of()); + + // Should be complete, since COMPLETE_ON_NO_COUNTS=true + Assert.assertTrue(verifier.calculateCompleteness(topic, 0L, 0L) + .get(KafkaAuditCountVerifier.CompletenessType.ClassicCompleteness)); + Assert.assertTrue(verifier.calculateCompleteness(topic, 0L, 0L) + .get(KafkaAuditCountVerifier.CompletenessType.TotalCountCompleteness)); + } } diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/CompletenessWatermarkUpdater.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/CompletenessWatermarkUpdater.java new file mode 100644 index 00000000000..b022ac33983 --- /dev/null +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/CompletenessWatermarkUpdater.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.iceberg.writer; + +import com.google.common.annotations.VisibleForTesting; +import java.io.IOException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.SortedSet; +import lombok.extern.slf4j.Slf4j; +import org.apache.gobblin.completeness.verifier.KafkaAuditCountVerifier; +import org.apache.gobblin.configuration.State; +import org.apache.gobblin.time.TimeIterator; + +import static org.apache.gobblin.iceberg.writer.IcebergMetadataWriterConfigKeys.*; + +/** + * A class for completeness watermark updater. + * It computes the new watermarks and updates below entities: + * 1. the properties in {@link IcebergMetadataWriter.TableMetadata} + * 2. {@link gobblin.configuration.State} + * 3. the completionWatermark in {@link IcebergMetadataWriter.TableMetadata} + */ +@Slf4j +public class CompletenessWatermarkUpdater { + private final String topic; + private final String auditCheckGranularity; + + protected final String timeZone; + protected final IcebergMetadataWriter.TableMetadata tableMetadata; + protected final Map propsToUpdate; + protected final State stateToUpdate; + protected KafkaAuditCountVerifier auditCountVerifier; + + public CompletenessWatermarkUpdater(String topic, String auditCheckGranularity, String timeZone, + IcebergMetadataWriter.TableMetadata tableMetadata, Map propsToUpdate, State stateToUpdate, + KafkaAuditCountVerifier auditCountVerifier) { + this.tableMetadata = tableMetadata; + this.topic = topic; + this.auditCheckGranularity = auditCheckGranularity; + this.timeZone = timeZone; + this.propsToUpdate = propsToUpdate; + this.stateToUpdate = stateToUpdate; + this.auditCountVerifier = auditCountVerifier; + } + + /** + * Update TableMetadata with the new completion watermark upon a successful audit check + * @param timestamps Sorted set in reverse order of timestamps to check audit counts for + * @param includeTotalCountCompletionWatermark If totalCountCompletionWatermark should be calculated + */ + public void run(SortedSet timestamps, boolean includeTotalCountCompletionWatermark) { + String tableName = tableMetadata.table.get().name(); + if (this.topic == null) { + log.error(String.format("Not performing audit check. %s is null. Please set as table property of %s", + TOPIC_NAME_KEY, tableName)); + } + computeAndUpdateWatermark(tableName, timestamps, includeTotalCountCompletionWatermark); + } + + private void computeAndUpdateWatermark(String tableName, SortedSet timestamps, + boolean includeTotalCountWatermark) { + log.info(String.format("Compute completion watermark for %s and timestamps %s with previous watermark %s, previous totalCount watermark %s, includeTotalCountWatermark=%b", + this.topic, timestamps, tableMetadata.completionWatermark, tableMetadata.totalCountCompletionWatermark, + includeTotalCountWatermark)); + + WatermarkUpdaterSet updaterSet = new WatermarkUpdaterSet(this.tableMetadata, this.timeZone, this.propsToUpdate, + this.stateToUpdate, includeTotalCountWatermark); + if(timestamps == null || timestamps.size() <= 0) { + log.error("Cannot create time iterator. Empty for null timestamps"); + return; + } + + ZonedDateTime now = ZonedDateTime.now(ZoneId.of(this.timeZone)); + TimeIterator.Granularity granularity = TimeIterator.Granularity.valueOf(this.auditCheckGranularity); + ZonedDateTime startDT = timestamps.first(); + ZonedDateTime endDT = timestamps.last(); + TimeIterator iterator = new TimeIterator(startDT, endDT, granularity, true); + try { + while (iterator.hasNext()) { + ZonedDateTime timestampDT = iterator.next(); + updaterSet.checkForEarlyStop(timestampDT, now, granularity); + if (updaterSet.allFinished()) { + break; + } + + ZonedDateTime auditCountCheckLowerBoundDT = TimeIterator.dec(timestampDT, granularity, 1); + Map results = + this.auditCountVerifier.calculateCompleteness(this.topic, + auditCountCheckLowerBoundDT.toInstant().toEpochMilli(), + timestampDT.toInstant().toEpochMilli()); + + updaterSet.computeAndUpdate(results, timestampDT); + } + } catch (IOException e) { + log.warn("Exception during audit count check: ", e); + } + } + + /** + * A class that contains both ClassicWatermakrUpdater and TotalCountWatermarkUpdater + */ + static class WatermarkUpdaterSet { + private final List updaters; + + WatermarkUpdaterSet(IcebergMetadataWriter.TableMetadata tableMetadata, String timeZone, + Map propsToUpdate, State stateToUpdate, boolean includeTotalCountWatermark) { + this.updaters = new ArrayList<>(); + this.updaters.add(new ClassicWatermarkUpdater(tableMetadata.completionWatermark, timeZone, tableMetadata, + propsToUpdate, stateToUpdate)); + if (includeTotalCountWatermark) { + this.updaters.add(new TotalCountWatermarkUpdater(tableMetadata.totalCountCompletionWatermark, timeZone, + tableMetadata, propsToUpdate, stateToUpdate)); + } + } + + void checkForEarlyStop(ZonedDateTime timestampDT, ZonedDateTime now, + TimeIterator.Granularity granularity) { + this.updaters.stream().forEach(updater + -> updater.checkForEarlyStop(timestampDT, now, granularity)); + } + + boolean allFinished() { + return this.updaters.stream().allMatch(updater -> updater.isFinished()); + } + + void computeAndUpdate(Map results, + ZonedDateTime timestampDT) { + this.updaters.stream() + .filter(updater -> !updater.isFinished()) + .forEach(updater -> updater.computeAndUpdate(results, timestampDT)); + } + } + + /** + * A stateful class for watermark updaters. + * The updater starts with finished=false state. + * Then computeAndUpdate() is called multiple times with the parameters: + * 1. The completeness audit results within (datepartition-1, datepartition) + * 2. the datepartition timestamp + * The method is call multiple times in descending order of the datepartition timestamp. + *

+ * When the audit result is complete for a timestamp, it updates below entities: + * 1. the properties in {@link IcebergMetadataWriter.TableMetadata} + * 2. {@link gobblin.configuration.State} + * 3. the completionWatermark in {@link IcebergMetadataWriter.TableMetadata} + * And it turns into finished=true state, in which the following computeAndUpdate() calls will be skipped. + */ + static abstract class WatermarkUpdater { + protected final long previousWatermark; + protected final ZonedDateTime prevWatermarkDT; + protected final String timeZone; + protected boolean finished = false; + protected final IcebergMetadataWriter.TableMetadata tableMetadata; + protected final Map propsToUpdate; + protected final State stateToUpdate; + + public WatermarkUpdater(long previousWatermark, String timeZone, IcebergMetadataWriter.TableMetadata tableMetadata, + Map propsToUpdate, State stateToUpdate) { + this.previousWatermark = previousWatermark; + this.timeZone = timeZone; + this.tableMetadata = tableMetadata; + this.propsToUpdate = propsToUpdate; + this.stateToUpdate = stateToUpdate; + + prevWatermarkDT = Instant.ofEpochMilli(previousWatermark).atZone(ZoneId.of(this.timeZone)); + } + + public void computeAndUpdate(Map results, + ZonedDateTime timestampDT) { + if (finished) { + return; + } + computeAndUpdateInternal(results, timestampDT); + } + + protected abstract void computeAndUpdateInternal(Map results, + ZonedDateTime timestampDT); + + protected boolean isFinished() { + return this.finished; + } + + protected void setFinished() { + this.finished = true; + } + + protected void checkForEarlyStop(ZonedDateTime timestampDT, ZonedDateTime now, + TimeIterator.Granularity granularity) { + if (isFinished() + || (timestampDT.isAfter(this.prevWatermarkDT) + && TimeIterator.durationBetween(this.prevWatermarkDT, now, granularity) > 0)) { + return; + } + setFinished(); + } + } + + @VisibleForTesting + void setAuditCountVerifier(KafkaAuditCountVerifier auditCountVerifier) { + this.auditCountVerifier = auditCountVerifier; + } + + static class ClassicWatermarkUpdater extends WatermarkUpdater { + public ClassicWatermarkUpdater(long previousWatermark, String timeZone, + IcebergMetadataWriter.TableMetadata tableMetadata, Map propsToUpdate, State stateToUpdate) { + super(previousWatermark, timeZone, tableMetadata, propsToUpdate, stateToUpdate); + } + + @Override + protected void computeAndUpdateInternal(Map results, + ZonedDateTime timestampDT) { + if (!results.get(KafkaAuditCountVerifier.CompletenessType.ClassicCompleteness)) { + return; + } + + setFinished(); + long updatedWatermark = timestampDT.toInstant().toEpochMilli(); + this.stateToUpdate.setProp( + String.format(STATE_COMPLETION_WATERMARK_KEY_OF_TABLE, + this.tableMetadata.table.get().name().toLowerCase(Locale.ROOT)), + updatedWatermark); + + if (updatedWatermark > this.previousWatermark) { + log.info(String.format("Updating %s for %s to %s", COMPLETION_WATERMARK_KEY, + this.tableMetadata.table.get().name(), updatedWatermark)); + this.propsToUpdate.put(COMPLETION_WATERMARK_KEY, String.valueOf(updatedWatermark)); + this.propsToUpdate.put(COMPLETION_WATERMARK_TIMEZONE_KEY, this.timeZone); + + this.tableMetadata.completionWatermark = updatedWatermark; + } + } + } + + static class TotalCountWatermarkUpdater extends WatermarkUpdater { + public TotalCountWatermarkUpdater(long previousWatermark, String timeZone, + IcebergMetadataWriter.TableMetadata tableMetadata, Map propsToUpdate, State stateToUpdate) { + super(previousWatermark, timeZone, tableMetadata, propsToUpdate, stateToUpdate); + } + + @Override + protected void computeAndUpdateInternal(Map results, + ZonedDateTime timestampDT) { + if (!results.get(KafkaAuditCountVerifier.CompletenessType.TotalCountCompleteness)) { + return; + } + + setFinished(); + long updatedWatermark = timestampDT.toInstant().toEpochMilli(); + this.stateToUpdate.setProp( + String.format(STATE_TOTAL_COUNT_COMPLETION_WATERMARK_KEY_OF_TABLE, + this.tableMetadata.table.get().name().toLowerCase(Locale.ROOT)), + updatedWatermark); + + if (updatedWatermark > previousWatermark) { + log.info(String.format("Updating %s for %s to %s", TOTAL_COUNT_COMPLETION_WATERMARK_KEY, + this.tableMetadata.table.get().name(), updatedWatermark)); + this.propsToUpdate.put(TOTAL_COUNT_COMPLETION_WATERMARK_KEY, String.valueOf(updatedWatermark)); + tableMetadata.totalCountCompletionWatermark = updatedWatermark; + } + } + } + +} diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java index b6c9e15dc11..19bc805eb6a 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java @@ -32,7 +32,6 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.SortedSet; @@ -122,7 +121,6 @@ import org.apache.gobblin.metrics.kafka.SchemaRegistryException; import org.apache.gobblin.source.extractor.extract.LongWatermark; import org.apache.gobblin.stream.RecordEnvelope; -import org.apache.gobblin.time.TimeIterator; import org.apache.gobblin.util.AvroUtils; import org.apache.gobblin.util.ClustersNames; import org.apache.gobblin.util.HadoopUtils; @@ -168,7 +166,9 @@ public class IcebergMetadataWriter implements MetadataWriter { private static final String ICEBERG_FILE_PATH_COLUMN = DataFile.FILE_PATH.name(); private final boolean completenessEnabled; + private final boolean totalCountCompletenessEnabled; private final WhitelistBlacklist completenessWhitelistBlacklist; + private final WhitelistBlacklist totalCountBasedCompletenessWhitelistBlacklist; private final String timeZone; private final DateTimeFormatter HOURLY_DATEPARTITION_FORMAT; private final String newPartitionColumn; @@ -234,8 +234,13 @@ public IcebergMetadataWriter(State state) throws IOException { FsPermission.getDefault()); } this.completenessEnabled = state.getPropAsBoolean(ICEBERG_COMPLETENESS_ENABLED, DEFAULT_ICEBERG_COMPLETENESS); + this.totalCountCompletenessEnabled = state.getPropAsBoolean(ICEBERG_TOTAL_COUNT_COMPLETENESS_ENABLED, + DEFAULT_ICEBERG_TOTAL_COUNT_COMPLETENESS); this.completenessWhitelistBlacklist = new WhitelistBlacklist(state.getProp(ICEBERG_COMPLETENESS_WHITELIST, ""), state.getProp(ICEBERG_COMPLETENESS_BLACKLIST, "")); + this.totalCountBasedCompletenessWhitelistBlacklist = new WhitelistBlacklist( + state.getProp(ICEBERG_TOTAL_COUNT_COMPLETENESS_WHITELIST, ""), + state.getProp(ICEBERG_TOTAL_COUNT_COMPLETENESS_BLACKLIST, "")); this.timeZone = state.getProp(TIME_ZONE_KEY, DEFAULT_TIME_ZONE); this.HOURLY_DATEPARTITION_FORMAT = DateTimeFormatter.ofPattern(DATEPARTITION_FORMAT) .withZone(ZoneId.of(this.timeZone)); @@ -258,7 +263,7 @@ protected void initializeCatalog() { } private org.apache.iceberg.Table getIcebergTable(TableIdentifier tid) throws NoSuchTableException { - TableMetadata tableMetadata = tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata()); + TableMetadata tableMetadata = tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata(this.conf)); if (!tableMetadata.table.isPresent()) { tableMetadata.table = Optional.of(catalog.loadTable(tid)); } @@ -304,7 +309,7 @@ private Long getAndPersistCurrentWatermark(TableIdentifier tid, String topicPart public void write(GobblinMetadataChangeEvent gmce, Map> newSpecsMap, Map> oldSpecsMap, HiveSpec tableSpec) throws IOException { TableIdentifier tid = TableIdentifier.of(tableSpec.getTable().getDbName(), tableSpec.getTable().getTableName()); - TableMetadata tableMetadata = tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata()); + TableMetadata tableMetadata = tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata(this.conf)); Table table; try { table = getIcebergTable(tid); @@ -327,6 +332,12 @@ public void write(GobblinMetadataChangeEvent gmce, Map> getLastOffset(TableMetadata tableMetadata) * the given {@link TableIdentifier} with the input {@link GobblinMetadataChangeEvent} */ private void mergeOffsets(GobblinMetadataChangeEvent gmce, TableIdentifier tid) { - TableMetadata tableMetadata = tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata()); + TableMetadata tableMetadata = tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata(this.conf)); tableMetadata.dataOffsetRange = Optional.of(tableMetadata.dataOffsetRange.or(() -> getLastOffset(tableMetadata))); Map> offsets = tableMetadata.dataOffsetRange.get(); for (Map.Entry entry : gmce.getTopicPartitionOffsetsRange().entrySet()) { @@ -424,7 +435,7 @@ public int compare(Range o1, Range o2) { protected void updateTableProperty(HiveSpec tableSpec, TableIdentifier tid, GobblinMetadataChangeEvent gmce) { org.apache.hadoop.hive.metastore.api.Table table = HiveMetaStoreUtils.getTable(tableSpec.getTable()); - TableMetadata tableMetadata = tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata()); + TableMetadata tableMetadata = tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata(this.conf)); tableMetadata.newProperties = Optional.of(IcebergUtils.getTableProperties(table)); String nativeName = tableMetadata.datasetName; String topic = nativeName.substring(nativeName.lastIndexOf("/") + 1); @@ -442,7 +453,7 @@ protected void updateTableProperty(HiveSpec tableSpec, TableIdentifier tid, Gobb */ private void computeCandidateSchema(GobblinMetadataChangeEvent gmce, TableIdentifier tid, HiveSpec spec) { Table table = getIcebergTable(tid); - TableMetadata tableMetadata = tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata()); + TableMetadata tableMetadata = tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata(this.conf)); org.apache.hadoop.hive.metastore.api.Table hiveTable = HiveMetaStoreUtils.getTable(spec.getTable()); tableMetadata.lastProperties = Optional.of(tableMetadata.lastProperties.or(() -> table.properties())); Map props = tableMetadata.lastProperties.get(); @@ -824,7 +835,7 @@ public void flush(String dbName, String tableName) throws IOException { writeLock.lock(); try { TableIdentifier tid = TableIdentifier.of(dbName, tableName); - TableMetadata tableMetadata = tableMetadataMap.getOrDefault(tid, new TableMetadata()); + TableMetadata tableMetadata = tableMetadataMap.getOrDefault(tid, new TableMetadata(this.conf)); if (tableMetadata.transaction.isPresent()) { Transaction transaction = tableMetadata.transaction.get(); Map props = tableMetadata.newProperties.or( @@ -839,7 +850,8 @@ public void flush(String dbName, String tableName) throws IOException { log.info("Sending audit counts for {} took {} ms", topicName, TimeUnit.NANOSECONDS.toMillis(context.stop())); } if (tableMetadata.completenessEnabled) { - checkAndUpdateCompletenessWatermark(tableMetadata, topicName, tableMetadata.datePartitions, props); + updateWatermarkWithFilesRegistered(topicName, tableMetadata, props, + tableMetadata.totalCountCompletenessEnabled); } } if (tableMetadata.deleteFiles.isPresent()) { @@ -849,16 +861,7 @@ public void flush(String dbName, String tableName) throws IOException { // The logic is to check the window [currentHour-1,currentHour] and update the watermark if there are no audit counts if(!tableMetadata.appendFiles.isPresent() && !tableMetadata.deleteFiles.isPresent() && tableMetadata.completenessEnabled) { - if (tableMetadata.completionWatermark > DEFAULT_COMPLETION_WATERMARK) { - log.info(String.format("Checking kafka audit for %s on change_property ", topicName)); - SortedSet timestamps = new TreeSet<>(); - ZonedDateTime dtAtBeginningOfHour = ZonedDateTime.now(ZoneId.of(this.timeZone)).truncatedTo(ChronoUnit.HOURS); - timestamps.add(dtAtBeginningOfHour); - checkAndUpdateCompletenessWatermark(tableMetadata, topicName, timestamps, props); - } else { - log.info(String.format("Need valid watermark, current watermark is %s, Not checking kafka audit for %s", - tableMetadata.completionWatermark, topicName)); - } + updateWatermarkWithNoFilesRegistered(topicName, tableMetadata, props); } //Set high waterMark @@ -909,94 +912,36 @@ public void flush(String dbName, String tableName) throws IOException { } } - @Override - public void reset(String dbName, String tableName) throws IOException { - this.tableMetadataMap.remove(TableIdentifier.of(dbName, tableName)); + private CompletenessWatermarkUpdater getWatermarkUpdater(String topicName, TableMetadata tableMetadata, + Map propsToUpdate) { + return new CompletenessWatermarkUpdater(topicName, this.auditCheckGranularity, this.timeZone, + tableMetadata, propsToUpdate, this.state, this.auditCountVerifier.get()); } - /** - * Update TableMetadata with the new completion watermark upon a successful audit check - * @param tableMetadata metadata of table - * @param topic topic name - * @param timestamps Sorted set in reverse order of timestamps to check audit counts for - * @param props table properties map - */ - private void checkAndUpdateCompletenessWatermark(TableMetadata tableMetadata, String topic, SortedSet timestamps, - Map props) { - String tableName = tableMetadata.table.get().name(); - if (topic == null) { - log.error(String.format("Not performing audit check. %s is null. Please set as table property of %s", - TOPIC_NAME_KEY, tableName)); - } - long newCompletenessWatermark = - computeCompletenessWatermark(tableName, topic, timestamps, tableMetadata.completionWatermark); - if (newCompletenessWatermark > tableMetadata.completionWatermark) { - log.info(String.format("Updating %s for %s to %s", COMPLETION_WATERMARK_KEY, tableMetadata.table.get().name(), - newCompletenessWatermark)); - props.put(COMPLETION_WATERMARK_KEY, String.valueOf(newCompletenessWatermark)); - props.put(COMPLETION_WATERMARK_TIMEZONE_KEY, this.timeZone); - tableMetadata.completionWatermark = newCompletenessWatermark; - } + private void updateWatermarkWithFilesRegistered(String topicName, TableMetadata tableMetadata, + Map propsToUpdate, boolean includeTotalCountCompletionWatermark) { + getWatermarkUpdater(topicName, tableMetadata, propsToUpdate) + .run(tableMetadata.datePartitions, includeTotalCountCompletionWatermark); } - /** - * NOTE: completion watermark for a window [t1, t2] is marked as t2 if audit counts match - * for that window (aka its is set to the beginning of next window) - * For each timestamp in sorted collection of timestamps in descending order - * if timestamp is greater than previousWatermark - * and hour(now) > hour(prevWatermark) - * check audit counts for completeness between - * a source and reference tier for [timestamp -1 , timstamp unit of granularity] - * If the audit count matches update the watermark to the timestamp and break - * else continue - * else - * break - * Using a {@link TimeIterator} that operates over a range of time in 1 unit - * given the start, end and granularity - * @param catalogDbTableName - * @param topicName - * @param timestamps a sorted set of timestamps in decreasing order - * @param previousWatermark previous completion watermark for the table - * @return updated completion watermark - */ - private long computeCompletenessWatermark(String catalogDbTableName, String topicName, SortedSet timestamps, long previousWatermark) { - log.info(String.format("Compute completion watermark for %s and timestamps %s with previous watermark %s", topicName, timestamps, previousWatermark)); - long completionWatermark = previousWatermark; - ZonedDateTime now = ZonedDateTime.now(ZoneId.of(this.timeZone)); - try { - if(timestamps == null || timestamps.size() <= 0) { - log.error("Cannot create time iterator. Empty for null timestamps"); - return previousWatermark; - } - TimeIterator.Granularity granularity = TimeIterator.Granularity.valueOf(this.auditCheckGranularity); - ZonedDateTime prevWatermarkDT = Instant.ofEpochMilli(previousWatermark) - .atZone(ZoneId.of(this.timeZone)); - ZonedDateTime startDT = timestamps.first(); - ZonedDateTime endDT = timestamps.last(); - TimeIterator iterator = new TimeIterator(startDT, endDT, granularity, true); - while (iterator.hasNext()) { - ZonedDateTime timestampDT = iterator.next(); - if (timestampDT.isAfter(prevWatermarkDT) - && TimeIterator.durationBetween(prevWatermarkDT, now, granularity) > 0) { - long timestampMillis = timestampDT.toInstant().toEpochMilli(); - ZonedDateTime auditCountCheckLowerBoundDT = TimeIterator.dec(timestampDT, granularity, 1); - if (auditCountVerifier.get().isComplete(topicName, - auditCountCheckLowerBoundDT.toInstant().toEpochMilli(), timestampMillis)) { - completionWatermark = timestampMillis; - // Also persist the watermark into State object to share this with other MetadataWriters - // we enforce ourselves to always use lower-cased table name here - String catalogDbTableNameLowerCased = catalogDbTableName.toLowerCase(Locale.ROOT); - this.state.setProp(String.format(STATE_COMPLETION_WATERMARK_KEY_OF_TABLE, catalogDbTableNameLowerCased), completionWatermark); - break; - } - } else { - break; - } - } - } catch (IOException e) { - log.warn("Exception during audit count check: ", e); + private void updateWatermarkWithNoFilesRegistered(String topicName, TableMetadata tableMetadata, + Map propsToUpdate) { + if (tableMetadata.completionWatermark > DEFAULT_COMPLETION_WATERMARK) { + log.info(String.format("Checking kafka audit for %s on change_property ", topicName)); + SortedSet timestamps = new TreeSet<>(); + ZonedDateTime dtAtBeginningOfHour = ZonedDateTime.now(ZoneId.of(this.timeZone)).truncatedTo(ChronoUnit.HOURS); + timestamps.add(dtAtBeginningOfHour); + + getWatermarkUpdater(topicName, tableMetadata, propsToUpdate).run(timestamps, true); + } else { + log.info(String.format("Need valid watermark, current watermark is %s, Not checking kafka audit for %s", + tableMetadata.completionWatermark, topicName)); } - return completionWatermark; + } + + @Override + public void reset(String dbName, String tableName) throws IOException { + this.tableMetadataMap.remove(TableIdentifier.of(dbName, tableName)); } private void submitSnapshotCommitEvent(Snapshot snapshot, TableMetadata tableMetadata, String dbName, @@ -1031,6 +976,10 @@ private void submitSnapshotCommitEvent(Snapshot snapshot, TableMetadata tableMet } if (tableMetadata.completenessEnabled) { gobblinTrackingEvent.addMetadata(COMPLETION_WATERMARK_KEY, Long.toString(tableMetadata.completionWatermark)); + if (tableMetadata.totalCountCompletenessEnabled) { + gobblinTrackingEvent.addMetadata(TOTAL_COUNT_COMPLETION_WATERMARK_KEY, + Long.toString(tableMetadata.totalCountCompletionWatermark)); + } } eventSubmitter.submit(gobblinTrackingEvent); } @@ -1109,7 +1058,7 @@ public void writeEnvelope(RecordEnvelope recordEnvelope, Map currentWatermark) { - if (!tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata()).lowWatermark.isPresent()) { + if (!tableMetadataMap.computeIfAbsent(tid, t -> new TableMetadata(this.conf)).lowWatermark.isPresent()) { //This means we haven't register this table or met some error before, we need to reset the low watermark tableMetadataMap.get(tid).lowWatermark = Optional.of(currentOffset - 1); tableMetadataMap.get(tid).setDatasetName(gmce.getDatasetIdentifier().getNativeName()); @@ -1117,6 +1066,11 @@ public void writeEnvelope(RecordEnvelope recordEnvelope, Map table = Optional.absent(); /** @@ -1175,18 +1129,18 @@ public class TableMetadata { Optional lastSchemaVersion = Optional.absent(); Optional lowWatermark = Optional.absent(); long completionWatermark = DEFAULT_COMPLETION_WATERMARK; + long totalCountCompletionWatermark = DEFAULT_COMPLETION_WATERMARK; SortedSet datePartitions = new TreeSet<>(Collections.reverseOrder()); List serializedAuditCountMaps = new ArrayList<>(); @Setter String datasetName; boolean completenessEnabled; + boolean totalCountCompletenessEnabled; boolean newPartitionColumnEnabled; + Configuration conf; - Cache addedFiles = CacheBuilder.newBuilder() - .expireAfterAccess(conf.getInt(ADDED_FILES_CACHE_EXPIRING_TIME, DEFAULT_ADDED_FILES_CACHE_EXPIRING_TIME), - TimeUnit.HOURS) - .build(); + Cache addedFiles; long lowestGMCEEmittedTime = Long.MAX_VALUE; /** @@ -1240,5 +1194,13 @@ void reset(Map props, Long lowWaterMark) { this.datePartitions.clear(); this.serializedAuditCountMaps.clear(); } + + TableMetadata(Configuration conf) { + this.conf = conf; + addedFiles = CacheBuilder.newBuilder() + .expireAfterAccess(this.conf.getInt(ADDED_FILES_CACHE_EXPIRING_TIME, DEFAULT_ADDED_FILES_CACHE_EXPIRING_TIME), + TimeUnit.HOURS) + .build(); + } } } diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterConfigKeys.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterConfigKeys.java index 73c206d5f3d..6d4b7b629ff 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterConfigKeys.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterConfigKeys.java @@ -21,9 +21,14 @@ public class IcebergMetadataWriterConfigKeys { public static final String ICEBERG_COMPLETENESS_ENABLED = "iceberg.completeness.enabled"; public static final boolean DEFAULT_ICEBERG_COMPLETENESS = false; + public static final String ICEBERG_TOTAL_COUNT_COMPLETENESS_ENABLED = "iceberg.completeness.totalCount.enabled"; + public static final boolean DEFAULT_ICEBERG_TOTAL_COUNT_COMPLETENESS = false; public static final String ICEBERG_COMPLETENESS_WHITELIST = "iceberg.completeness.whitelist"; + public static final String ICEBERG_TOTAL_COUNT_COMPLETENESS_WHITELIST = "iceberg.totalCount.completeness.whitelist"; public static final String ICEBERG_COMPLETENESS_BLACKLIST = "iceberg.completeness.blacklist"; + public static final String ICEBERG_TOTAL_COUNT_COMPLETENESS_BLACKLIST = "iceberg.totalCount.completeness.blacklist"; public static final String COMPLETION_WATERMARK_KEY = "completionWatermark"; + public static final String TOTAL_COUNT_COMPLETION_WATERMARK_KEY = "totalCountCompletionWatermark"; public static final String COMPLETION_WATERMARK_TIMEZONE_KEY = "completionWatermarkTimezone"; public static final long DEFAULT_COMPLETION_WATERMARK = -1L; public static final String TIME_ZONE_KEY = "iceberg.completeness.timezone"; @@ -41,8 +46,7 @@ public class IcebergMetadataWriterConfigKeys { public static final String ICEBERG_NEW_PARTITION_WHITELIST = "iceberg.new.partition.whitelist"; public static final String ICEBERG_NEW_PARTITION_BLACKLIST = "iceberg.new.partition.blacklist"; public static final String STATE_COMPLETION_WATERMARK_KEY_OF_TABLE = "completion.watermark.%s"; + public static final String STATE_TOTAL_COUNT_COMPLETION_WATERMARK_KEY_OF_TABLE = "totalCount.completion.watermark.%s"; public static final String ICEBERG_ENABLE_CUSTOM_METADATA_RETENTION_POLICY = "iceberg.enable.custom.metadata.retention.policy"; public static final boolean DEFAULT_ICEBERG_ENABLE_CUSTOM_METADATA_RETENTION_POLICY = true; - - } diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/CompletenessWatermarkUpdaterTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/CompletenessWatermarkUpdaterTest.java new file mode 100644 index 00000000000..0b784873376 --- /dev/null +++ b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/CompletenessWatermarkUpdaterTest.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.gobblin.iceberg.writer; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.SortedSet; +import java.util.TreeSet; +import org.apache.gobblin.completeness.verifier.KafkaAuditCountVerifier; +import org.apache.gobblin.configuration.State; +import org.apache.gobblin.time.TimeIterator; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.Table; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static org.apache.gobblin.iceberg.writer.IcebergMetadataWriterConfigKeys.*; +import static org.mockito.Mockito.*; + + +public class CompletenessWatermarkUpdaterTest { + static final String TOPIC = "testTopic"; + static final String TABLE_NAME = "testTopic_tableName"; + static final String TIME_ZONE = "America/Los_Angeles"; + static final String AUDIT_CHECK_GRANULARITY = "HOUR"; + + static final ZonedDateTime NOW = ZonedDateTime.now(ZoneId.of(TIME_ZONE)).truncatedTo(ChronoUnit.HOURS); + static final ZonedDateTime ONE_HOUR_AGO = TimeIterator.dec(NOW, TimeIterator.Granularity.valueOf(AUDIT_CHECK_GRANULARITY), 1); + static final ZonedDateTime TWO_HOUR_AGO = TimeIterator.dec(NOW, TimeIterator.Granularity.valueOf(AUDIT_CHECK_GRANULARITY), 2); + static final ZonedDateTime THREE_HOUR_AGO = TimeIterator.dec(NOW, TimeIterator.Granularity.valueOf(AUDIT_CHECK_GRANULARITY), 3); + + @Test + public void testClassicWatermarkOnly() throws IOException { + TestParams params = createTestParams(); + + // Round 1: the expected completion watermark bootstraps to ONE_HOUR_AGO + // total completion watermark is not enabled + KafkaAuditCountVerifier verifier = mockKafkaAuditCountVerifier(ImmutableList.of( + new AuditCountVerificationResult(TWO_HOUR_AGO, ONE_HOUR_AGO, true /* isCompleteClassic */, false /* isCompleteTotalCount */), + new AuditCountVerificationResult(THREE_HOUR_AGO, TWO_HOUR_AGO, true, true))); + CompletenessWatermarkUpdater updater = + new CompletenessWatermarkUpdater("testTopic", AUDIT_CHECK_GRANULARITY, TIME_ZONE, params.tableMetadata, params.props, params.state, verifier); + SortedSet timestamps = new TreeSet<>(Collections.reverseOrder()); + timestamps.add(ONE_HOUR_AGO); + timestamps.add(TWO_HOUR_AGO); + boolean includeTotalCountCompletionWatermark = false; + updater.run(timestamps, includeTotalCountCompletionWatermark); + + validateCompletionWaterMark(ONE_HOUR_AGO, params); + validateEmptyTotalCompletionWatermark(params); + + // Round 2: the expected completion watermark moves from ONE_HOUR_AGO to NOW + // total completion watermark is not enabled + verifier = mockKafkaAuditCountVerifier(ImmutableList.of( + new AuditCountVerificationResult(ONE_HOUR_AGO, NOW, true /* isCompleteClassic */, false /* isCompleteTotalCount */), + new AuditCountVerificationResult(TWO_HOUR_AGO, ONE_HOUR_AGO, true, true), + new AuditCountVerificationResult(THREE_HOUR_AGO, TWO_HOUR_AGO, true, true))); + updater.setAuditCountVerifier(verifier); + timestamps.add(NOW); + updater.run(timestamps, includeTotalCountCompletionWatermark); + + validateCompletionWaterMark(NOW, params); + validateEmptyTotalCompletionWatermark(params); + } + + @Test + public void testClassicAndTotalCountWatermark() throws IOException { + TestParams params = createTestParams(); + + // Round 1: the expected completion watermark bootstraps to ONE_HOUR_AGO + // the expected total completion watermark bootstraps to TOW_HOUR_AGO + KafkaAuditCountVerifier verifier = mockKafkaAuditCountVerifier(ImmutableList.of( + new AuditCountVerificationResult(TWO_HOUR_AGO, ONE_HOUR_AGO, true /* isCompleteClassic */, false /* isCompleteTotalCount */), + new AuditCountVerificationResult(THREE_HOUR_AGO, TWO_HOUR_AGO, true, true))); + CompletenessWatermarkUpdater updater = + new CompletenessWatermarkUpdater("testTopic", AUDIT_CHECK_GRANULARITY, TIME_ZONE, params.tableMetadata, params.props, params.state, verifier); + SortedSet timestamps = new TreeSet<>(Collections.reverseOrder()); + timestamps.add(ONE_HOUR_AGO); + timestamps.add(TWO_HOUR_AGO); + boolean includeTotalCountCompletionWatermark = true; + updater.run(timestamps, includeTotalCountCompletionWatermark); + + validateCompletionWaterMark(ONE_HOUR_AGO, params); + validateTotalCompletionWatermark(TWO_HOUR_AGO, params); + + // Round 2: the expected completion watermark moves from ONE_HOUR_AGO to NOW + // the expected total completion watermark moves from TOW_HOUR_AGO to ONE_HOUR_AGO + verifier = mockKafkaAuditCountVerifier(ImmutableList.of( + new AuditCountVerificationResult(ONE_HOUR_AGO, NOW, true /* isCompleteClassic */, false /* isCompleteTotalCount */), + new AuditCountVerificationResult(TWO_HOUR_AGO, ONE_HOUR_AGO, true, true), + new AuditCountVerificationResult(THREE_HOUR_AGO, TWO_HOUR_AGO, true, true))); + updater.setAuditCountVerifier(verifier); + timestamps.add(NOW); + updater.run(timestamps, includeTotalCountCompletionWatermark); + + validateCompletionWaterMark(NOW, params); + validateTotalCompletionWatermark(ONE_HOUR_AGO, params); +} + + static void validateCompletionWaterMark(ZonedDateTime expectedDT, TestParams params) { + long expected = expectedDT.toInstant().toEpochMilli(); + + // 1. assert updated tableMetadata.completionWatermark + Assert.assertEquals(params.tableMetadata.completionWatermark, expected); + // 2. assert updated property + Assert.assertEquals(params.props.get(COMPLETION_WATERMARK_KEY), String.valueOf(expected)); + Assert.assertEquals(params.props.get(COMPLETION_WATERMARK_TIMEZONE_KEY), TIME_ZONE); + // 3. assert updated state + String watermarkKey = String.format(STATE_COMPLETION_WATERMARK_KEY_OF_TABLE, + params.tableMetadata.table.get().name().toLowerCase(Locale.ROOT)); + Assert.assertEquals(params.state.getProp(watermarkKey), String.valueOf(expected)); + } + + static void validateTotalCompletionWatermark(ZonedDateTime expectedDT, TestParams params) { + long expected = expectedDT.toInstant().toEpochMilli(); + + // 1. expect updated tableMetadata.totalCountCompletionWatermark + Assert.assertEquals(params.tableMetadata.totalCountCompletionWatermark, expected); + // 2. expect updated property + Assert.assertEquals(params.props.get(TOTAL_COUNT_COMPLETION_WATERMARK_KEY), String.valueOf(expected)); + // 3. expect updated state + String totalCountWatermarkKey = String.format(STATE_TOTAL_COUNT_COMPLETION_WATERMARK_KEY_OF_TABLE, + params.tableMetadata.table.get().name().toLowerCase(Locale.ROOT)); + Assert.assertEquals(params.state.getProp(totalCountWatermarkKey), String.valueOf(expected)); + } + + static void validateEmptyTotalCompletionWatermark(TestParams params) { + Assert.assertEquals(params.tableMetadata.totalCountCompletionWatermark, DEFAULT_COMPLETION_WATERMARK); + Assert.assertNull(params.props.get(TOTAL_COUNT_COMPLETION_WATERMARK_KEY)); + String totalCountWatermarkKey = String.format(STATE_TOTAL_COUNT_COMPLETION_WATERMARK_KEY_OF_TABLE, + params.tableMetadata.table.get().name().toLowerCase(Locale.ROOT)); + Assert.assertNull(params.state.getProp(totalCountWatermarkKey)); + } + + static class TestParams { + IcebergMetadataWriter.TableMetadata tableMetadata; + Map props; + State state; + } + + static TestParams createTestParams() throws IOException { + TestParams params = new TestParams(); + params.tableMetadata = new IcebergMetadataWriter.TableMetadata(new Configuration()); + + Table table = mock(Table.class); + when(table.name()).thenReturn(TABLE_NAME); + params.tableMetadata.table = Optional.of(table); + + params.props = new HashMap<>(); + params.state = new State(); + + return params; + } + + static class AuditCountVerificationResult { + AuditCountVerificationResult(ZonedDateTime start, ZonedDateTime end, boolean isCompleteClassic, boolean isCompleteTotalCount) { + this.start = start; + this.end = end; + this.isCompleteClassic = isCompleteClassic; + this.isCompleteTotalCount = isCompleteTotalCount; + } + ZonedDateTime start; + ZonedDateTime end; + boolean isCompleteClassic; + boolean isCompleteTotalCount; + } + + static KafkaAuditCountVerifier mockKafkaAuditCountVerifier(List resultsToMock) + throws IOException { + KafkaAuditCountVerifier verifier = mock(IcebergMetadataWriterTest.TestAuditCountVerifier.class); + for (AuditCountVerificationResult result : resultsToMock) { + Mockito.when(verifier.calculateCompleteness(TOPIC, result.start.toInstant().toEpochMilli(), result.end.toInstant().toEpochMilli())) + .thenReturn(ImmutableMap.of( + KafkaAuditCountVerifier.CompletenessType.ClassicCompleteness, result.isCompleteClassic, + KafkaAuditCountVerifier.CompletenessType.TotalCountCompleteness, result.isCompleteTotalCount)); + } + return verifier; + } +} diff --git a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java index e5819d6baab..6feb8adae0c 100644 --- a/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java +++ b/gobblin-iceberg/src/test/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriterTest.java @@ -194,6 +194,7 @@ private State getStateWithCompletenessConfig() { State state = getState(); state.setProp(ICEBERG_NEW_PARTITION_ENABLED, true); state.setProp(ICEBERG_COMPLETENESS_ENABLED, true); + state.setProp(ICEBERG_TOTAL_COUNT_COMPLETENESS_ENABLED, true); state.setProp(NEW_PARTITION_KEY, "late"); state.setProp(NEW_PARTITION_TYPE_KEY, "int"); state.setProp(AuditCountClientFactory.AUDIT_COUNT_CLIENT_FACTORY, TestAuditClientFactory.class.getName()); @@ -223,6 +224,8 @@ public void testWriteAddFileGMCE() throws IOException { Assert.assertFalse(table.properties().containsKey("offset.range.testTopic-1")); Assert.assertEquals(table.location(), new File(tmpDir, "testDB/testTopic/_iceberg_metadata/").getAbsolutePath() + "/" + dbName); + Assert.assertFalse(table.properties().containsKey(COMPLETION_WATERMARK_KEY)); + Assert.assertFalse(table.properties().containsKey(TOTAL_COUNT_COMPLETION_WATERMARK_KEY)); gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopic-1", "1000-2000").build()); GenericRecord genericGmce_1000_2000 = GenericData.get().deepCopy(gmce.getSchema(), gmce); @@ -238,6 +241,8 @@ public void testWriteAddFileGMCE() throws IOException { // Assert low watermark and high watermark set properly Assert.assertEquals(table.properties().get("gmce.low.watermark.GobblinMetadataChangeEvent_test-1"), "9"); Assert.assertEquals(table.properties().get("gmce.high.watermark.GobblinMetadataChangeEvent_test-1"), "20"); + Assert.assertFalse(table.properties().containsKey(COMPLETION_WATERMARK_KEY)); + Assert.assertFalse(table.properties().containsKey(TOTAL_COUNT_COMPLETION_WATERMARK_KEY)); /*test flush twice*/ gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopic-1", "2000-3000").build()); @@ -257,6 +262,8 @@ public void testWriteAddFileGMCE() throws IOException { Assert.assertEquals(table.currentSnapshot().allManifests(table.io()).size(), 2); Assert.assertEquals(table.properties().get("gmce.low.watermark.GobblinMetadataChangeEvent_test-1"), "20"); Assert.assertEquals(table.properties().get("gmce.high.watermark.GobblinMetadataChangeEvent_test-1"), "30"); + Assert.assertFalse(table.properties().containsKey(COMPLETION_WATERMARK_KEY)); + Assert.assertFalse(table.properties().containsKey(TOTAL_COUNT_COMPLETION_WATERMARK_KEY)); /* Test it will skip event with lower watermark*/ gmce.setTopicPartitionOffsetsRange(ImmutableMap.builder().put("testTopic-1", "3000-4000").build()); @@ -268,6 +275,8 @@ public void testWriteAddFileGMCE() throws IOException { table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(0)); Assert.assertEquals(table.properties().get("offset.range.testTopic-1"), "0-3000"); Assert.assertEquals(table.currentSnapshot().allManifests(table.io()).size(), 2); + Assert.assertFalse(table.properties().containsKey(COMPLETION_WATERMARK_KEY)); + Assert.assertFalse(table.properties().containsKey(TOTAL_COUNT_COMPLETION_WATERMARK_KEY)); } //Make sure hive test execute later and close the metastore @@ -420,7 +429,9 @@ public void testWriteAddFileGMCECompleteness() throws IOException { // Test when completeness watermark = -1 bootstrap case KafkaAuditCountVerifier verifier = Mockito.mock(TestAuditCountVerifier.class); - Mockito.when(verifier.isComplete("testTopicCompleteness", timestampMillis - TimeUnit.HOURS.toMillis(1), timestampMillis)).thenReturn(true); + Mockito.when(verifier.calculateCompleteness("testTopicCompleteness", timestampMillis - TimeUnit.HOURS.toMillis(1), timestampMillis)) + .thenReturn(ImmutableMap.of(KafkaAuditCountVerifier.CompletenessType.ClassicCompleteness, true, + KafkaAuditCountVerifier.CompletenessType.TotalCountCompleteness, true)); IcebergMetadataWriter imw = (IcebergMetadataWriter) gobblinMCEWriterWithCompletness.metadataWriters.iterator().next(); imw.setAuditCountVerifier(verifier); gobblinMCEWriterWithCompletness.flush(); @@ -429,8 +440,10 @@ public void testWriteAddFileGMCECompleteness() throws IOException { Assert.assertEquals(table.properties().get(TOPIC_NAME_KEY), "testTopic"); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_TIMEZONE_KEY), "America/Los_Angeles"); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_KEY), String.valueOf(timestampMillis)); + Assert.assertEquals(table.properties().get(TOTAL_COUNT_COMPLETION_WATERMARK_KEY), String.valueOf(timestampMillis)); // 1631811600000L correspond to 2020-09-16-10 in PT Assert.assertEquals(imw.state.getPropAsLong(String.format(STATE_COMPLETION_WATERMARK_KEY_OF_TABLE, table.name().toLowerCase(Locale.ROOT))), 1631811600000L); + Assert.assertEquals(imw.state.getPropAsLong(String.format(STATE_TOTAL_COUNT_COMPLETION_WATERMARK_KEY_OF_TABLE, table.name().toLowerCase(Locale.ROOT))), 1631811600000L); Iterator dfl = FindFiles.in(table).withMetadataMatching(Expressions.startsWith("file_path", hourlyFile.getAbsolutePath())).collect().iterator(); Assert.assertTrue(dfl.hasNext()); @@ -453,6 +466,7 @@ public void testWriteAddFileGMCECompleteness() throws IOException { gobblinMCEWriterWithCompletness.flush(); table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(1)); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_KEY), String.valueOf(timestampMillis)); + Assert.assertEquals(table.properties().get(TOTAL_COUNT_COMPLETION_WATERMARK_KEY), String.valueOf(timestampMillis)); dfl = FindFiles.in(table).withMetadataMatching(Expressions.startsWith("file_path", hourlyFile1.getAbsolutePath())).collect().iterator(); Assert.assertTrue(dfl.hasNext()); @@ -475,12 +489,17 @@ public void testWriteAddFileGMCECompleteness() throws IOException { new KafkaPartition.Builder().withTopicName("GobblinMetadataChangeEvent_test").withId(1).build(), new LongWatermark(60L)))); - Mockito.when(verifier.isComplete("testTopicCompleteness", timestampMillis1 - TimeUnit.HOURS.toMillis(1), timestampMillis1)).thenReturn(true); + Mockito.when(verifier.calculateCompleteness("testTopicCompleteness", timestampMillis1 - TimeUnit.HOURS.toMillis(1), timestampMillis1)) + .thenReturn(ImmutableMap.of(KafkaAuditCountVerifier.CompletenessType.ClassicCompleteness, true, + KafkaAuditCountVerifier.CompletenessType.TotalCountCompleteness, true)); + gobblinMCEWriterWithCompletness.flush(); table = catalog.loadTable(catalog.listTables(Namespace.of(dbName)).get(1)); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_KEY), String.valueOf(timestampMillis1)); + Assert.assertEquals(table.properties().get(TOTAL_COUNT_COMPLETION_WATERMARK_KEY), String.valueOf(timestampMillis1)); // watermark 1631815200000L correspond to 2021-09-16-11 in PT Assert.assertEquals(imw.state.getPropAsLong(String.format(STATE_COMPLETION_WATERMARK_KEY_OF_TABLE, table.name().toLowerCase(Locale.ROOT))), 1631815200000L); + Assert.assertEquals(imw.state.getPropAsLong(String.format(STATE_TOTAL_COUNT_COMPLETION_WATERMARK_KEY_OF_TABLE, table.name().toLowerCase(Locale.ROOT))), 1631815200000L); dfl = FindFiles.in(table).withMetadataMatching(Expressions.startsWith("file_path", hourlyFile2.getAbsolutePath())).collect().iterator(); Assert.assertTrue(dfl.hasNext()); @@ -511,7 +530,10 @@ public void testChangePropertyGMCECompleteness() throws IOException { KafkaAuditCountVerifier verifier = Mockito.mock(TestAuditCountVerifier.class); // For quiet topics always check for previous hour window - Mockito.when(verifier.isComplete("testTopicCompleteness", expectedCWDt.minusHours(1).toInstant().toEpochMilli(), expectedWatermark)).thenReturn(true); + Mockito.when(verifier.calculateCompleteness("testTopicCompleteness", expectedCWDt.minusHours(1).toInstant().toEpochMilli(), expectedWatermark)) + .thenReturn(ImmutableMap.of(KafkaAuditCountVerifier.CompletenessType.ClassicCompleteness, true, + KafkaAuditCountVerifier.CompletenessType.TotalCountCompleteness, true)); + ((IcebergMetadataWriter) gobblinMCEWriterWithCompletness.metadataWriters.iterator().next()).setAuditCountVerifier(verifier); gobblinMCEWriterWithCompletness.flush(); @@ -521,7 +543,7 @@ public void testChangePropertyGMCECompleteness() throws IOException { Assert.assertEquals(table.properties().get(TOPIC_NAME_KEY), "testTopic"); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_TIMEZONE_KEY), "America/Los_Angeles"); Assert.assertEquals(table.properties().get(COMPLETION_WATERMARK_KEY), String.valueOf(expectedWatermark)); - + Assert.assertEquals(table.properties().get(TOTAL_COUNT_COMPLETION_WATERMARK_KEY), String.valueOf(expectedWatermark)); } private String writeRecord(File file) throws IOException { From 68e25904af3ec0f07bf4099208b18678aaba038c Mon Sep 17 00:00:00 2001 From: umustafi Date: Tue, 25 Jul 2023 10:24:00 -0700 Subject: [PATCH 29/30] [GOBBLIN-1858] Fix logs relating to multi-active lease arbiter (#3720) Co-authored-by: Urmi Mustafi --- .../gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java | 4 ++-- .../service/modules/orchestration/FlowTriggerHandler.java | 4 ++-- .../gobblin/service/modules/orchestration/Orchestrator.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java index 9318d8b56f6..88f143a3b22 100644 --- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java +++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/api/MysqlMultiActiveLeaseArbiter.java @@ -450,13 +450,13 @@ public boolean recordLeaseSuccess(LeaseObtainedStatus status) updateStatement.setTimestamp(++i, new Timestamp(status.getLeaseAcquisitionTimestamp())); int numRowsUpdated = updateStatement.executeUpdate(); if (numRowsUpdated == 0) { - log.info("Multi-active lease arbiter lease attempt: [%s, eventTimestamp: %s] - FAILED to complete because " + log.info("Multi-active lease arbiter lease attempt: [{}, eventTimestamp: {}] - FAILED to complete because " + "lease expired or event cleaned up before host completed required actions", flowAction, status.getEventTimestamp()); return false; } if( numRowsUpdated == 1) { - log.info("Multi-active lease arbiter lease attempt: [%s, eventTimestamp: %s] - COMPLETED, no longer leasing" + log.info("Multi-active lease arbiter lease attempt: [{}, eventTimestamp: {}] - COMPLETED, no longer leasing" + " this event after this.", flowAction, status.getEventTimestamp()); return true; }; diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java index 7fdee038414..90379e730ff 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/FlowTriggerHandler.java @@ -181,11 +181,11 @@ private void scheduleReminderForEvent(Properties jobProps, MultiActiveLeaseArbit // Create a new trigger for the flow in job scheduler that is set to fire at the minimum reminder wait time calculated Trigger trigger = JobScheduler.createTriggerForJob(key, jobProps); try { - log.info("Flow Trigger Handler - [%s, eventTimestamp: %s] - attempting to schedule reminder for event %s in %s millis", + log.info("Flow Trigger Handler - [{}, eventTimestamp: {}] - attempting to schedule reminder for event {} in {} millis", flowAction, originalEventTimeMillis, status.getEventTimeMillis(), trigger.getNextFireTime()); this.schedulerService.getScheduler().scheduleJob(trigger); } catch (SchedulerException e) { - log.warn("Failed to add job reminder due to SchedulerException for job %s trigger event %s ", key, status.getEventTimeMillis(), e); + log.warn("Failed to add job reminder due to SchedulerException for job {} trigger event {} ", key, status.getEventTimeMillis(), e); } log.info(String.format("Flow Trigger Handler - [%s, eventTimestamp: %s] - SCHEDULED REMINDER for event %s in %s millis", flowAction, originalEventTimeMillis, status.getEventTimeMillis(), trigger.getNextFireTime())); diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java index a0c19678955..84521087bfd 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/orchestration/Orchestrator.java @@ -329,7 +329,7 @@ public void orchestrate(Spec spec, Properties jobProps, long triggerTimestampMil DagActionStore.DagAction flowAction = new DagActionStore.DagAction(flowGroup, flowName, flowExecutionId, DagActionStore.FlowActionType.LAUNCH); flowTriggerHandler.get().handleTriggerEvent(jobProps, flowAction, triggerTimestampMillis); - _log.info("Multi-active scheduler finished handling trigger event: [%s, triggerEventTimestamp: %s]", flowAction, + _log.info("Multi-active scheduler finished handling trigger event: [{}, triggerEventTimestamp: {}]", flowAction, triggerTimestampMillis); } else if (this.dagManager.isPresent()) { submitFlowToDagManager((FlowSpec) spec, jobExecutionPlanDag); From 8198404ecacc919eefabf22307ab8abf1b97e885 Mon Sep 17 00:00:00 2001 From: Jack Moseley Date: Wed, 26 Jul 2023 12:29:09 -0700 Subject: [PATCH 30/30] Fix bug with total count watermark whitelist (#3724) --- .../gobblin/iceberg/writer/IcebergMetadataWriter.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java index 19bc805eb6a..d614e25240c 100644 --- a/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java +++ b/gobblin-iceberg/src/main/java/org/apache/gobblin/iceberg/writer/IcebergMetadataWriter.java @@ -861,7 +861,8 @@ public void flush(String dbName, String tableName) throws IOException { // The logic is to check the window [currentHour-1,currentHour] and update the watermark if there are no audit counts if(!tableMetadata.appendFiles.isPresent() && !tableMetadata.deleteFiles.isPresent() && tableMetadata.completenessEnabled) { - updateWatermarkWithNoFilesRegistered(topicName, tableMetadata, props); + updateWatermarkWithNoFilesRegistered(topicName, tableMetadata, props, + tableMetadata.totalCountCompletenessEnabled); } //Set high waterMark @@ -925,14 +926,14 @@ private void updateWatermarkWithFilesRegistered(String topicName, TableMetadata } private void updateWatermarkWithNoFilesRegistered(String topicName, TableMetadata tableMetadata, - Map propsToUpdate) { + Map propsToUpdate, boolean includeTotalCountCompletionWatermark) { if (tableMetadata.completionWatermark > DEFAULT_COMPLETION_WATERMARK) { log.info(String.format("Checking kafka audit for %s on change_property ", topicName)); SortedSet timestamps = new TreeSet<>(); ZonedDateTime dtAtBeginningOfHour = ZonedDateTime.now(ZoneId.of(this.timeZone)).truncatedTo(ChronoUnit.HOURS); timestamps.add(dtAtBeginningOfHour); - getWatermarkUpdater(topicName, tableMetadata, propsToUpdate).run(timestamps, true); + getWatermarkUpdater(topicName, tableMetadata, propsToUpdate).run(timestamps, includeTotalCountCompletionWatermark); } else { log.info(String.format("Need valid watermark, current watermark is %s, Not checking kafka audit for %s", tableMetadata.completionWatermark, topicName));