From 13b6291e7b32dcebdd65884e3c682634370f83fb Mon Sep 17 00:00:00 2001 From: jsonwan Date: Thu, 26 Dec 2024 17:53:37 +0800 Subject: [PATCH 1/3] =?UTF-8?q?perf:=20=E5=AE=9A=E6=97=B6=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E6=9C=8D=E5=8A=A1=E6=94=AF=E6=8C=81=E4=BC=98=E9=9B=85?= =?UTF-8?q?=E5=81=9C=E6=9C=BA=20#2852?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 操作Quartz引擎前先检查其关闭状态。 --- .../handler/DefaultQuartzTaskHandler.java | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/handler/DefaultQuartzTaskHandler.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/handler/DefaultQuartzTaskHandler.java index 598cc31e41..890c888418 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/handler/DefaultQuartzTaskHandler.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/handler/DefaultQuartzTaskHandler.java @@ -26,18 +26,25 @@ import com.tencent.bk.job.crontab.timer.AbstractQuartzTaskHandler; import com.tencent.bk.job.crontab.timer.QuartzJob; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.quartz.*; +import org.quartz.JobDetail; +import org.quartz.JobKey; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.Trigger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.Assert; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * QuartzTaskHandler 的默认实现 **/ +@Slf4j @Component public class DefaultQuartzTaskHandler extends AbstractQuartzTaskHandler { @@ -60,6 +67,11 @@ public void addJob(QuartzJob quartzJob) throws SchedulerException { Set triggers = createTriggers(quartzJob); + if (scheduler.isShutdown()) { + log.info("scheduler is shutdown, ignore add job {}!", quartzJob.getKey().getName()); + return; + } + if (CollectionUtils.isEmpty(triggers)) { this.scheduler.addJob(jobDetail, false); } else { @@ -84,6 +96,11 @@ public void deleteJob(JobKey jobKey) throws SchedulerException { Assert.notNull(jobKey, "jobKey cannot be empty!"); Assert.notNull(jobKey.getName(), "jobKey name cannot be empty!"); + if (scheduler.isShutdown()) { + log.info("scheduler is shutdown, ignore delete job {}!", jobKey.getName()); + return; + } + this.scheduler.deleteJob(jobKey); } @@ -91,6 +108,15 @@ public void deleteJob(JobKey jobKey) throws SchedulerException { public void deleteJob(List jobKeys) throws SchedulerException { Assert.notNull(jobKeys, "jobKeys cannot be empty!"); + if (scheduler.isShutdown()) { + log.info( + "scheduler is shutdown, ignore delete {} job keys: {}", + jobKeys.size(), + jobKeys.stream().map(JobKey::getName).collect(Collectors.toList()) + ); + return; + } + this.scheduler.deleteJobs(jobKeys); } @@ -99,6 +125,10 @@ public void deleteJob(List jobKeys) throws SchedulerException { */ @Override public void pauseAll() throws SchedulerException { + if (scheduler.isShutdown()) { + log.info("scheduler is shutdown, ignore pauseAll!"); + return; + } this.scheduler.pauseAll(); } } From 003c6fa53be5974fc866996618cc78248c01f10e Mon Sep 17 00:00:00 2001 From: jsonwan Date: Thu, 26 Dec 2024 18:21:04 +0800 Subject: [PATCH 2/3] =?UTF-8?q?perf:=20=E5=AE=9A=E6=97=B6=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E6=9C=8D=E5=8A=A1=E6=94=AF=E6=8C=81=E4=BC=98=E9=9B=85?= =?UTF-8?q?=E5=81=9C=E6=9C=BA=20#2852?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 批量加载定时任务到Quartz支持被中断； 2. 进程关闭时取消批量加载任务。 --- .../bk/job/crontab/runner/LoadCronJobRunner.java | 14 +++++++++++++- .../crontab/service/CronJobBatchLoadService.java | 2 +- .../service/impl/CronJobBatchLoadServiceImpl.java | 9 ++++++++- .../service/impl/CronJobLoadingServiceImpl.java | 4 +++- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java index 615fe5798a..3d3a20f25d 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java @@ -31,6 +31,8 @@ import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Component; +import javax.annotation.PreDestroy; +import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; /** @@ -43,6 +45,8 @@ public class LoadCronJobRunner implements CommandLineRunner { private final CronJobLoadingService cronJobLoadingService; private final ThreadPoolExecutor crontabInitRunnerExecutor; + private Future loadCronJobFuture; + @Autowired public LoadCronJobRunner(CronJobLoadingService cronJobLoadingService, @Qualifier("crontabInitRunnerExecutor") ThreadPoolExecutor crontabInitRunnerExecutor) { @@ -52,9 +56,17 @@ public LoadCronJobRunner(CronJobLoadingService cronJobLoadingService, @Override public void run(String... args) { - crontabInitRunnerExecutor.submit(() -> { + loadCronJobFuture = crontabInitRunnerExecutor.submit(() -> { log.info("loadCronToQuartzOnStartup"); cronJobLoadingService.loadAllCronJob(); }); } + + @PreDestroy + public void destroy() { + if (loadCronJobFuture != null) { + boolean result = loadCronJobFuture.cancel(true); + log.info("loadCronJobFuture cancel result:{}", result); + } + } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobBatchLoadService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobBatchLoadService.java index 5b83cc01b3..d07b14bfbe 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobBatchLoadService.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobBatchLoadService.java @@ -38,7 +38,7 @@ public interface CronJobBatchLoadService { * @param limit 一批定时任务的数量 * @return 加载结果数据 */ - CronLoadResult batchLoadCronToQuartz(int start, int limit); + CronLoadResult batchLoadCronToQuartz(int start, int limit) throws InterruptedException; @Data class CronLoadResult { diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobBatchLoadServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobBatchLoadServiceImpl.java index 6b06290653..23d60d9253 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobBatchLoadServiceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobBatchLoadServiceImpl.java @@ -49,12 +49,13 @@ public CronJobBatchLoadServiceImpl(CronJobService cronJobService) { @Override @JobTransactional(transactionManager = "jobCrontabTransactionManager", timeout = 30) - public CronLoadResult batchLoadCronToQuartz(int start, int limit) { + public CronLoadResult batchLoadCronToQuartz(int start, int limit) throws InterruptedException { int successNum = 0; int failedNum = 0; List failedCronList = new ArrayList<>(); List cronJobBasicInfoList = cronJobService.listEnabledCronBasicInfoForUpdate(start, limit); for (CronJobBasicInfoDTO cronJobBasicInfoDTO : cronJobBasicInfoList) { + checkInterrupt(); boolean result = false; try { result = cronJobService.addJobToQuartz( @@ -93,4 +94,10 @@ public CronLoadResult batchLoadCronToQuartz(int start, int limit) { cronLoadResult.setFailedCronList(failedCronList); return cronLoadResult; } + + private void checkInterrupt() throws InterruptedException { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException("batchLoadCronToQuartz thread is interrupted, exit"); + } + } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobLoadingServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobLoadingServiceImpl.java index 373f448616..e0cf7b5b12 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobLoadingServiceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobLoadingServiceImpl.java @@ -57,6 +57,8 @@ public void loadAllCronJob() { } loadingCronToQuartz = true; loadAllCronJobToQuartz(); + } catch (InterruptedException e) { + log.info("loadAllCronJob interrupted, application may be closing"); } catch (Exception e) { log.warn("Fail to loadAllCronJob", e); } finally { @@ -65,7 +67,7 @@ public void loadAllCronJob() { } } - private void loadAllCronJobToQuartz() { + private void loadAllCronJobToQuartz() throws InterruptedException { int start = 0; int limit = 100; int currentFetchNum; From 251e26160bf96b16bf58987c80b157f8ee1aeeaf Mon Sep 17 00:00:00 2001 From: jsonwan Date: Thu, 26 Dec 2024 21:48:21 +0800 Subject: [PATCH 3/3] =?UTF-8?q?perf:=20=E5=AE=9A=E6=97=B6=E4=BB=BB?= =?UTF-8?q?=E5=8A=A1=E6=9C=8D=E5=8A=A1=E6=94=AF=E6=8C=81=E4=BC=98=E9=9B=85?= =?UTF-8?q?=E5=81=9C=E6=9C=BA=20#2852?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. 使用Spring Bean的关闭机制替代Quartz插件关闭钩子，避免Quartz过早关闭。 --- .../com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java | 1 + .../charts/bk-job/templates/job-crontab/configmap.yaml | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java index 3d3a20f25d..44928c1860 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java @@ -64,6 +64,7 @@ public void run(String... args) { @PreDestroy public void destroy() { + log.info("destroy LoadCronJobRunner"); if (loadCronJobFuture != null) { boolean result = loadCronJobFuture.cancel(true); log.info("loadCronJobFuture cancel result:{}", result); diff --git a/support-files/kubernetes/charts/bk-job/templates/job-crontab/configmap.yaml b/support-files/kubernetes/charts/bk-job/templates/job-crontab/configmap.yaml index df91a5f51a..9e1cb16169 100644 --- a/support-files/kubernetes/charts/bk-job/templates/job-crontab/configmap.yaml +++ b/support-files/kubernetes/charts/bk-job/templates/job-crontab/configmap.yaml @@ -93,6 +93,7 @@ data: shutdown-timeout: 100ms quartz: job-store-type: MEMORY + waitForJobsToCompleteOnShutdown: true properties: org: quartz: @@ -100,9 +101,6 @@ data: class: org.quartz.simpl.RAMJobStore misfireThreshold: 60000 plugin: - shutdownhook: - class: org.quartz.plugins.management.ShutdownHookPlugin - cleanShutdown: true triggHistory: class: org.quartz.plugins.history.LoggingJobHistoryPlugin scheduler: