Skip to content

Commit

Permalink
[Optimization-3215][admin] Optimize the configuration of dinky address (
Browse files Browse the repository at this point in the history
#3216)

Co-authored-by: wenmo <[email protected]>
  • Loading branch information
aiwenmo and aiwenmo authored Mar 1, 2024
1 parent 9382416 commit 9d06354
Show file tree
Hide file tree
Showing 46 changed files with 73 additions and 54 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

package org.dinky.data.dto;

import org.dinky.gateway.enums.GatewayType;
import org.dinky.data.enums.GatewayType;
import org.dinky.job.JobConfig;

import java.util.Map;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

package org.dinky.data.dto;

import org.dinky.gateway.enums.GatewayType;
import org.dinky.data.enums.GatewayType;
import org.dinky.job.JobConfig;

import io.swagger.annotations.ApiModel;
Expand Down
2 changes: 1 addition & 1 deletion dinky-admin/src/main/java/org/dinky/init/EnvInit.java
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public void run(ApplicationArguments args) throws Exception {
ApplicationContext application = SpringUtil.getApplicationContext();
Environment env = application.getEnvironment();
String port = env.getProperty("server.port");
System.setProperty("dinkyAddr", ipAddress + ":" + port);
System.setProperty("dinkyAddr", "http://" + ipAddress + ":" + port);
log.info(
"\n----------------------------------------------------------\n\t"
+ "Application 'Dinky' is running! Access URLs:\n\t"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.dinky.daemon.task.DaemonTask;
import org.dinky.daemon.task.DaemonTaskConfig;
import org.dinky.data.dto.ClusterInstanceDTO;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.enums.JobStatus;
import org.dinky.data.model.ClusterInstance;
import org.dinky.data.model.Task;
Expand All @@ -33,7 +34,6 @@
import org.dinky.data.model.job.JobInstance;
import org.dinky.data.model.mapping.ClusterConfigurationMapping;
import org.dinky.data.model.mapping.ClusterInstanceMapping;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.job.FlinkJobTask;
import org.dinky.job.Job;
import org.dinky.service.ClusterConfigurationService;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.dinky.data.constant.FlinkRestResultConstant;
import org.dinky.data.dto.ClusterConfigurationDTO;
import org.dinky.data.dto.JobDataDto;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.enums.JobStatus;
import org.dinky.data.flink.backpressure.FlinkJobNodeBackPressure;
import org.dinky.data.flink.checkpoint.CheckPointOverView;
Expand All @@ -37,7 +38,6 @@
import org.dinky.data.model.job.JobInstance;
import org.dinky.gateway.Gateway;
import org.dinky.gateway.config.GatewayConfig;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.gateway.exception.NotSupportGetStatusException;
import org.dinky.gateway.model.FlinkClusterConfig;
import org.dinky.job.JobConfig;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.dinky.assertion.Asserts;
import org.dinky.config.Dialect;
import org.dinky.data.dto.CatalogueTaskDTO;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.enums.JobLifeCycle;
import org.dinky.data.enums.JobStatus;
import org.dinky.data.enums.Status;
Expand All @@ -36,7 +37,6 @@
import org.dinky.data.model.job.JobHistory;
import org.dinky.data.model.job.JobInstance;
import org.dinky.data.result.Result;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.mapper.CatalogueMapper;
import org.dinky.mybatis.service.impl.SuperServiceImpl;
import org.dinky.service.CatalogueService;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@

import org.dinky.assertion.DinkyAssert;
import org.dinky.data.dto.ClusterConfigurationDTO;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.enums.Status;
import org.dinky.data.exception.BusException;
import org.dinky.data.model.ClusterConfiguration;
import org.dinky.data.model.Task;
import org.dinky.gateway.config.GatewayConfig;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.gateway.model.FlinkClusterConfig;
import org.dinky.gateway.result.TestResult;
import org.dinky.job.JobManager;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@
import org.dinky.cluster.FlinkCluster;
import org.dinky.cluster.FlinkClusterInfo;
import org.dinky.data.dto.ClusterInstanceDTO;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.enums.Status;
import org.dinky.data.exception.BusException;
import org.dinky.data.exception.DinkyException;
import org.dinky.data.model.ClusterConfiguration;
import org.dinky.data.model.ClusterInstance;
import org.dinky.data.model.Task;
import org.dinky.gateway.config.GatewayConfig;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.gateway.exception.GatewayException;
import org.dinky.gateway.model.FlinkClusterConfig;
import org.dinky.gateway.result.GatewayResult;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.dinky.data.dto.TaskDTO;
import org.dinky.data.dto.TaskRollbackVersionDTO;
import org.dinky.data.dto.TaskSubmitDto;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.enums.JobLifeCycle;
import org.dinky.data.enums.JobStatus;
import org.dinky.data.enums.ProcessStepType;
Expand Down Expand Up @@ -61,7 +62,6 @@
import org.dinky.function.compiler.CustomStringJavaCompiler;
import org.dinky.function.pool.UdfCodePool;
import org.dinky.function.util.UDFUtil;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.gateway.enums.SavePointStrategy;
import org.dinky.gateway.enums.SavePointType;
import org.dinky.gateway.model.FlinkClusterConfig;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@
import org.dinky.config.Dialect;
import org.dinky.data.annotations.SupportDialect;
import org.dinky.data.dto.TaskDTO;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.exception.NotSupportExecuteExcepition;
import org.dinky.data.exception.NotSupportExplainExcepition;
import org.dinky.data.result.SqlExplainResult;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.job.JobManager;
import org.dinky.job.JobResult;
import org.dinky.service.TaskService;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@
import org.dinky.config.Dialect;
import org.dinky.data.annotations.SupportDialect;
import org.dinky.data.dto.TaskDTO;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.result.SqlExplainResult;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.job.JobManager;
import org.dinky.job.JobResult;
import org.dinky.service.TaskService;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ public static List<FlinkConfigOption> loadOptionsByClassName(String name) {
configList.add(config);
}
} catch (ClassNotFoundException ignored) {
logger.warning("Could not get config option, class not found: " + name);
}
return configList;
}
Expand Down
4 changes: 3 additions & 1 deletion dinky-admin/src/main/resources/dinky-loader/FlinkConfClass
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ org.apache.flink.configuration.NettyShuffleEnvironmentOptions
org.apache.flink.configuration.RestartStrategyOptions
org.apache.flink.streaming.api.environment.ExecutionCheckpointingOptions
org.apache.flink.yarn.configuration.YarnConfigOptions
org.apache.flink.configuration.YarnConfigOptions
org.apache.flink.kubernetes.configuration.KubernetesConfigOptions
org.apache.flink.configuration.KubernetesConfigOptions
org.apache.flink.python.PythonOptions
org.dinky.constant.CustomerConfigureOptions
org.dinky.constant.CustomerConfigureOptions
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.dinky.constant.FlinkSQLConstant;
import org.dinky.data.app.AppParamConfig;
import org.dinky.data.app.AppTask;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.model.SystemConfiguration;
import org.dinky.executor.Executor;
import org.dinky.executor.ExecutorConfig;
Expand Down Expand Up @@ -172,9 +173,9 @@ private static void loadDep(String type, Integer taskId, ExecutorConfig executor
return;
}

if ("kubernetes-application".equals(type)) {
if (GatewayType.get(type).isKubernetesApplicationMode()) {
try {
String httpJar = "http://" + dinkyAddr + "/download/downloadDepJar/" + taskId;
String httpJar = dinkyAddr + "/download/downloadDepJar/" + taskId;
log.info("下载依赖 http-url为:{}", httpJar);
String flinkHome = System.getenv("FLINK_HOME");
String usrlib = flinkHome + "/usrlib";
Expand Down Expand Up @@ -214,7 +215,6 @@ private static void loadDep(String type, Integer taskId, ExecutorConfig executor
}
}
} catch (IOException e) {
log.error("");
throw new RuntimeException(e);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,8 @@ public static void monitorFlinkTask(JobClient jobClient, int taskId) {
private static void sendHook(int taskId, String jobId, int reTryCount) {
String dinkyAddr = SystemConfiguration.getInstances().getDinkyAddr().getValue();
try {
String url = StrFormatter.format(
"http://{}/api/jobInstance/hookJobDone?taskId={}&jobId={}", dinkyAddr, taskId, jobId);
String url =
StrFormatter.format("{}/api/jobInstance/hookJobDone?taskId={}&jobId={}", dinkyAddr, taskId, jobId);
String resultStr = HttpUtil.get(url);
// TODO 这里应该使用Result实体类,但是Result.class不在comm里,迁移改动太大,暂时不搞
String code = JsonUtils.parseObject(resultStr).get("code").toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
*
*/

package org.dinky.gateway.enums;
package org.dinky.data.enums;

import org.dinky.assertion.Asserts;

Expand Down Expand Up @@ -122,4 +122,14 @@ public boolean isApplicationMode() {
return false;
}
}

public boolean isKubernetesApplicationMode() {
switch (this) {
case KUBERNETES_APPLICATION:
case KUBERNETES_APPLICATION_OPERATOR:
return true;
default:
return false;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ sys.maven.settings.repositoryPassword.note=Maven private server authentication p
sys.env.settings.pythonHome=Python Env variables
sys.env.settings.pythonHome.note=Python environment variables, used to submit Python tasks and build Python Udf
sys.env.settings.dinkyAddr=Dinky Address
sys.env.settings.dinkyAddr.note=The address must be the same as the address configured in the Dinky Application background url
sys.env.settings.dinkyAddr.note=This address is an accessible Dinky address, such as http://127.0.0.1:8888
sys.env.settings.jobResendDiffSecond=Alert anti-resend interval
sys.env.settings.jobResendDiffSecond.note=During this interval, when the Alert information sent reaches the configured value of [Maximum number of alarm resend prevention], after reaching the threshold, the Alert information will no longer be sent; unit: seconds
sys.env.settings.diffMinuteMaxSendCount=Maximum number of alarms to prevent resending
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ sys.maven.settings.repositoryPassword.note=Maven 私服认证密码,如果需要
sys.env.settings.pythonHome=Python 环境变量
sys.env.settings.pythonHome.note=Python 环境变量,用于提交 Python 任务以及构建 Python Udf
sys.env.settings.dinkyAddr=Dinky 地址
sys.env.settings.dinkyAddr.note=该地址必须与Dinky Application后台url中配置的地址相同
sys.env.settings.dinkyAddr.note=该地址为可访问的 Dinky 地址,如 http://127.0.0.1:8888
sys.env.settings.jobResendDiffSecond=告警防重发间隔
sys.env.settings.jobResendDiffSecond.note=在此间隔内,发送告警信息达到 [告警防重发最大条数] 配置的值时,达到阈值后,不再发送告警信息; 单位:秒
sys.env.settings.diffMinuteMaxSendCount=告警防重发最大条数
Expand Down Expand Up @@ -281,7 +281,7 @@ process.submit.checkSql=检查作业
process.submit.execute=执行作业
process.submit.buildConfig=构建配置信息
process.submit.execute.commSql=执行普通sql
process.submit.execute.flinkSql=执行flinkSql
process.submit.execute.flinkSql=执行FlinkSql
process.register.exits=当前任务正在执行,请勿重复提交,如有问题请前往配置中心查看

# resource
Expand Down
2 changes: 1 addition & 1 deletion dinky-core/src/main/java/org/dinky/api/FlinkAPI.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import org.dinky.assertion.Asserts;
import org.dinky.data.constant.FlinkRestAPIConstant;
import org.dinky.data.constant.NetConstant;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.exception.BusException;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.gateway.enums.SavePointType;
import org.dinky.gateway.model.JobInfo;
import org.dinky.gateway.result.SavePointResult;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
package org.dinky.executor;

import org.dinky.assertion.Asserts;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.data.enums.GatewayType;

import java.util.ArrayList;
import java.util.HashMap;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
package org.dinky.explainer;

import org.dinky.assertion.Asserts;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.model.LineageRel;
import org.dinky.data.result.ExplainResult;
import org.dinky.data.result.SqlExplainResult;
Expand All @@ -28,7 +29,6 @@
import org.dinky.explainer.print_table.PrintStatementExplainer;
import org.dinky.function.data.model.UDF;
import org.dinky.function.util.UDFUtil;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.interceptor.FlinkInterceptor;
import org.dinky.job.JobConfig;
import org.dinky.job.JobManager;
Expand Down
2 changes: 1 addition & 1 deletion dinky-core/src/main/java/org/dinky/job/Job.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,10 @@

package org.dinky.job;

import org.dinky.data.enums.GatewayType;
import org.dinky.data.result.IResult;
import org.dinky.executor.Executor;
import org.dinky.executor.ExecutorConfig;
import org.dinky.gateway.enums.GatewayType;

import java.time.LocalDateTime;
import java.util.List;
Expand Down
2 changes: 1 addition & 1 deletion dinky-core/src/main/java/org/dinky/job/JobBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@

package org.dinky.job;

import org.dinky.data.enums.GatewayType;
import org.dinky.executor.Executor;
import org.dinky.gateway.enums.GatewayType;

public abstract class JobBuilder {

Expand Down
2 changes: 1 addition & 1 deletion dinky-core/src/main/java/org/dinky/job/JobConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@

import org.dinky.assertion.Asserts;
import org.dinky.data.constant.NetConstant;
import org.dinky.data.enums.GatewayType;
import org.dinky.executor.ExecutorConfig;
import org.dinky.gateway.config.FlinkConfig;
import org.dinky.gateway.config.GatewayConfig;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.gateway.enums.SavePointStrategy;
import org.dinky.gateway.model.FlinkClusterConfig;

Expand Down
2 changes: 1 addition & 1 deletion dinky-core/src/main/java/org/dinky/job/JobManager.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.dinky.context.FlinkUdfPathContextHolder;
import org.dinky.context.RowLevelPermissionsContext;
import org.dinky.data.annotations.ProcessStep;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.enums.ProcessStepType;
import org.dinky.data.exception.BusException;
import org.dinky.data.model.SystemConfiguration;
Expand All @@ -45,7 +46,6 @@
import org.dinky.gateway.config.FlinkConfig;
import org.dinky.gateway.config.GatewayConfig;
import org.dinky.gateway.enums.ActionType;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.gateway.enums.SavePointType;
import org.dinky.gateway.result.GatewayResult;
import org.dinky.gateway.result.SavePointResult;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,12 @@

import org.dinky.assertion.Asserts;
import org.dinky.constant.FlinkSQLConstant;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.result.IResult;
import org.dinky.data.result.InsertResult;
import org.dinky.data.result.ResultBuilder;
import org.dinky.executor.Executor;
import org.dinky.gateway.Gateway;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.gateway.result.GatewayResult;
import org.dinky.interceptor.FlinkInterceptor;
import org.dinky.interceptor.FlinkInterceptorResult;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@

package org.dinky.core;

import org.dinky.data.enums.GatewayType;
import org.dinky.data.result.ResultPool;
import org.dinky.data.result.SelectResult;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.job.JobConfig;
import org.dinky.job.JobManager;
import org.dinky.job.JobResult;
Expand Down
2 changes: 1 addition & 1 deletion dinky-core/src/test/java/org/dinky/job/JobConfigTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

import static org.junit.jupiter.api.Assertions.*;

import org.dinky.gateway.enums.GatewayType;
import org.dinky.data.enums.GatewayType;

import org.apache.flink.configuration.RestOptions;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.dinky.classloader.DinkyClassLoader;
import org.dinky.config.Dialect;
import org.dinky.context.FlinkUdfPathContextHolder;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.exception.DinkyException;
import org.dinky.data.model.FlinkUdfManifest;
import org.dinky.data.model.SystemConfiguration;
Expand All @@ -33,7 +34,6 @@
import org.dinky.function.constant.PathConstant;
import org.dinky.function.data.model.UDF;
import org.dinky.function.pool.UdfCodePool;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.pool.ClassEntity;
import org.dinky.pool.ClassPool;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@

import org.dinky.assertion.Asserts;
import org.dinky.context.FlinkUdfPathContextHolder;
import org.dinky.data.enums.GatewayType;
import org.dinky.data.enums.JobStatus;
import org.dinky.gateway.config.GatewayConfig;
import org.dinky.gateway.enums.ActionType;
import org.dinky.gateway.enums.GatewayType;
import org.dinky.gateway.exception.GatewayException;
import org.dinky.gateway.exception.NotSupportGetStatusException;
import org.dinky.gateway.model.JobInfo;
Expand Down
Loading

0 comments on commit 9d06354

Please sign in to comment.