Merge pull request #138 from FlowCI/develop

Develop

config/app-api.properties

@@ -25,6 +25,10 @@ domain.api = http://localhost:8080
 domain.web = http://localhost:3000
 domain.cc = http://localhost:8080
 
+system.email = [email protected]
+system.username = admin
+system.password = 123456
+
 task.job.toggle.execution_timeout = false
 ## 6s expire job
 task.job.toggle.execution_create_session_duration = 600

platform-api/README.md

@@ -14,7 +14,13 @@
 
 **FLOW_YML_ERROR_MSG**: error message if FLOW_YML_STATUS = ERROR
 
-**FLOW_DEPLOY_KEY_NAME**: the credential name for RSA deploy key pair
+**FLOW_ENV_OUTPUT_PREFIX**: env output prefix used for pass env variable which start with this prefix to next step
+
+**FLOW_TASK_CRONTAB_CONTENT**: flow crontab content, the format like * * * * * ?,
+ and FLOW_TASK_CRONTAB_BRANCH are required 
+
+**FLOW_TASK_CRONTAB_BRANCH**: flow crontab branch
+
 
 ### Job
 
@@ -24,6 +30,8 @@
 
 **FLOW_JOB_AGENT_INFO**:  readonly, the agent path which job running on as 'zone#name' format
 
+**FLOW_JOB_LOG_PATH**: readonly: the download path for job log
+
 ### Git 
         
 **FLOW_GIT_SOURCE**: {UNDEFINED_SSH | UNDEFINED_HTTP | GITLAB | GITHUB| CODING| OSCHINA | BITBUCKET} 

platform-api/pom.xml

@@ -149,6 +149,16 @@
       <artifactId>velocity</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>org.quartz-scheduler</groupId>
+      <artifactId>quartz</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>org.quartz-scheduler</groupId>
+      <artifactId>quartz-jobs</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>

platform-api/src/main/java/com/flow/platform/api/config/AppConfig.java

@@ -34,24 +34,27 @@
 import org.apache.velocity.app.VelocityEngine;
 import org.apache.velocity.runtime.RuntimeConstants;
 import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.event.ApplicationEventMulticaster;
 import org.springframework.context.event.SimpleApplicationEventMulticaster;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 
 /**
  * @author yang
  */
 @Configuration
-@Import({CachingConfig.class, DatabaseConfig.class})
+@Import({SchedulerConfig.class, CachingConfig.class, DatabaseConfig.class})
 public class AppConfig extends AppConfigBase {
 
     public final static String NAME = "API";
 
-    public final static String VERSION = "0.1.0";
+    public final static String VERSION = "v0.1-alpha";
 
     public final static String DEFAULT_YML_FILE = ".flow.yml";
 
@@ -67,11 +70,6 @@ public class AppConfig extends AppConfigBase {
 
     private final static int MULTICASTER_ASYNC_POOL_SIZE = 1;
 
-    public final static String DEFAULT_USER_EMAIL = "[email protected]";
-    public final static String DEFAULT_USER_NAME = "admin";
-
-    public final static String DEFAULT_USER_PASSWORD = "123456";
-
     private final static ThreadPoolTaskExecutor executor =
         ThreadUtil.createTaskExecutor(ASYNC_POOL_SIZE, ASYNC_POOL_SIZE / 10, 100, THREAD_NAME_PREFIX);
 
@@ -85,6 +83,20 @@ public class AppConfig extends AppConfigBase {
     @Value("${domain.cc}")
     private String ccDomain;
 
+    @Value(value = "${system.email}")
+    private String email;
+
+    @Value(value = "${system.username}")
+    private String username;
+
+    @Value(value = "${system.password}")
+    private String password;
+
+    @Bean
+    public User superUser() {
+        return new User(email, username, password);
+    }
+
     @Bean
     public Path workspace() {
         try {
@@ -125,7 +137,7 @@ public ThreadPoolTaskExecutor taskExecutor() {
      */
     @Bean
     public PlatformQueue<CmdCallbackQueueItem> cmdCallbackQueue() {
-        return new InMemoryQueue<>(executor, 50);
+        return new InMemoryQueue<>(executor, 50, "CmdCallbackQueue");
     }
 
     @Override

platform-api/src/main/java/com/flow/platform/api/config/SchedulerConfig.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2017 flow.ci
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.flow.platform.api.config;
+
+import com.flow.platform.api.task.NodeCrontabTask;
+import com.flow.platform.core.exception.IllegalStatusException;
+import java.io.IOException;
+import org.quartz.JobBuilder;
+import org.quartz.JobDetail;
+import org.quartz.Scheduler;
+import org.quartz.SchedulerException;
+import org.quartz.impl.StdSchedulerFactory;
+import org.springframework.beans.factory.annotation.Configurable;
+import org.springframework.context.annotation.Bean;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.scheduling.annotation.EnableScheduling;
+import org.springframework.scheduling.annotation.SchedulingConfigurer;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
+import org.springframework.scheduling.config.ScheduledTaskRegistrar;
+
+/**
+ * @author yang
+ */
+@Configurable
+@EnableScheduling
+public class SchedulerConfig implements SchedulingConfigurer {
+
+    private final static int DEFAULT_SCHEDULER_POOL_SIZE = 10;
+
+    /**
+     * Thread pool for spring scheduling
+     */
+    @Bean(destroyMethod = "shutdown")
+    public ThreadPoolTaskScheduler schedulerTaskExecutor() {
+        ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler();
+        scheduler.setPoolSize(DEFAULT_SCHEDULER_POOL_SIZE);
+        scheduler.setDaemon(true);
+        scheduler.setThreadNamePrefix("api-task-");
+        scheduler.initialize();
+        return scheduler;
+    }
+
+    /**
+     * Setup quartz scheduler
+     */
+    @Bean
+    public Scheduler quartzScheduler() {
+        try {
+            StdSchedulerFactory factory = new StdSchedulerFactory();
+            factory.initialize(new ClassPathResource("quartz.properties").getInputStream());
+            return factory.getScheduler();
+        } catch (SchedulerException | IOException e) {
+            throw new IllegalStatusException("Unable to init quartz: " + e.getMessage());
+        }
+    }
+
+    /**
+     * Setup flow crontab job detail
+     */
+    @Bean
+    public JobDetail nodeCrontabDetail() {
+        return JobBuilder.newJob(NodeCrontabTask.class).storeDurably(true).build();
+    }
+
+    @Override
+    public void configureTasks(ScheduledTaskRegistrar taskRegistrar) {
+        taskRegistrar.setTaskScheduler(schedulerTaskExecutor());
+    }
+}

platform-api/src/main/java/com/flow/platform/api/config/WebConfig.java

@@ -18,6 +18,7 @@
 
 import com.flow.platform.api.resource.PropertyResourceLoader;
 import com.flow.platform.api.security.AuthenticationInterceptor;
+import com.flow.platform.api.security.OptionsInterceptor;
 import com.flow.platform.api.security.token.JwtTokenGenerator;
 import com.flow.platform.api.security.token.TokenGenerator;
 import com.flow.platform.core.http.converter.RawGsonMessageConverter;
@@ -44,6 +45,7 @@
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.security.web.util.matcher.AntPathRequestMatcher;
 import org.springframework.security.web.util.matcher.RequestMatcher;
+import org.springframework.web.context.request.WebRequestInterceptor;
 import org.springframework.web.multipart.MultipartResolver;
 import org.springframework.web.multipart.commons.CommonsMultipartResolver;
 import org.springframework.web.servlet.config.annotation.CorsRegistry;
@@ -53,11 +55,11 @@
 
 @Configuration
 @EnableWebMvc
-@EnableScheduling
 @ComponentScan({
     "com.flow.platform.core.controller",
     "com.flow.platform.api.controller",
     "com.flow.platform.api.service",
+    "com.flow.platform.api.envs.handler",
     "com.flow.platform.api.security",
     "com.flow.platform.api.dao",
     "com.flow.platform.api.context",
@@ -113,21 +115,30 @@ public AuthenticationInterceptor authInterceptor() {
         List<RequestMatcher> matchers = Lists.newArrayList(
             new AntPathRequestMatcher("/flows/**"),
             new AntPathRequestMatcher("/user/register"),
+            new AntPathRequestMatcher("/user/delete"),
+            new AntPathRequestMatcher("/user"),
+            new AntPathRequestMatcher("/user/role/update"),
             new AntPathRequestMatcher("/jobs/**"),
             new AntPathRequestMatcher("/credentials/**"),
             new AntPathRequestMatcher("/actions/**"),
-            new AntPathRequestMatcher("/message/**")
+            new AntPathRequestMatcher("/message/**"),
+            new AntPathRequestMatcher("/agents/create"),
+            new AntPathRequestMatcher("/agents"),
+            new AntPathRequestMatcher("/roles/**"),
+            new AntPathRequestMatcher("/thread/config")
         );
         return new AuthenticationInterceptor(matchers);
     }
 
+
     @Bean
     public RawGsonMessageConverter jsonConverter() {
         return this.jsonConverter;
     }
 
     @Override
     public void addInterceptors(InterceptorRegistry registry) {
+        registry.addInterceptor(new OptionsInterceptor());
         registry.addInterceptor(authInterceptor());
     }
 

platform-api/src/main/java/com/flow/platform/api/consumer/CmdCallbackQueueConsumer.java

@@ -18,6 +18,7 @@
 
 import com.flow.platform.api.domain.CmdCallbackQueueItem;
 import com.flow.platform.api.service.job.JobService;
+import com.flow.platform.core.exception.NotFoundException;
 import com.flow.platform.core.queue.PlatformQueue;
 import com.flow.platform.core.queue.QueueListener;
 import com.flow.platform.util.Logger;
@@ -51,8 +52,22 @@ public void onQueueItem(CmdCallbackQueueItem item) {
         }
         try {
             jobService.callback(item);
-        }catch (Throwable throwable){
+        } catch (NotFoundException notFoundException) {
+
+            // re-enqueue cmd callback if job not found since transaction problem
+            reEnqueueJobCallback(item, 1000);
+
+        } catch (Throwable throwable) {
             LOGGER.traceMarker("onQueueItem", String.format("exception - %s", throwable));
         }
     }
+
+    private void reEnqueueJobCallback(CmdCallbackQueueItem item, long wait) {
+        try {
+            Thread.sleep(wait);
+        } catch (Throwable ignore) {
+        }
+
+        jobService.enterQueue(item);
+    }
 }

platform-api/src/main/java/com/flow/platform/api/consumer/JobEventPushHandler.java

@@ -46,6 +46,7 @@ protected void push(Job job) {
         } catch (NotFoundException e) {
             jobConsist = job;
         }
+        jobConsist.setStatus(job.getStatus());
         String jobTopic = String.format("%s/%s", WebSocketConfig.TOPIC_FOR_JOB, jobConsist.getNodePath());
         super.push(jobTopic, jobConsist);
     }

platform-api/src/main/java/com/flow/platform/api/controller/ActionController.java

@@ -15,14 +15,17 @@
  */
 package com.flow.platform.api.controller;
 
+import com.flow.platform.api.domain.permission.Actions;
 import com.flow.platform.api.domain.request.ActionParam;
 import com.flow.platform.api.domain.user.Action;
+import com.flow.platform.api.security.WebSecurity;
 import com.flow.platform.api.service.user.ActionService;
 import java.util.List;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PatchMapping;
 import org.springframework.web.bind.annotation.PathVariable;
+import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RestController;
@@ -55,6 +58,7 @@ public class ActionController {
      *  ]
      */
     @GetMapping
+    @WebSecurity(action = Actions.ADMIN_SHOW)
     public List<Action> index() {
         return actionService.list();
     }
@@ -81,7 +85,36 @@ public List<Action> index() {
      *  }
      */
     @PatchMapping(path = "/{name}")
+    @WebSecurity(action = Actions.ADMIN_UPDATE)
     public Action update(@PathVariable String name, @RequestBody ActionParam body) {
         return actionService.update(name, body);
     }
+
+    /**
+     * @api {post} /actions
+     * @apiParam {String} create action
+     * @apiParamExample {json} Request-Body
+     *  {
+     *      name: FLOW_CREATE,
+     *      description: xxx,
+     *      tag: USER
+     *  }
+     * @apiGroup Action
+     *
+     * @apiSuccessExample {json} Success-Response
+     *  {
+     *      name: FLOW_CREATE,
+     *      alias: create flow,
+     *      description: xxxx,
+     *      tag: DEFAULT,
+     *      createdAt: xxx,
+     *      updatedAt: xxx
+     *  }
+     */
+
+    @PostMapping
+    @WebSecurity(action = Actions.ADMIN_CREATE)
+    public Action create(@RequestBody Action action) {
+        return actionService.create(action);
+    }
 }