feat: Turbo运营数据统计 #156

src/backend/turbo/api-turbo/src/main/kotlin/com/tencent/devops/turbo/api/IUserCustomScheduleTaskController.kt

@@ -3,10 +3,12 @@ package com.tencent.devops.turbo.api
 import com.tencent.devops.common.util.constants.AUTH_HEADER_DEVOPS_PROJECT_ID
 import com.tencent.devops.common.util.constants.AUTH_HEADER_DEVOPS_USER_ID
 import com.tencent.devops.turbo.pojo.CustomScheduleJobModel
+import com.tencent.devops.api.pojo.Response
 import io.swagger.annotations.Api
 import io.swagger.annotations.ApiOperation
 import io.swagger.annotations.ApiParam
 import org.springframework.http.MediaType
+import org.springframework.web.bind.annotation.DeleteMapping
 import org.springframework.web.bind.annotation.GetMapping
 import org.springframework.web.bind.annotation.PostMapping
 import org.springframework.web.bind.annotation.RequestBody
@@ -34,7 +36,24 @@ interface IUserCustomScheduleTaskController {
         @ApiParam(value = "编译加速模式信息", required = true)
         @RequestBody
         customScheduleJobModel: CustomScheduleJobModel
-    ): Boolean
+    ): Response<Boolean>
+
+    @ApiOperation("删除计划任务")
+    @DeleteMapping(
+        "/deleteScheduleJob",
+        produces = [MediaType.APPLICATION_JSON_VALUE]
+    )
+    fun deleteScheduleJob(
+        @ApiParam(value = "用户信息", required = true)
+        @RequestHeader(AUTH_HEADER_DEVOPS_USER_ID)
+        user: String,
+        @ApiParam(value = "项目id", required = true)
+        @RequestHeader(AUTH_HEADER_DEVOPS_PROJECT_ID)
+        projectId: String,
+        @ApiParam(value = "任务名称", required = true)
+        @RequestParam(value = "jobName")
+        jobName: String
+    ): Response<Boolean>
 
     @ApiOperation("触发定时任务执行")
     @GetMapping(
@@ -51,5 +70,5 @@ interface IUserCustomScheduleTaskController {
         @ApiParam(value = "任务名称", required = true)
         @RequestParam(value = "jobName")
         jobName: String
-    ): String?
+    ): Response<String>
 }

src/backend/turbo/biz-turbo/build.gradle.kts

@@ -16,10 +16,10 @@ dependencies {
     api("com.tencent.bk.devops.ci.common:common-api:${Versions.ciVersion}"){
         isTransitive = false
     }
-    api("com.tencent.bk.devops.ci.auth:api-auth:${Versions.ciAuthVersion}"){
+    api("com.tencent.bk.devops.ci.auth:api-auth:${Versions.ciVersion}"){
         isTransitive = false
     }
-    api("com.tencent.bk.devops.ci.common:common-auth-api:${Versions.ciAuthVersion}"){
+    api("com.tencent.bk.devops.ci.common:common-auth-api:${Versions.ciVersion}"){
         isTransitive = false
     }
 }

src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/controller/UserCustomScheduleTaskController.kt

@@ -1,5 +1,6 @@
 package com.tencent.devops.turbo.controller
 
+import com.tencent.devops.api.pojo.Response
 import com.tencent.devops.common.api.exception.TurboException
 import com.tencent.devops.common.api.exception.code.IS_NOT_ADMIN_MEMBER
 import com.tencent.devops.common.util.constants.NO_ADMIN_MEMBER_MESSAGE
@@ -21,17 +22,24 @@ class UserCustomScheduleTaskController @Autowired constructor(
         user: String,
         projectId: String,
         customScheduleJobModel: CustomScheduleJobModel
-    ): Boolean {
-        if (!turboAuthService.validatePlatformMember(projectId, user)) {
+    ): Response<Boolean> {
+        if (!turboAuthService.getAuthResult(projectId, user)) {
+            throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE)
+        }
+        return Response.success(customScheduleJobService.customScheduledJobAdd(customScheduleJobModel))
+    }
+
+    override fun deleteScheduleJob(user: String, projectId: String, jobName: String): Response<Boolean> {
+        if (!turboAuthService.getAuthResult(projectId, user)) {
             throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE)
         }
-        return customScheduleJobService.customScheduledJobAdd(customScheduleJobModel)
+        return Response.success(customScheduleJobService.customScheduledJobDel(jobName))
     }
 
-    override fun triggerCustomScheduleJob(user: String, projectId: String, jobName: String): String? {
+    override fun triggerCustomScheduleJob(user: String, projectId: String, jobName: String): Response<String> {
         if (!turboAuthService.getAuthResult(projectId, user)) {
             throw TurboException(errorCode = IS_NOT_ADMIN_MEMBER, errorMessage = NO_ADMIN_MEMBER_MESSAGE)
         }
-        return customScheduleJobService.trigger(jobName)
+        return Response.success(customScheduleJobService.trigger(jobName))
     }
 }

src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TbsDaySummaryRepository.kt

@@ -0,0 +1,13 @@
+package com.tencent.devops.turbo.dao.repository
+
+import com.tencent.devops.turbo.model.TTbsDaySummaryEntity
+import org.springframework.data.mongodb.repository.MongoRepository
+import org.springframework.stereotype.Repository
+
+@Repository
+interface TbsDaySummaryRepository : MongoRepository<TTbsDaySummaryEntity, String> {
+    /**
+     * 根据日期删除
+     */
+    fun removeAllByDay(day: String)
+}

src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dao/repository/TurboPlanRepository.kt

@@ -29,4 +29,9 @@ interface TurboPlanRepository : MongoRepository<TTurboPlanEntity, String> {
      * 根据项目id和编译加速方案名称和id判断是否存在
      */
     fun existsByProjectIdAndPlanNameAndIdNot(projectId: String, planName: String, planId: String): Boolean
+
+    /**
+     * 根据方案id集合批量查询
+     */
+    fun findByIdIn(planIdSet: List<String?>): List<TTurboPlanEntity>
 }

src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/dto/TBSDaySummaryDto.kt

@@ -0,0 +1,30 @@
+package com.tencent.devops.turbo.dto
+
+import com.fasterxml.jackson.annotation.JsonProperty
+
+data class TBSDaySummaryDto(
+    val day: String,
+
+    val user: String?,
+
+    @JsonProperty("project_id")
+    val projectId: String,
+
+    /**
+     * 加速时长(单位：秒)
+     */
+    @JsonProperty("total_time")
+    val totalTime: Double,
+
+    /**
+     * 加速时长*cpu核数(单位秒*核)
+     */
+    @JsonProperty("total_time_with_cpu")
+    val totalTimeWithCpu: Double,
+
+    /**
+     * 加速次数
+     */
+    @JsonProperty("total_record_number")
+    val totalRecordNumber: Double,
+)

src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/job/TBSDaySummaryJob.kt

@@ -0,0 +1,176 @@
+package com.tencent.devops.turbo.job
+
+import com.tencent.devops.common.client.Client
+import com.tencent.devops.common.util.DateTimeUtils
+import com.tencent.devops.common.util.JsonUtil
+import com.tencent.devops.project.api.service.ServiceProjectResource
+import com.tencent.devops.project.pojo.ProjectVO
+import com.tencent.devops.turbo.dao.repository.TbsDaySummaryRepository
+import com.tencent.devops.turbo.dao.repository.TurboEngineConfigRepository
+import com.tencent.devops.turbo.dao.repository.TurboPlanRepository
+import com.tencent.devops.turbo.dto.TBSDaySummaryDto
+import com.tencent.devops.turbo.model.TTbsDaySummaryEntity
+import com.tencent.devops.turbo.sdk.TBSSdkApi
+import org.quartz.Job
+import org.quartz.JobExecutionContext
+import org.slf4j.LoggerFactory
+import org.springframework.beans.factory.annotation.Autowired
+import java.time.LocalDate
+import java.time.LocalDateTime
+
+@Suppress("SpringJavaAutowiredMembersInspection")
+class TBSDaySummaryJob @Autowired constructor(
+    private val client: Client,
+    private val tbsDaySummaryRepository: TbsDaySummaryRepository,
+    private val turboEngineConfigRepository: TurboEngineConfigRepository,
+    private val turboPlanRepository: TurboPlanRepository
+) : Job {
+
+    companion object {
+        private val logger = LoggerFactory.getLogger(this::class.java)
+        private const val PAGE_SIZE = 3000
+    }
+
+    /**
+     * 执行入口
+     */
+    override fun execute(context: JobExecutionContext) {
+        logger.info("TBS day summary job start executing: ${JsonUtil.toJson(context.jobDetail)}")
+
+        val jobParam = context.jobDetail.jobDataMap
+        val statisticsDateStr = if (jobParam.containsKey("statisticsDate")) {
+            jobParam["statisticsDate"] as String
+        } else {
+            // 统计昨天
+            val statLocalDate = LocalDate.now().minusDays(1)
+            DateTimeUtils.localDate2DateStr(statLocalDate)
+        }
+
+        // 清理待统计的数据，防止重复统计
+        tbsDaySummaryRepository.removeAllByDay(statisticsDateStr)
+
+        val projectVOMap =  mutableMapOf<String, ProjectVO>()
+
+        val engineConfigEntities = turboEngineConfigRepository.findAll()
+        engineConfigEntities.forEach { engineConfig ->
+            logger.info("query engineConfig: ${engineConfig.engineCode}")
+            val daySummaryDtoList = try {
+                TBSSdkApi.queryTbsDaySummary(
+                    engineCode = engineConfig.engineCode,
+                    queryParam = mapOf(
+                        "day" to statisticsDateStr
+                    )
+                )
+            } catch (e: Exception) {
+                logger.error("queryTbsDaySummary error: ${e.message}")
+                return@forEach
+            }
+
+            logger.info("daySummaryDtoList size: ${daySummaryDtoList.size}")
+            if (daySummaryDtoList.isEmpty()) {
+                logger.warn("queryTbsDaySummary result is empty! engineCode: ${engineConfig.engineCode}")
+                return@forEach
+            }
+
+            // 把TBS的接口数据整理成entity
+            val summaryEntityList = this.dto2SummaryEntityList(daySummaryList = daySummaryDtoList)
+            val summaryListList = summaryEntityList.chunked(PAGE_SIZE)
+            for (summaryList in summaryListList) {
+
+                // 根据planId批量获取方案信息
+                val planIds = summaryList.map { it.planId }.toSet()
+                val turboPlanList = turboPlanRepository.findByIdIn(planIds.toList())
+                logger.info("turboPlanRepository.findByIdIn result size: ${turboPlanList.size}")
+                val planEntityMap = turboPlanList.associateBy { it.id }
+
+                // 赋值plan信息和项目id
+                for (summaryEntity in summaryList) {
+                    val planEntity = planEntityMap[summaryEntity.planId]
+                    summaryEntity.planCreator = planEntity?.createdBy
+                    summaryEntity.planName = planEntity?.planName
+                    summaryEntity.projectId = planEntity?.projectId
+                }
+
+                // 取出项目ID集合用于获取项目组织架构信息
+                val projectIdSet = turboPlanList.map { it.projectId }.toSet()
+                val notInProjectMapKeySet = projectIdSet.subtract(projectVOMap.keys)
+
+                // 获取项目信息清单
+                val projectVOList = this.getProjectVOListByProjectIds(projectIds = notInProjectMapKeySet.toList())
+                if (projectVOList.isNotEmpty()) {
+                    projectVOMap.putAll(projectVOList.associateBy { it.englishName })
+                }
+
+                for (it in summaryList) {
+                    it.projectName = projectVOMap[it.projectId]?.projectName
+                    it.bgName = projectVOMap[it.projectId]?.bgName
+                    it.bgId = projectVOMap[it.projectId]?.bgId?.toInt()
+                    it.deptName = projectVOMap[it.projectId]?.deptName
+                    it.deptId = projectVOMap[it.projectId]?.deptId?.toInt()
+                    it.centerName = projectVOMap[it.projectId]?.centerName
+                    it.centerId = projectVOMap[it.projectId]?.centerId?.toInt()
+                    it.productId = projectVOMap[it.projectId]?.productId
+                }
+            }
+
+            tbsDaySummaryRepository.saveAll(summaryEntityList)
+            logger.info("save summary entity size: ${summaryEntityList.size}")
+        }
+        logger.info("TBS day summary job execution completed!")
+    }
+
+    /**
+     * 把TBS的接口数据整理成entity
+     */
+    private fun dto2SummaryEntityList(daySummaryList: List<TBSDaySummaryDto>): List<TTbsDaySummaryEntity> {
+        val summaryEntities = mutableListOf<TTbsDaySummaryEntity>()
+
+        daySummaryList.forEach { summary ->
+            // distcc与其它不一样，它的projectId就是planId
+            val planIdAndEngineCode = summary.projectId
+
+            val planId: String
+            val engineCode: String
+
+            // "60d54b87a26123319d011bob_cc"
+            if (planIdAndEngineCode.contains("_")) {
+                val stringArr = planIdAndEngineCode.split("_")
+                planId = stringArr[0]
+                engineCode = if (stringArr[1] == "cc") "disttask-cc" else if (stringArr[1] == "ue4") "disttask-ue4"
+                    else stringArr[1]
+            } else {
+                planId = planIdAndEngineCode
+                engineCode = "distcc"
+            }
+
+            val entity = TTbsDaySummaryEntity(
+                day = summary.day,
+                engineCode = engineCode,
+                planId = planId,
+                user = if (engineCode == "disttask-ue4") summary.user else null,
+                totalTime = summary.totalTime,
+                totalTimeWithCpu = summary.totalTimeWithCpu,
+                totalRecordNumber = summary.totalRecordNumber,
+                createdDate = LocalDateTime.now()
+            )
+            summaryEntities.add(entity)
+        }
+        return summaryEntities
+    }
+
+    /**
+     * 根据项目id获取项目信息
+     */
+    private fun getProjectVOListByProjectIds(projectIds: List<String>): List<ProjectVO> {
+        var list = emptyList<ProjectVO>()
+        if (projectIds.isNotEmpty()) {
+            val result = client.get(ServiceProjectResource::class.java).listByProjectCodeList(projectIds)
+            if (result.isNotOk() || result.data == null) {
+                logger.error("ServiceProjectResource#get request is failed!")
+                return list
+            }
+            list = result.data!!
+        }
+        return list
+    }
+}

src/backend/turbo/biz-turbo/src/main/kotlin/com/tencent/devops/turbo/sdk/TBSSdkApi.kt

@@ -9,6 +9,7 @@ import com.tencent.devops.turbo.config.TBSProperties
 import com.tencent.devops.turbo.dto.DistccRequestBody
 import com.tencent.devops.turbo.dto.DistccResponse
 import com.tencent.devops.turbo.dto.ParamEnumDto
+import com.tencent.devops.turbo.dto.TBSDaySummaryDto
 import com.tencent.devops.turbo.dto.TBSTurboStatDto
 import com.tencent.devops.turbo.dto.WhiteListDto
 import com.tencent.devops.web.util.SpringContextHolder
@@ -122,13 +123,17 @@ object TBSSdkApi {
         queryParam: Map<String, Any> = mutableMapOf(),
         jsonBody: String = "",
         headers: MutableMap<String, String> = mutableMapOf(),
-        method: String = "GET"
+        method: String = "GET",
+        customPath: String? = null
     ): String {
         val requestBody = jsonBody.toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull())
         val properties = SpringContextHolder.getBean<TBSProperties>()
-        val customPath =
+        val templatePath =
             properties.urlTemplate!!.replace("{engine}", engineCode).replace("{resource_type}", resourceName)
-        var url = "${properties.rootPath}/$customPath"
+        var url = "${properties.rootPath}/$templatePath"
+        if (!customPath.isNullOrBlank()) {
+            url = url.plus(customPath)
+        }
         if (pathParam.isNotEmpty()) {
             pathParam.forEach {
                 url = url.plus("/$it")
@@ -207,4 +212,32 @@ object TBSSdkApi {
             JsonUtil.to(responseStr, object : TypeReference<DistccResponse<MutableList<String>>>() {}).data
         }
     }
+
+    /**
+     * 获取TBS每日汇总统计
+     */
+    fun queryTbsDaySummary(engineCode: String, queryParam: Map<String, Any>): List<TBSDaySummaryDto> {
+        val responseStr = if (engineCode.contains("disttask")) {
+            // tbs后台接口路径处理
+            tbsCommonRequest(
+                engineCode = "disttask",
+                resourceName = "summary",
+                queryParam = queryParam,
+                customPath = if (engineCode.contains("ue4"))"/groupbyuser/scene/ue4" else null
+            )
+        } else {
+            tbsCommonRequest(
+                engineCode = engineCode,
+                resourceName = "summary",
+                queryParam = queryParam
+            )
+        }
+        val response = JsonUtil.to(responseStr, object : TypeReference<DistccResponse<List<TBSDaySummaryDto>>>() {})
+        if (response.code != 0 || !response.result) {
+            throw TurboException(errorCode = TURBO_THIRDPARTY_SYSTEM_FAIL, errorMessage = "fail to invoke request: "
+                + response.message
+            )
+        }
+        return response.data ?: listOf()
+    }
 }