diff --git a/src/backend/core/build.gradle b/src/backend/core/build.gradle index 0f92e33..a61fe7f 100644 --- a/src/backend/core/build.gradle +++ b/src/backend/core/build.gradle @@ -92,7 +92,7 @@ allprojects { compile "org.hashids:hashids:1.0.3" compile group: 'commons-codec', name: 'commons-codec', version: '1.15' compile("org.apache.commons:commons-compress:1.15") - + compile "org.tukaani:xz:1.2" testCompile group: 'junit', name: 'junit', version: '4.12' } diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/Scan.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/Scan.kt index 5f4dbc8..9e0f8a7 100644 --- a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/Scan.kt +++ b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/Scan.kt @@ -50,8 +50,7 @@ class Scan( LogUtils.printLog("toolName: $toolName") LogUtils.printLog("compile_tools: ${ToolConstants.COMPILE_TOOLS}") if (toolName in ToolConstants.COMPILE_TOOLS) { - val toolFolder = commandParam.projectBuildPath + File.separator + ".temp" + File.separator + "codecc_scan" + - File.separator + "codecc_agent" + File.separator + "bin" + File.separator + toolName + val toolFolder = "/data/codecc_software/${toolName}_scan" val command = CodeccConfig.getConfig("${toolName.toUpperCase()}_SCAN_COMMAND")!! .replace("##", " ") .replace("{input.json}", inputFile) diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/ScanComposer.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/ScanComposer.kt index 122d3a1..75891b7 100644 --- a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/ScanComposer.kt +++ b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/ScanComposer.kt @@ -76,6 +76,7 @@ object ScanComposer { } // check third env CodeccConfig.checkThirdEnv(commandParam, toolName) + CodeccConfig.downloadToolZip(commandParam, toolName) // 排队开始 CodeccWeb.codeccUploadTaskLog(analyzeConfigInfo.taskId, streamName, toolName, commandParam.landunParam, 1, 3) diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/pojo/ToolConstants.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/pojo/ToolConstants.kt index 665703b..3ed5277 100644 --- a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/pojo/ToolConstants.kt +++ b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/pojo/ToolConstants.kt @@ -11,10 +11,11 @@ object ToolConstants { const val CLOC = "cloc" const val SCC = "scc" const val CLANG = "clang" + const val CLANGWARNING = "clangwarning" const val SPOTBUGS = "spotbugs" const val GITHUBSTATISTIC = "githubstatistic" const val RIPS = "rips" - val COMPILE_TOOLS = listOf(COVERITY, KLOCWORK, PINPOINT,CODEQL, CLANG, SPOTBUGS) + val COMPILE_TOOLS = listOf(COVERITY, KLOCWORK, PINPOINT,CODEQL, CLANG, CLANGWARNING, SPOTBUGS) val NOLINT_TOOLS = listOf(DUPC, CCN, TSCCN, CLOC, SCC, COVERITY, KLOCWORK, PINPOINT, GITHUBSTATISTIC) val CODE_TOOLS_ACOUNT = listOf(CLOC, SCC) } diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/tools/FileUtil.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/tools/FileUtil.kt index 32925f8..7c9f99a 100644 --- a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/tools/FileUtil.kt +++ b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/tools/FileUtil.kt @@ -30,6 +30,7 @@ import org.apache.commons.codec.digest.DigestUtils import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream +import org.apache.commons.compress.compressors.xz.XZCompressorInputStream import java.io.BufferedInputStream import java.io.BufferedOutputStream import java.io.File @@ -190,6 +191,27 @@ object FileUtil { } } + fun unzipTxzFile(txzFile: String, destDir: String = "./") { + val blockSize = 4096 + val inputStream = TarArchiveInputStream(XZCompressorInputStream(File(txzFile).inputStream()), blockSize) + while (true) { + val entry = inputStream.nextTarEntry ?: break + if (entry.isDirectory) { // 是目录 + val dir = File(destDir, entry.name) + if (!dir.exists()) dir.mkdirs() + } else { // 是文件 + File(destDir, entry.name).outputStream().use { outputStream -> + while (true) { + val buf = ByteArray(4096) + val len = inputStream.read(buf) + if (len == -1) break + outputStream.write(buf, 0, len) + } + } + } + } + } + fun unzipFile(zipFile: String, destDir: String = "./") { val blockSize = 4096 val inputStream = ZipArchiveInputStream(BufferedInputStream(FileInputStream(File(zipFile)), blockSize)) diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccConfig.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccConfig.kt index 824e568..68d64ea 100644 --- a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccConfig.kt +++ b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccConfig.kt @@ -188,6 +188,26 @@ object CodeccConfig { fun checkThirdEnv(commandParam: CommandParam, toolName: String) { } + @Synchronized + fun downloadToolZip(commandParam: CommandParam, toolName: String) { + val softwareRootPath = "/data/codecc_software" + var toolSourceName = "" + var toolBinaryName = "" + var suffix = "" + if ((toolName in ToolConstants.COMPILE_TOOLS) && !checkPythonAll(softwareRootPath)) { + throw CodeccUserConfigException("Codecc need install python3") + } + if (ToolConstants.CLANG == toolName) { + toolSourceName = "clang_scan.zip" + toolBinaryName = "clang-${getConfig("CLANG_NEW_VERSION")}" + suffix = "tar.xz" + commandParam.clangHomeBin = CodeccWeb.downloadCompileTool(toolName, toolSourceName, toolBinaryName, suffix) + }else if(ToolConstants.CLANGWARNING == toolName) { + toolSourceName = "clangwarning_scan.zip" + CodeccWeb.downloadCompileTool(toolName, toolSourceName, toolBinaryName, suffix) + } + } + private fun checkPython(home: String): Boolean { val cmd = "python3 --version" val output = try { diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccWeb.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccWeb.kt index b8ff5dd..47cad9f 100644 --- a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccWeb.kt +++ b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccWeb.kt @@ -6,7 +6,6 @@ import com.fasterxml.jackson.module.kotlin.readValue import com.tencent.bk.devops.atom.api.BaseApi import com.tencent.bk.devops.plugin.utils.JsonUtil import com.tencent.bk.devops.plugin.utils.OkhttpUtils -import com.tencent.devops.docker.ScanComposer import com.tencent.devops.docker.pojo.* import com.tencent.devops.docker.tools.FileUtil import com.tencent.devops.docker.tools.LogUtils @@ -25,7 +24,12 @@ import org.apache.commons.codec.digest.DigestUtils import java.io.File import java.io.FileInputStream import java.io.FileOutputStream +import java.io.InputStream +import java.io.BufferedOutputStream +import java.io.BufferedInputStream import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL object CodeccWeb : BaseApi() { @@ -74,6 +78,67 @@ object CodeccWeb : BaseApi() { return binFolder } + fun downloadCompileTool(toolName: String, toolSourceZip: String, toolBinaryName: String, suffix: String) : String{ + var fileOut: FileOutputStream? + var conn: HttpURLConnection? + var inputStream: InputStream? + var rootPath = "/data/codecc_software" + var url = "https://github.com/TencentBlueKing/codeccScan/raw/master/${toolName}_scan/$toolSourceZip" + val toolHome = "$rootPath/${toolName}_scan" + if (File(toolHome).exists() && File(toolHome).list()?.isNotEmpty() == true) { + if (File("$toolHome/$toolBinaryName").exists()) { + return "$toolHome/$toolBinaryName" + } + } + if (!File(rootPath).exists()) { + File(rootPath).mkdirs() + } + try { + LogUtils.printLog("start to download tool zip") + val httpUrl = URL(url) + conn = httpUrl.openConnection() as HttpURLConnection + conn.setRequestMethod("GET") + conn.setDoInput(true) + conn.setDoOutput(true) + conn.setUseCaches(false) + conn.connect() + inputStream = conn.getInputStream() + val bis = BufferedInputStream(inputStream) + val toolFullPath = rootPath + File.separator + toolSourceZip + fileOut = FileOutputStream(toolFullPath) + val bos = BufferedOutputStream(fileOut) + val buf = ByteArray(4096) + var length = bis.read(buf) + //保存文件 + while (length != -1) { + bos.write(buf, 0, length) + length = bis.read(buf) + } + bos.close() + bis.close() + conn.disconnect() + + FileUtil.unzipFile(toolFullPath, toolHome) + if (toolBinaryName != ""){ + if ("tar.gz" == suffix) { + FileUtil.unzipTgzFile("$toolHome/$toolBinaryName.$suffix", toolHome) + } else if ("zip" == suffix) { + FileUtil.unzipFile("$toolHome/$toolBinaryName.$suffix", toolHome) + } else if ("tar.xz" == suffix) { + FileUtil.unzipTxzFile("$toolHome/$toolBinaryName.$suffix", toolHome) + } + } + } catch (e: Exception) { + e.printStackTrace() + throw CodeccDependentException("get the download file $toolName failed! please check it!: ${e.message}") + } + return if (File("$toolHome/$toolBinaryName").exists()) { + return "$toolHome/$toolBinaryName" + } else { + return "$toolHome" + } + } + fun download(filePath: String, resultName: String, downloadType: String, landunParam: LandunParam): Boolean { var size = 0L val headers = getHeader(landunParam) diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/pojo/LinuxCodeccConstants.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/pojo/LinuxCodeccConstants.kt index 77973f7..c250fcb 100644 --- a/src/backend/core/src/main/kotlin/com/tencent/devops/pojo/LinuxCodeccConstants.kt +++ b/src/backend/core/src/main/kotlin/com/tencent/devops/pojo/LinuxCodeccConstants.kt @@ -83,7 +83,6 @@ class LinuxCodeccConstants(bkWorkspace: String) { } else { File(THIRD_CODECC_FOLDER, "gometalinter_linux.zip") } - val THIRD_CLANG_FILE = File(THIRD_CODECC_FOLDER, "clang-8.0.zip") val COVRITY_HOME = if (AgentEnv.isThirdParty()) { THIRD_COVERITY_FILE.canonicalPath.removeSuffix(".tar.gz") @@ -97,12 +96,6 @@ class LinuxCodeccConstants(bkWorkspace: String) { File("/data/bkdevops/apps/codecc/kw-analysis/bin") } - val CLANG_PATH = if (AgentEnv.isThirdParty()) { - File(THIRD_CODECC_FOLDER, THIRD_CLANG_FILE.name) - } else { - File("/data/bkdevops/apps/codecc/clang-8.0/bin") - } - val PYTHON2_PATH = if (AgentEnv.isThirdParty()) { if (CodeccEnvHelper.getOS() == OSType.MAC_OS) { File("/usr/bin") diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccParamsHelper.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccParamsHelper.kt index 7186737..93e5cb7 100644 --- a/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccParamsHelper.kt +++ b/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccParamsHelper.kt @@ -329,10 +329,6 @@ object CodeccParamsHelper { } } - fun getClangToolPath(constants: LinuxCodeccConstants): String { - return constants.CLANG_PATH.canonicalPath - } - fun getPyLint2Path(constants: LinuxCodeccConstants): String { return if (CodeccEnvHelper.getOS() != OSType.WINDOWS) { constants.PYLINT2_PATH diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccUtils.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccUtils.kt index 86fe0e0..a5fa8ff 100644 --- a/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccUtils.kt +++ b/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccUtils.kt @@ -49,7 +49,6 @@ import com.tencent.devops.pojo.codeccHost import com.tencent.devops.pojo.exception.CodeccUserConfigException import com.tencent.devops.pojo.imageRegistryPwdKey import com.tencent.devops.utils.CodeccParamsHelper.addCommonParams -import com.tencent.devops.utils.CodeccParamsHelper.getClangToolPath import com.tencent.devops.utils.CodeccParamsHelper.getCovToolPath import com.tencent.devops.utils.CodeccParamsHelper.getGoMetaLinterPath import com.tencent.devops.utils.CodeccParamsHelper.getGoRootPath @@ -216,9 +215,6 @@ open class CodeccUtils { if (!AgentEnv.isThirdParty() && scanTools.contains("KLOCWORK")) { map["KLOCWORK_HOME_BIN"] = getKlocToolPath(constants) } - if (!AgentEnv.isThirdParty() && scanTools.contains("CLANG")) { - map["CLANG_HOME_BIN"] = getClangToolPath(constants) - } if (!param.goPath.isNullOrBlank()) { map["GO_PATH"] = param.goPath!! } @@ -321,7 +317,7 @@ open class CodeccUtils { klockWorkHomeBin = CodeccConfig.getConfig("KLOCWORK_HOME_BIN") ?: getKlocToolPath(constants), pinpointHomeBin = CodeccConfig.getConfig("PINPOINT_HOME_BIN") ?: "/data/bkdevops/apps/codecc/pinpoint", codeqlHomeBin = CodeccConfig.getConfig("CODEQL_HOME_BIN") ?: "/data/bkdevops/apps/codecc/codeql", - clangHomeBin = CodeccConfig.getConfig("CLANG_HOME_BIN") ?: getClangToolPath(constants), + clangHomeBin = CodeccConfig.getConfig("CLANG_HOME_BIN") ?: "/data/codecc_software/clang_scan/clang-11.0/bin", spotBugsHomeBin = workspace.canonicalPath + "/.temp/codecc_scan/codecc_agent/bin/spotbugs/tool/spotbugs-4.0.6/bin", goPath = if (!param.goPath.isNullOrBlank()) { param.goPath!! @@ -477,9 +473,6 @@ open class CodeccUtils { if (!AgentEnv.isThirdParty() && scanTools.contains("KLOCWORK")) command.add( "-DKLOCWORK_HOME_BIN=${getKlocToolPath(constants)}" ) - if (!AgentEnv.isThirdParty() && scanTools.contains("CLANG")) command.add( - "-DCLANG_HOME_BIN=${getClangToolPath(constants)}" - ) if (!param.goPath.isNullOrBlank()) command.add("-DGO_PATH=${param.goPath}") command.add("-DCODECC_API_WEB_SERVER=" + codeccExecuteConfig.atomContext.getSensitiveConfParam("CODECC_API_WEB_SERVER").removePrefix("http://").removeSuffix(":80")) command.add("-DNFS_SERVER=" + codeccExecuteConfig.atomContext.getSensitiveConfParam("NFS_SERVER")) diff --git a/src/backend/core/src/main/resources/config.properties b/src/backend/core/src/main/resources/config.properties index 689dc0a..88c921a 100644 --- a/src/backend/core/src/main/resources/config.properties +++ b/src/backend/core/src/main/resources/config.properties @@ -10,14 +10,14 @@ KLOCWORK_NEW_VERSION=12.3 CODEQL_OLD_VERSION= CODEQL_NEW_VERSION=v2.1.0 CLANG_OLD_VERSION= -CLANG_NEW_VERSION=8.0 +CLANG_NEW_VERSION=11.0 SPOTBUGS_OLD_VERSION= SPOTBUGS_NEW_VERSION=4.0.6 PINPOINT_HOME_BIN=/data/bkdevops/apps/codecc/pinpoint COVERITY_HOME_BIN=/data/bkdevops/apps/coverity/cov-analysis-linux64-2019.06/bin KLOCWORK_HOME_BIN=/data/bkdevops/apps/codecc/kw-analysis/bin CODEQL_HOME_BIN=/data/bkdevops/apps/codecc/codeql -CLANG_HOME_BIN=/data/bkdevops/apps/codecc/clang-8.0/bin +CLANG_HOME_BIN=/data/codecc_software/clang_scan/clang-11.0/bin SPOTBUGS_HOME_BIN=/data/bkdevops/apps/codecc/spotbugs/bin KLOCWORK_SCAN_COMMAND=python3 ./sdk/src/scan.py --input={input.json} --output={output.json} KLOCWORK_TRIGGER_SHELL=python3 ./sdk/src/scan.py --input={input.json} --output={output.json} @@ -97,9 +97,12 @@ RIPS_RUN_TYPE=docker CODEQL_SCAN_COMMAND=python3 ./sdk/src/scan.py --input={input.json} --output={output.json} CODEQL_TRIGGER_SHELL=python3 ./sdk/src/scan.py --input={input.json} --output={output.json} CODEQL_RUN_TYPE=local -CLANG_SCAN_COMMAND=python3 ./sdk/src/scan.py --input={input.json} --output={output.json} -CLANG_TRIGGER_SHELL=python3 ./sdk/src/scan.py --input={input.json} --output={output.json} +CLANG_SCAN_COMMAND=python3 ./src/scan.py --input={input.json} --output={output.json} +CLANG_TRIGGER_SHELL=python3 ./src/scan.py --input={input.json} --output={output.json} +CLANGWARNING_SCAN_COMMAND=python3 ./src/clangwarning.py --input={input.json} --output={output.json} +CLANGWARNING_TRIGGER_SHELL=python3 ./src/clangwarning.py --input={input.json} --output={output.json} CLANG_RUN_TYPE=local +CLANGWARNING_RUN_TYPE=local SPOTBUGS_SCAN_COMMAND=python3 ./src/spotbugs.py --input={input.json} --output={output.json} SPOTBUGS_TRIGGER_SHELL=python3 ./src/spotbugs.py --input={input.json} --output={output.json} SPOTBUGS_RUN_TYPE=local diff --git a/src/frontend/src/components/Basic.vue b/src/frontend/src/components/Basic.vue index 93a3bd4..cc4321d 100644 --- a/src/frontend/src/components/Basic.vue +++ b/src/frontend/src/components/Basic.vue @@ -83,7 +83,7 @@ { id: 'script', label: '编译脚本', - rely: ['COVERITY', 'KLOCWORK', 'PINPOINT', 'CODEQL', 'CLANG', 'SPOTBUGS'], + rely: ['COVERITY', 'KLOCWORK', 'PINPOINT', 'CODEQL', 'CLANG', 'CLANGWARNING', 'SPOTBUGS'], item: ['scriptType', 'script'] } ],