Merge pull request #6 from JamiKX1/master

feat: 新增clang和clangwarning工具 #5
TencentBlueKing · Oct 15, 2021 · 328f5fe · 328f5fe
2 parents fec6e3d + 9f6403e
commit 328f5fe
Show file tree

Hide file tree

Showing 13 changed files with 132 additions and 30 deletions.
diff --git a/src/backend/core/build.gradle b/src/backend/core/build.gradle
@@ -92,7 +92,7 @@ allprojects {
         compile "org.hashids:hashids:1.0.3"
         compile group: 'commons-codec', name: 'commons-codec', version: '1.15'
         compile("org.apache.commons:commons-compress:1.15")
-
+        compile "org.tukaani:xz:1.2"
         testCompile group: 'junit', name: 'junit', version: '4.12'
     }
 

diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/Build.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/Build.kt
@@ -131,7 +131,7 @@ object Build {
 //            runCompileTools = false //just for test
             if (runCompileTools) {
                 // 编译型工具，串行
-                CodeccWeb.downloadAnyUnzip(commandParam)
+                //CodeccWeb.downloadAnyUnzip(commandParam)
                 executor.execute {
                     var curTool = ""
                     try {

diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/Scan.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/Scan.kt
@@ -50,8 +50,7 @@ class Scan(
         LogUtils.printLog("toolName: $toolName")
         LogUtils.printLog("compile_tools: ${ToolConstants.COMPILE_TOOLS}")
         if (toolName in ToolConstants.COMPILE_TOOLS) {
-            val toolFolder = commandParam.projectBuildPath + File.separator + ".temp" + File.separator + "codecc_scan" +
-                File.separator + "codecc_agent" + File.separator + "bin" + File.separator + toolName
+            val toolFolder = "/data/codecc_software/${toolName}_scan"
             val command = CodeccConfig.getConfig("${toolName.toUpperCase()}_SCAN_COMMAND")!!
                 .replace("##", " ")
                 .replace("{input.json}", inputFile)

diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/ScanComposer.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/ScanComposer.kt
@@ -76,6 +76,7 @@ object ScanComposer {
         }
         // check third env
         CodeccConfig.checkThirdEnv(commandParam, toolName)
+        CodeccConfig.downloadToolZip(commandParam, toolName)
 
         // 排队开始
         CodeccWeb.codeccUploadTaskLog(analyzeConfigInfo.taskId, streamName, toolName, commandParam.landunParam, 1, 3)

diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/pojo/ToolConstants.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/pojo/ToolConstants.kt
@@ -11,10 +11,11 @@ object ToolConstants {
     const val CLOC = "cloc"
     const val SCC = "scc"
     const val CLANG = "clang"
+    const val CLANGWARNING = "clangwarning"
     const val SPOTBUGS = "spotbugs"
     const val GITHUBSTATISTIC = "githubstatistic"
     const val RIPS = "rips"
-    val COMPILE_TOOLS = listOf(COVERITY, KLOCWORK, PINPOINT,CODEQL, CLANG, SPOTBUGS)
+    val COMPILE_TOOLS = listOf(COVERITY, KLOCWORK, PINPOINT,CODEQL, CLANG, CLANGWARNING, SPOTBUGS)
     val NOLINT_TOOLS = listOf(DUPC, CCN, TSCCN, CLOC, SCC, COVERITY, KLOCWORK, PINPOINT, GITHUBSTATISTIC)
     val CODE_TOOLS_ACOUNT = listOf(CLOC, SCC)
 }
diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/tools/FileUtil.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/tools/FileUtil.kt
@@ -30,6 +30,7 @@ import org.apache.commons.codec.digest.DigestUtils
 import org.apache.commons.compress.archivers.tar.TarArchiveInputStream
 import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
+import org.apache.commons.compress.compressors.xz.XZCompressorInputStream
 import java.io.BufferedInputStream
 import java.io.BufferedOutputStream
 import java.io.File
@@ -190,6 +191,27 @@ object FileUtil {
         }
     }
 
+    fun unzipTxzFile(txzFile: String, destDir: String = "./") {
+        val blockSize = 4096
+        val inputStream = TarArchiveInputStream(XZCompressorInputStream(File(txzFile).inputStream()), blockSize)
+        while (true) {
+            val entry = inputStream.nextTarEntry ?: break
+            if (entry.isDirectory) { // 是目录
+                val dir = File(destDir, entry.name)
+                if (!dir.exists()) dir.mkdirs()
+            } else { // 是文件
+                File(destDir, entry.name).outputStream().use { outputStream ->
+                    while (true) {
+                        val buf = ByteArray(4096)
+                        val len = inputStream.read(buf)
+                        if (len == -1) break
+                        outputStream.write(buf, 0, len)
+                    }
+                }
+            }
+        }
+    }
+
     fun unzipFile(zipFile: String, destDir: String = "./") {
         val blockSize = 4096
         val inputStream = ZipArchiveInputStream(BufferedInputStream(FileInputStream(File(zipFile)), blockSize))

diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccConfig.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccConfig.kt
@@ -13,6 +13,7 @@ import com.tencent.devops.docker.tools.LogUtils
 import com.tencent.devops.pojo.OSType
 import com.tencent.devops.pojo.exception.CodeccUserConfigException
 import com.tencent.devops.utils.CodeccEnvHelper
+import com.tencent.devops.utils.script.CommandLineUtils
 import com.tencent.devops.utils.script.ScriptUtils
 import java.io.File
 import java.util.*
@@ -188,6 +189,32 @@ object CodeccConfig {
     fun checkThirdEnv(commandParam: CommandParam, toolName: String) {
     }
 
+    @Synchronized
+    fun downloadToolZip(commandParam: CommandParam, toolName: String) {
+        val softwareRootPath = "/data/codecc_software"
+        if (!File(softwareRootPath).exists()) {
+            File(softwareRootPath).mkdirs()
+        }
+        var toolSourceName = ""
+        var toolBinaryName = ""
+        var suffix = ""
+        if ((toolName in ToolConstants.COMPILE_TOOLS) && !checkPythonAll(softwareRootPath)) {
+            throw CodeccUserConfigException("Codecc need install python3")
+        }
+        if (ToolConstants.CLANG == toolName) {
+            toolSourceName = "clang_scan.zip"
+            toolBinaryName = "clang-${getConfig("CLANG_NEW_VERSION")}"
+            suffix = "tar.xz"
+            val clangVersion = toolBinaryName.replace(Regex("\\.\\d*"), "")
+            commandParam.clangHomeBin = CodeccWeb.downloadCompileTool(toolName, toolSourceName, toolBinaryName, suffix)
+            CommandLineUtils.execute("chmod -R 755 ${commandParam.clangHomeBin}", File("."), true)
+            ScriptUtils.execute("ln -s -f $clangVersion clang;ln -s -f clang clang++", File("$softwareRootPath/clang_scan/$toolBinaryName/bin"))
+        }else if(ToolConstants.CLANGWARNING == toolName) {
+            toolSourceName = "clangwarning_scan.zip"
+            CodeccWeb.downloadCompileTool(toolName, toolSourceName, toolBinaryName, suffix)
+        }
+    }
+
     private fun checkPython(home: String): Boolean {
         val cmd = "python3 --version"
         val output = try {

diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccWeb.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/docker/utils/CodeccWeb.kt
@@ -6,7 +6,6 @@ import com.fasterxml.jackson.module.kotlin.readValue
 import com.tencent.bk.devops.atom.api.BaseApi
 import com.tencent.bk.devops.plugin.utils.JsonUtil
 import com.tencent.bk.devops.plugin.utils.OkhttpUtils
-import com.tencent.devops.docker.ScanComposer
 import com.tencent.devops.docker.pojo.*
 import com.tencent.devops.docker.tools.FileUtil
 import com.tencent.devops.docker.tools.LogUtils
@@ -25,7 +24,12 @@ import org.apache.commons.codec.digest.DigestUtils
 import java.io.File
 import java.io.FileInputStream
 import java.io.FileOutputStream
+import java.io.InputStream
+import java.io.BufferedOutputStream
+import java.io.BufferedInputStream
 import java.io.IOException
+import java.net.HttpURLConnection
+import java.net.URL
 
 object CodeccWeb : BaseApi() {
 
@@ -74,6 +78,69 @@ object CodeccWeb : BaseApi() {
         return binFolder
     }
 
+    fun downloadCompileTool(toolName: String, toolSourceZip: String, toolBinaryName: String, suffix: String) : String{
+        var fileOut: FileOutputStream?
+        var conn: HttpURLConnection?
+        var inputStream: InputStream?
+        var rootPath = "/data/codecc_software"
+        var url = "https://hub.fastgit.org/TencentBlueKing/codeccScan/raw/master/${toolName}_scan/$toolSourceZip"
+        val toolHome = "$rootPath/${toolName}_scan"
+        if (File(toolHome).exists() && File(toolHome).list()?.isNotEmpty() == true) {
+            if (File("$toolHome/$toolBinaryName").exists()) {
+                return "$toolHome/$toolBinaryName"
+            }
+        }
+        if (!File(rootPath).exists()) {
+            File(rootPath).mkdirs()
+        }
+        try {
+            LogUtils.printLog("start to download tool zip")
+            val httpUrl = URL(url)
+            conn = httpUrl.openConnection() as HttpURLConnection
+            conn.setRequestMethod("GET")
+            conn.setDoInput(true)
+            conn.setDoOutput(true)
+            conn.setUseCaches(false)
+            conn.connect()
+            inputStream = conn.getInputStream()
+            val bis = BufferedInputStream(inputStream)
+            val toolFullPath = rootPath + File.separator + toolSourceZip
+            fileOut = FileOutputStream(toolFullPath)
+            val bos = BufferedOutputStream(fileOut)
+            val buf = ByteArray(4096)
+            var length = bis.read(buf)
+            //保存文件
+            while (length != -1) {
+                bos.write(buf, 0, length)
+                length = bis.read(buf)
+            }
+            bos.close()
+            bis.close()
+            conn.disconnect()
+
+            LogUtils.printLog("start to unzip tool and tool source zip")
+            FileUtil.unzipFile(toolFullPath, toolHome)
+            if (toolBinaryName != ""){
+                if ("tar.gz" == suffix) {
+                    FileUtil.unzipTgzFile("$toolHome/$toolBinaryName.$suffix", toolHome)
+                } else if ("zip" == suffix) {
+                    FileUtil.unzipFile("$toolHome/$toolBinaryName.$suffix", toolHome)
+                } else if ("tar.xz" == suffix) {
+                    FileUtil.unzipTxzFile("$toolHome/$toolBinaryName.$suffix", toolHome)
+                }
+            }
+        } catch (e: Exception) {
+            e.printStackTrace()
+            throw CodeccDependentException("get the download file $toolName failed! please check it!: ${e.message}")
+        }
+        LogUtils.printLog("download tool zip done.")
+        return if (File("$toolHome/$toolBinaryName").exists()) {
+            return "$toolHome/$toolBinaryName"
+        } else {
+            return "$toolHome"
+        }
+    }
+
     fun download(filePath: String, resultName: String, downloadType: String, landunParam: LandunParam): Boolean {
         var size = 0L
         val headers = getHeader(landunParam)

diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/pojo/LinuxCodeccConstants.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/pojo/LinuxCodeccConstants.kt
@@ -83,7 +83,6 @@ class LinuxCodeccConstants(bkWorkspace: String) {
     } else {
         File(THIRD_CODECC_FOLDER, "gometalinter_linux.zip")
     }
-    val THIRD_CLANG_FILE = File(THIRD_CODECC_FOLDER, "clang-8.0.zip")
 
     val COVRITY_HOME = if (AgentEnv.isThirdParty()) {
         THIRD_COVERITY_FILE.canonicalPath.removeSuffix(".tar.gz")
@@ -97,12 +96,6 @@ class LinuxCodeccConstants(bkWorkspace: String) {
         File("/data/bkdevops/apps/codecc/kw-analysis/bin")
     }
 
-    val CLANG_PATH = if (AgentEnv.isThirdParty()) {
-        File(THIRD_CODECC_FOLDER, THIRD_CLANG_FILE.name)
-    } else {
-        File("/data/bkdevops/apps/codecc/clang-8.0/bin")
-    }
-
     val PYTHON2_PATH = if (AgentEnv.isThirdParty()) {
         if (CodeccEnvHelper.getOS() == OSType.MAC_OS) {
             File("/usr/bin")

diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccParamsHelper.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccParamsHelper.kt
@@ -329,10 +329,6 @@ object CodeccParamsHelper {
         }
     }
 
-    fun getClangToolPath(constants: LinuxCodeccConstants): String {
-        return constants.CLANG_PATH.canonicalPath
-    }
-
     fun getPyLint2Path(constants: LinuxCodeccConstants): String {
         return if (CodeccEnvHelper.getOS() != OSType.WINDOWS) {
             constants.PYLINT2_PATH

diff --git a/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccUtils.kt b/src/backend/core/src/main/kotlin/com/tencent/devops/utils/CodeccUtils.kt
@@ -49,7 +49,6 @@ import com.tencent.devops.pojo.codeccHost
 import com.tencent.devops.pojo.exception.CodeccUserConfigException
 import com.tencent.devops.pojo.imageRegistryPwdKey
 import com.tencent.devops.utils.CodeccParamsHelper.addCommonParams
-import com.tencent.devops.utils.CodeccParamsHelper.getClangToolPath
 import com.tencent.devops.utils.CodeccParamsHelper.getCovToolPath
 import com.tencent.devops.utils.CodeccParamsHelper.getGoMetaLinterPath
 import com.tencent.devops.utils.CodeccParamsHelper.getGoRootPath
@@ -216,9 +215,6 @@ open class CodeccUtils {
         if (!AgentEnv.isThirdParty() && scanTools.contains("KLOCWORK")) {
             map["KLOCWORK_HOME_BIN"] = getKlocToolPath(constants)
         }
-        if (!AgentEnv.isThirdParty() && scanTools.contains("CLANG")) {
-            map["CLANG_HOME_BIN"] = getClangToolPath(constants)
-        }
         if (!param.goPath.isNullOrBlank()) {
             map["GO_PATH"] = param.goPath!!
         }
@@ -321,7 +317,7 @@ open class CodeccUtils {
             klockWorkHomeBin = CodeccConfig.getConfig("KLOCWORK_HOME_BIN") ?: getKlocToolPath(constants),
             pinpointHomeBin = CodeccConfig.getConfig("PINPOINT_HOME_BIN") ?: "/data/bkdevops/apps/codecc/pinpoint",
             codeqlHomeBin = CodeccConfig.getConfig("CODEQL_HOME_BIN") ?: "/data/bkdevops/apps/codecc/codeql",
-            clangHomeBin = CodeccConfig.getConfig("CLANG_HOME_BIN") ?: getClangToolPath(constants),
+            clangHomeBin = CodeccConfig.getConfig("CLANG_HOME_BIN") ?: "/data/codecc_software/clang_scan/clang-11.0/bin",
             spotBugsHomeBin = workspace.canonicalPath + "/.temp/codecc_scan/codecc_agent/bin/spotbugs/tool/spotbugs-4.0.6/bin",
             goPath = if (!param.goPath.isNullOrBlank()) {
                 param.goPath!!
@@ -477,9 +473,6 @@ open class CodeccUtils {
         if (!AgentEnv.isThirdParty() && scanTools.contains("KLOCWORK")) command.add(
             "-DKLOCWORK_HOME_BIN=${getKlocToolPath(constants)}"
         )
-        if (!AgentEnv.isThirdParty() && scanTools.contains("CLANG")) command.add(
-            "-DCLANG_HOME_BIN=${getClangToolPath(constants)}"
-        )
         if (!param.goPath.isNullOrBlank()) command.add("-DGO_PATH=${param.goPath}")
         command.add("-DCODECC_API_WEB_SERVER=" + codeccExecuteConfig.atomContext.getSensitiveConfParam("CODECC_API_WEB_SERVER").removePrefix("http://").removeSuffix(":80"))
         command.add("-DNFS_SERVER=" + codeccExecuteConfig.atomContext.getSensitiveConfParam("NFS_SERVER"))

diff --git a/src/backend/core/src/main/resources/config.properties b/src/backend/core/src/main/resources/config.properties
@@ -10,14 +10,14 @@ KLOCWORK_NEW_VERSION=12.3
 CODEQL_OLD_VERSION=
 CODEQL_NEW_VERSION=v2.1.0
 CLANG_OLD_VERSION=
-CLANG_NEW_VERSION=8.0
+CLANG_NEW_VERSION=11.0
 SPOTBUGS_OLD_VERSION=
 SPOTBUGS_NEW_VERSION=4.0.6
 PINPOINT_HOME_BIN=/data/bkdevops/apps/codecc/pinpoint
 COVERITY_HOME_BIN=/data/bkdevops/apps/coverity/cov-analysis-linux64-2019.06/bin
 KLOCWORK_HOME_BIN=/data/bkdevops/apps/codecc/kw-analysis/bin
 CODEQL_HOME_BIN=/data/bkdevops/apps/codecc/codeql
-CLANG_HOME_BIN=/data/bkdevops/apps/codecc/clang-8.0/bin
+CLANG_HOME_BIN=/data/codecc_software/clang_scan/clang-11.0/bin
 SPOTBUGS_HOME_BIN=/data/bkdevops/apps/codecc/spotbugs/bin
 KLOCWORK_SCAN_COMMAND=python3 ./sdk/src/scan.py --input={input.json} --output={output.json}
 KLOCWORK_TRIGGER_SHELL=python3 ./sdk/src/scan.py --input={input.json} --output={output.json}
@@ -97,9 +97,12 @@ RIPS_RUN_TYPE=docker
 CODEQL_SCAN_COMMAND=python3 ./sdk/src/scan.py --input={input.json} --output={output.json}
 CODEQL_TRIGGER_SHELL=python3 ./sdk/src/scan.py --input={input.json} --output={output.json}
 CODEQL_RUN_TYPE=local
-CLANG_SCAN_COMMAND=python3 ./sdk/src/scan.py --input={input.json} --output={output.json}
-CLANG_TRIGGER_SHELL=python3 ./sdk/src/scan.py --input={input.json} --output={output.json}
+CLANG_SCAN_COMMAND=python3 ./src/scan.py --input={input.json} --output={output.json}
+CLANG_TRIGGER_SHELL=python3 ./src/scan.py --input={input.json} --output={output.json}
+CLANGWARNING_SCAN_COMMAND=python3 ./src/clangwarning.py --input={input.json} --output={output.json}
+CLANGWARNING_TRIGGER_SHELL=python3 ./src/clangwarning.py --input={input.json} --output={output.json}
 CLANG_RUN_TYPE=local
+CLANGWARNING_RUN_TYPE=local
 SPOTBUGS_SCAN_COMMAND=python3 ./src/spotbugs.py --input={input.json} --output={output.json}
 SPOTBUGS_TRIGGER_SHELL=python3 ./src/spotbugs.py --input={input.json} --output={output.json}
 SPOTBUGS_RUN_TYPE=local

diff --git a/src/frontend/src/components/Basic.vue b/src/frontend/src/components/Basic.vue
@@ -83,7 +83,7 @@
                     {
                         id: 'script',
                         label: '编译脚本',
-                        rely: ['COVERITY', 'KLOCWORK', 'PINPOINT', 'CODEQL', 'CLANG', 'SPOTBUGS'],
+                        rely: ['COVERITY', 'KLOCWORK', 'PINPOINT', 'CODEQL', 'CLANG', 'CLANGWARNING', 'SPOTBUGS'],
                         item: ['scriptType', 'script']
                     }
                 ],