diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/basic/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/basic/mgr.go index 0acb0bce..30e57958 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/basic/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/basic/mgr.go @@ -523,7 +523,7 @@ func (m *Mgr) GetToolChainTimestamp(key string) (int64, error) { } // SearchToolChain search toolchain files by cmd, ensure only execute once -func (m *Mgr) SearchToolChain(cmd string) error { +func (m *Mgr) SearchToolChain(cmd, path string) error { m.searchToolchainLock.Lock() defer m.searchToolchainLock.Unlock() @@ -535,7 +535,7 @@ func (m *Mgr) SearchToolChain(cmd string) error { m.searchToolChainCache[cmd] = true // TODO : search toolchain with cmd now - toolchain, err := searchToolChain(cmd) + toolchain, err := searchToolChain(cmd, path) if err == nil && toolchain != nil { m.SetToolChain(toolchain) } diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/basic/utils.go b/src/backend/booster/bk_dist/controller/pkg/manager/basic/utils.go index 354e9c25..d7f2149e 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/basic/utils.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/basic/utils.go @@ -216,18 +216,18 @@ func getToolchainID() string { // ++++++++++++++++++++to search toolchain++++++++++++++++++++ -func searchToolChain(cmd string) (*types.ToolChain, error) { - blog.Infof("basic: real start search toolchian for cmd:%s", cmd) - defer blog.Infof("basic: end search toolchian for cmd:%s", cmd) +func searchToolChain(cmd, path string) (*types.ToolChain, error) { + blog.Infof("basic: real start search toolchian for cmd:%s path:%s", cmd, path) + defer blog.Infof("basic: end search toolchian for cmd:%s path:%s", cmd, path) cmdbase := filepath.Base(cmd) switch cmdbase { case "clang", "clang++": - return searchClang(cmd) + return searchClang(cmd, path) case "gcc", "g++": - return searchGcc(cmd) + return searchGcc(cmd, path) case "cc", "c++": - return searchGcc(cmd) + return searchGcc(cmd, path) } return nil, nil @@ -266,7 +266,13 @@ func searchCC(exe string) []string { return []string{cc1, cc1plus} } -func searchAS(exe string) []string { +func searchAS(exe, path string) []string { + as, err := dcUtil.LookPath("as", path, "") + if err == nil { + blog.Infof("basic: found as:[%s] with path:%s", as, path) + return []string{as} + } + cmd := fmt.Sprintf("%s -print-prog-name=as", exe) blog.Infof("basic: ready run cmd:[%s] for exe:%s", cmd, exe) sandbox := dcSyscall.Sandbox{} @@ -347,8 +353,8 @@ func getLddFiles(exe string) []string { return files } -func searchGcc(cmd string) (*types.ToolChain, error) { - blog.Infof("basic: search gcc toolchain with exe:%s", cmd) +func searchGcc(cmd, path string) (*types.ToolChain, error) { + blog.Infof("basic: search gcc toolchain with exe:%s path:%s", cmd, path) i := dcFile.Lstat(cmd) if !i.Exist() { @@ -379,7 +385,7 @@ func searchGcc(cmd string) (*types.ToolChain, error) { } // search as - fs = searchAS(cmd) + fs = searchAS(cmd, path) for _, i := range fs { t.Files = append(t.Files, dcSDK.ToolFile{ LocalFullPath: i, @@ -483,8 +489,8 @@ func searchCrtbegin(exe string) []string { return nil } -func searchClang(cmd string) (*types.ToolChain, error) { - blog.Infof("basic: search clang toolchain with exe:%s", cmd) +func searchClang(cmd, path string) (*types.ToolChain, error) { + blog.Infof("basic: search clang toolchain with exe:%s path:%s", cmd, path) i := dcFile.Lstat(cmd) if !i.Exist() { diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go index a03533d8..7d84ff15 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/remote/mgr.go @@ -1804,8 +1804,9 @@ func (m *Mgr) getToolChainFromExecuteRequest(req *types.RemoteTaskExecuteRequest } else { // TODO : 如果环境变量中指定了需要自动探测工具链,则需要自动探测 if dcSyscall.NeedSearchToolchain(req.Sandbox.Env) { - blog.Infof("remote: start search toolchain with key:%s now", c.ExeToolChainKey) - err := m.work.Basic().SearchToolChain(c.ExeToolChainKey) + path := req.Sandbox.Env.GetOriginEnv("PATH") + blog.Infof("remote: start search toolchain with key:%s path:%s", c.ExeToolChainKey, path) + err := m.work.Basic().SearchToolChain(c.ExeToolChainKey, path) blog.Infof("remote: end search toolchain with key:%s error:%v", c.ExeToolChainKey, err) if err == nil { diff --git a/src/backend/booster/bk_dist/controller/pkg/types/interface.go b/src/backend/booster/bk_dist/controller/pkg/types/interface.go index 628b2d59..a9cb65a9 100644 --- a/src/backend/booster/bk_dist/controller/pkg/types/interface.go +++ b/src/backend/booster/bk_dist/controller/pkg/types/interface.go @@ -270,5 +270,5 @@ type BasicMgr interface { DecRegistered() // search toolchain files by cmd, ensure only execute once - SearchToolChain(cmd string) error + SearchToolChain(cmd, path string) error } diff --git a/src/backend/booster/bk_dist/handler/cc/handler.go b/src/backend/booster/bk_dist/handler/cc/handler.go index eac7300c..1fb32c4c 100644 --- a/src/backend/booster/bk_dist/handler/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/cc/handler.go @@ -40,6 +40,7 @@ import ( const ( appendEnvKey = "INCLUDE=" + pathEnvKey = "PATH=" ) var ( @@ -380,6 +381,8 @@ func (cc *TaskCC) copyPumpHeadFile(workdir string) error { uniqlines = append(linkdirs, uniqlines...) } + uniqlines = dcUtil.UniqArr(uniqlines) + // save to cc.pumpHeadFile newdata := strings.Join(uniqlines, sep) err = os.WriteFile(cc.pumpHeadFile, []byte(newdata), os.ModePerm) @@ -737,14 +740,11 @@ func (cc *TaskCC) trypumpwithcache(command []string) (*dcSDK.BKDistCommand, erro results = append(results, sourcedependfile) } - // set env which need append to remote + // sync env of PATH to remote envs := []string{} for _, v := range cc.sandbox.Env.Source() { - if strings.HasPrefix(v, appendEnvKey) { + if strings.HasPrefix(v, pathEnvKey) { envs = append(envs, v) - // set flag we hope append env, not overwrite - flag := fmt.Sprintf("%s=true", dcEnv.GetEnvKey(env.KeyRemoteEnvAppend)) - envs = append(envs, flag) break } } @@ -960,6 +960,16 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKD } blog.Infof("cc: [%s] expect result files: %v", cc.tag, cc.outputFile) + // sync env of PATH to remote + envs := []string{} + for _, v := range cc.sandbox.Env.Source() { + if strings.HasPrefix(v, pathEnvKey) { + envs = append(envs, v) + break + } + } + blog.Infof("cc: env which ready sent to remote:[%v]", envs) + return &dcSDK.BKDistCommand{ Commands: []dcSDK.BKCommand{ { @@ -970,6 +980,7 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKD Params: cc.serverSideArgs[1:], Inputfiles: cc.sendFiles, ResultFiles: cc.outputFile, + Env: envs, }, }, CustomSave: true, @@ -1234,6 +1245,7 @@ func (cc *TaskCC) preBuild(args []string) error { cc.firstIncludeFile = getFirstIncludeFile(scannedData.args) cc.inputFile = scannedData.inputFile cc.outputFile = append([]string{scannedData.outputFile}, scannedData.additionOutputFile...) + cc.includePaths = scannedData.includePaths // handle the cross-compile issues. targetArgs := cc.scannedArgs diff --git a/src/backend/booster/bk_dist/handler/cc/utils.go b/src/backend/booster/bk_dist/handler/cc/utils.go index e35fbc95..862f6feb 100644 --- a/src/backend/booster/bk_dist/handler/cc/utils.go +++ b/src/backend/booster/bk_dist/handler/cc/utils.go @@ -428,6 +428,7 @@ var ( "-imacros": true, "-iprefix": true, "-iwithprefix": true, + "-iquote": true, "-isystem": true, "-iwithprefixbefore": true, "-idirafter": true, @@ -456,6 +457,7 @@ var ( "-MF": true, "-MT": true, "-MQ": true, + "-iquote": true, "-isystem": true, "@": true, // such as @"..\XXX\XXX.rsp" "--gcc-toolchain": true, @@ -780,6 +782,40 @@ func scanArgs(args []string, sandbox *dcSyscall.Sandbox) (*ccArgs, error) { continue } + if strings.HasPrefix(arg, "-iquote") { + // if -I just a prefix, save the remain of this line. + if len(arg) > 7 { + r.includePaths = append(r.includePaths, strings.Trim(arg[7:], "\"")) + continue + } + + // if file name is in the next index, then take it. + index++ + if index >= len(args) { + blog.Warnf("cc: scan args: no file found after -iquote") + return nil, ErrorMissingOption + } + r.includePaths = append(r.includePaths, strings.Trim(args[index], "\"")) + continue + } + + if strings.HasPrefix(arg, "-isystem") { + // if -I just a prefix, save the remain of this line. + if len(arg) > 8 { + r.includePaths = append(r.includePaths, strings.Trim(arg[8:], "\"")) + continue + } + + // if file name is in the next index, then take it. + index++ + if index >= len(args) { + blog.Warnf("cc: scan args: no file found after -isystem") + return nil, ErrorMissingOption + } + r.includePaths = append(r.includePaths, strings.Trim(args[index], "\"")) + continue + } + if strings.HasPrefix(arg, "-include") { keylen := 8 if arg == "-include-pch" {