diff --git a/src/backend/booster/bk_dist/booster/command/command.go b/src/backend/booster/bk_dist/booster/command/command.go index a25c72a6..e4693652 100644 --- a/src/backend/booster/bk_dist/booster/command/command.go +++ b/src/backend/booster/bk_dist/booster/command/command.go @@ -23,97 +23,98 @@ import ( // define const vars const ( - FlagProjectID = "project_id" - FlagBuildID = "build_id" - FlagArgs = "args" - FlagGccVersion = "gcc_version" - FlagBazel = "bazel" - FlagBazelPlus = "bazel_plus" - FlagBazel4Plus = "bazel4_plus" - FlagBazelNoLauncher = "bazel_no_launcher" - FlagLauncher = "launcher" - FlagLog = "log" - FlagLogDir = "log_dir" - FlagLogToConsole = "log_to_console" - FlagExecutorLog = "executor_log" - FlagTest = "test" - FlagCommandPath = "command_path" - FlagLimit = "limit" - FlagNoLocal = "no_local" - FlagLocal = "local" - FlagDegraded = "degraded" - FlagSaveCode = "save_code" - FlagJobs = "jobs" - FlagMaxJobs = "max_jobs" - FlagPresetJobs = "preset_jobs" - FlagMaxDegradedJobs = "max_degraded_jobs" - FlagMaxLocalTotalJobs = "max_local_total_jobs" - FlagMaxLocalExeJobs = "max_local_exe_jobs" - FlagMaxLocalPreJobs = "max_local_pre_jobs" - FlagMaxLocalPostJobs = "max_local_post_jobs" - FlagHook = "hook" - FlagBoosterType = "booster_type" - FlagHookConfig = "hook_config" - FlagHookPreloadLib = "hook_preload_library" - FlagAdditionFile = "addition_file" - FlagDashboard = "dashboard" - FlagWorkerList = "worker_list" - FlagCheckMd5 = "check_md5" - FlagOutputEnvJSONFile = "output_env_json_file" - FlagOutputEnvSourceFile = "output_env_source_file" - FlagCommitSuicide = "commit_suicide" - FlagToolChainJSONFile = "tool_chain_json_file" - FlagSearchToolchain = "search_toolchain" - FlagBatchMode = "batch_mode" - FlagDirectives = "directives" - FlagGlobalSlots = "global_slots" - FlagSudoController = "sudo_controller" - FlagIOTimeoutSecs = "io_timeout_secs" - FlagPump = "pump" - FlagPumpDisableMacro = "pump_disable_macro" - FlagPumpIncludeSysHeader = "pump_include_sys_header" - FlagPumpCheck = "pump_check" - FlagPumpCache = "pump_cache" - FlagPumpCacheDir = "pump_cache_dir" - FlagPumpCacheSizeMaxMB = "pump_cache_size_max_MB" - FlagPumpCacheRemoveAll = "pump_cache_remove_all" - FlagPumpBlackList = "pump_black_list" - FlagPumpMinActionNum = "pump_min_action_num" - FlagPumpDisableStatCache = "pump_disable_stat_cache" - FlagPumpSearchLink = "pump_search_link" - FlagPumpSearchLinkFile = "pump_search_link_file" - FlagPumpSearchLinkDir = "pump_search_link_dir" - FlagPumpLstatByDir = "pump_lstat_by_dir" - FlagPumpCorrectCap = "pump_correct_cap" - FlagForceLocalList = "force_local_list" - FlagNoWork = "no_work" - FlagControllerNoWait = "controller_no_wait" - FlagControllerRemainTime = "controller_remain_time" - FlagServer = "server" - FlagCacheServer = "cache_server" - FlagWorkerSideCache = "worker_side_cache" - FlagLocalRecord = "local_record" - FlagWriteMemroMemroy = "write_memory" - FlagIdleKeepSecs = "idle_keep_secs" - FlagResourceTimeoutSecs = "resource_timeout_secs" - FlagLocalIdleCPUPercent = "use_local_cpu_percent" - FlagDisableFileLock = "disable_file_lock" - FlagAutoResourceMgr = "auto_resource_mgr" - FlagResIdleSecsForFree = "res_idle_secs_for_free" - FlagSendCork = "send_cork" - FlagSendFileMemoryLimit = "send_file_memory_limit" - FlagSendMemoryCache = "send_memory_cache" - FlagNetErrorLimit = "net_error_limit" - FlagRemoteRetryTimes = "remote_retry_times" - FlagEnableLink = "enable_link" - FlagEnableLib = "enable_lib" - FlagLongTCP = "long_tcp" - FlagUseDefaultWorker = "use_default_worker" - FlagDynamicPort = "dynamic_port" - FlagWorkerOfferSlot = "worker_offer_slot" - FlagCleanTmpFilesDayAgo = "clean_tmp_files_day_ago" - FlagIgnoreHttpStatus = "ignore_http_status" - FlagResultCacheType = "result_cache_type" + FlagProjectID = "project_id" + FlagBuildID = "build_id" + FlagArgs = "args" + FlagGccVersion = "gcc_version" + FlagBazel = "bazel" + FlagBazelPlus = "bazel_plus" + FlagBazel4Plus = "bazel4_plus" + FlagBazelNoLauncher = "bazel_no_launcher" + FlagLauncher = "launcher" + FlagLog = "log" + FlagLogDir = "log_dir" + FlagLogToConsole = "log_to_console" + FlagExecutorLog = "executor_log" + FlagTest = "test" + FlagCommandPath = "command_path" + FlagLimit = "limit" + FlagNoLocal = "no_local" + FlagLocal = "local" + FlagDegraded = "degraded" + FlagSaveCode = "save_code" + FlagJobs = "jobs" + FlagMaxJobs = "max_jobs" + FlagPresetJobs = "preset_jobs" + FlagMaxDegradedJobs = "max_degraded_jobs" + FlagMaxLocalTotalJobs = "max_local_total_jobs" + FlagMaxLocalExeJobs = "max_local_exe_jobs" + FlagMaxLocalPreJobs = "max_local_pre_jobs" + FlagMaxLocalPostJobs = "max_local_post_jobs" + FlagHook = "hook" + FlagBoosterType = "booster_type" + FlagHookConfig = "hook_config" + FlagHookPreloadLib = "hook_preload_library" + FlagAdditionFile = "addition_file" + FlagDashboard = "dashboard" + FlagWorkerList = "worker_list" + FlagCheckMd5 = "check_md5" + FlagOutputEnvJSONFile = "output_env_json_file" + FlagOutputEnvSourceFile = "output_env_source_file" + FlagCommitSuicide = "commit_suicide" + FlagToolChainJSONFile = "tool_chain_json_file" + FlagSearchToolchain = "search_toolchain" + FlagBatchMode = "batch_mode" + FlagDirectives = "directives" + FlagGlobalSlots = "global_slots" + FlagSudoController = "sudo_controller" + FlagIOTimeoutSecs = "io_timeout_secs" + FlagPump = "pump" + FlagPumpDisableMacro = "pump_disable_macro" + FlagPumpIncludeSysHeader = "pump_include_sys_header" + FlagPumpCheck = "pump_check" + FlagPumpCache = "pump_cache" + FlagPumpCacheDir = "pump_cache_dir" + FlagPumpCacheSizeMaxMB = "pump_cache_size_max_MB" + FlagPumpCacheRemoveAll = "pump_cache_remove_all" + FlagPumpBlackList = "pump_black_list" + FlagPumpMinActionNum = "pump_min_action_num" + FlagPumpDisableStatCache = "pump_disable_stat_cache" + FlagPumpSearchLink = "pump_search_link" + FlagPumpSearchLinkFile = "pump_search_link_file" + FlagPumpSearchLinkDir = "pump_search_link_dir" + FlagPumpLstatByDir = "pump_lstat_by_dir" + FlagPumpCorrectCap = "pump_correct_cap" + FlagForceLocalList = "force_local_list" + FlagNoWork = "no_work" + FlagControllerNoWait = "controller_no_wait" + FlagControllerRemainTime = "controller_remain_time" + FlagServer = "server" + FlagCacheServer = "cache_server" + FlagWorkerSideCache = "worker_side_cache" + FlagLocalRecord = "local_record" + FlagWriteMemroMemroy = "write_memory" + FlagIdleKeepSecs = "idle_keep_secs" + FlagResourceTimeoutSecs = "resource_timeout_secs" + FlagLocalIdleCPUPercent = "use_local_cpu_percent" + FlagDisableFileLock = "disable_file_lock" + FlagAutoResourceMgr = "auto_resource_mgr" + FlagResIdleSecsForFree = "res_idle_secs_for_free" + FlagSendCork = "send_cork" + FlagSendFileMemoryLimit = "send_file_memory_limit" + FlagSendMemoryCache = "send_memory_cache" + FlagNetErrorLimit = "net_error_limit" + FlagRemoteRetryTimes = "remote_retry_times" + FlagEnableLink = "enable_link" + FlagEnableLib = "enable_lib" + FlagLongTCP = "long_tcp" + FlagUseDefaultWorker = "use_default_worker" + FlagDynamicPort = "dynamic_port" + FlagWorkerOfferSlot = "worker_offer_slot" + FlagCleanTmpFilesDayAgo = "clean_tmp_files_day_ago" + FlagIgnoreHttpStatus = "ignore_http_status" + FlagResultCacheType = "result_cache_type" + FlagResultCacheTriggleSecs = "result_cache_triggle_secs" EnvBuildIDOld = "TURBO_PLAN_BUILD_ID" EnvBuildID = "TBS_BUILD_ID" @@ -487,6 +488,10 @@ var ( Name: "result_cache_type", Usage: "use which result cache type,0:no cache,1:local cache,2:remote cache,3:both", }, + commandCli.IntFlag{ + Name: "result_cache_triggle_secs", + Usage: "only remote execute time(seconds) great than this will triggle result cache", + }, } ) diff --git a/src/backend/booster/bk_dist/booster/command/process.go b/src/backend/booster/bk_dist/booster/command/process.go index 6a0c6fc3..d8599be0 100644 --- a/src/backend/booster/bk_dist/booster/command/process.go +++ b/src/backend/booster/bk_dist/booster/command/process.go @@ -249,6 +249,11 @@ func newBooster(c *commandCli.Context) (*pkg.Booster, error) { cache_server, _ := getCacheServerHost(c) + resultCacheTriggleSecs := -1 + if c.IsSet(FlagResultCacheTriggleSecs) && c.Int(FlagResultCacheTriggleSecs) >= 0 { + resultCacheTriggleSecs = c.Int(FlagResultCacheTriggleSecs) + } + // generate a new booster. cmdConfig := dcType.BoosterConfig{ Type: dcType.GetBoosterType(bt), @@ -258,68 +263,69 @@ func newBooster(c *commandCli.Context) (*pkg.Booster, error) { Args: c.String(FlagArgs), Cmd: strings.Join(os.Args, " "), Works: dcType.BoosterWorks{ - Stdout: os.Stdout, - Stderr: os.Stderr, - RunDir: runDir, - User: usr.Username, - LimitPerWorker: c.Int(FlagLimit), - Jobs: c.Int(FlagJobs), - MaxJobs: c.Int(FlagMaxJobs), - Presetjobs: c.Int(FlagPresetJobs), - MaxDegradedJobs: c.Int(FlagMaxDegradedJobs), - MaxLocalTotalJobs: defaultCPULimit(c.Int(FlagMaxLocalTotalJobs)), - MaxLocalPreJobs: c.Int(FlagMaxLocalPreJobs), - MaxLocalExeJobs: c.Int(FlagMaxLocalExeJobs), - MaxLocalPostJobs: c.Int(FlagMaxLocalPostJobs), - HookPreloadLibPath: c.String(FlagHookPreloadLib), - HookConfigPath: c.String(FlagHookConfig), - HookMode: c.Bool(FlagHook), - NoLocal: c.Bool(FlagNoLocal), - Local: c.Bool(FlagLocal), - WorkerSideCache: c.Bool(FlagWorkerSideCache), - LocalRecord: c.Bool(FlagLocalRecord), - Degraded: c.Bool(FlagDegraded), - ExecutorLogLevel: c.String(FlagExecutorLog), - Environments: make(map[string]string), - Bazel: c.Bool(FlagBazel), - BazelPlus: c.Bool(FlagBazelPlus), - Bazel4Plus: c.Bool(FlagBazel4Plus), - BazelNoLauncher: c.Bool(FlagBazelNoLauncher), - Launcher: c.Bool(FlagLauncher) || c.Bool(FlagBazelPlus) || c.Bool(FlagBazel4Plus), - AdditionFiles: c.StringSlice(FlagAdditionFile), - WorkerList: c.StringSlice(FlagWorkerList), - CheckMd5: c.Bool(FlagCheckMd5), - OutputEnvJSONFile: c.StringSlice(FlagOutputEnvJSONFile), - OutputEnvSourceFile: c.StringSlice(FlagOutputEnvSourceFile), - CommitSuicide: c.Bool(FlagCommitSuicide), - ToolChainJSONFile: c.String(FlagToolChainJSONFile), - SupportDirectives: c.Bool(FlagDirectives), - GlobalSlots: c.Bool(FlagGlobalSlots), - IOTimeoutSecs: c.Int(FlagIOTimeoutSecs), - Pump: c.Bool(FlagPump), - PumpDisableMacro: c.Bool(FlagPumpDisableMacro), - PumpIncludeSysHeader: c.Bool(FlagPumpIncludeSysHeader), - PumpCheck: c.Bool(FlagPumpCheck), - PumpCache: c.Bool(FlagPumpCache), - PumpCacheDir: c.String(FlagPumpCacheDir), - PumpCacheSizeMaxMB: pumpCacheSizeMaxMB, - PumpCacheRemoveAll: c.Bool(FlagPumpCacheRemoveAll), - PumpBlackList: c.StringSlice(FlagPumpBlackList), - PumpMinActionNum: int32(pumpMinActionNum), - PumpDisableStatCache: c.Bool(FlagPumpDisableStatCache), - PumpSearchLink: c.Bool(FlagPumpSearchLink), - PumpSearchLinkFile: c.String(FlagPumpSearchLinkFile), - PumpSearchLinkDir: c.StringSlice(FlagPumpSearchLinkDir), - PumpLstatByDir: c.Bool(FlagPumpLstatByDir), - PumpCorrectCap: c.Bool(FlagPumpCorrectCap), - ForceLocalList: c.StringSlice(FlagForceLocalList), - NoWork: c.Bool(FlagNoWork), - WriteMemroy: c.Bool(FlagWriteMemroMemroy), - IdleKeepSecs: c.Int(FlagIdleKeepSecs), - CleanTmpFilesDayAgo: cleanTmpFilesDayAgo, - SearchToolchain: c.Bool(FlagSearchToolchain), - IgnoreHttpStatus: c.Bool(FlagIgnoreHttpStatus), - ResultCacheType: c.Int(FlagResultCacheType), + Stdout: os.Stdout, + Stderr: os.Stderr, + RunDir: runDir, + User: usr.Username, + LimitPerWorker: c.Int(FlagLimit), + Jobs: c.Int(FlagJobs), + MaxJobs: c.Int(FlagMaxJobs), + Presetjobs: c.Int(FlagPresetJobs), + MaxDegradedJobs: c.Int(FlagMaxDegradedJobs), + MaxLocalTotalJobs: defaultCPULimit(c.Int(FlagMaxLocalTotalJobs)), + MaxLocalPreJobs: c.Int(FlagMaxLocalPreJobs), + MaxLocalExeJobs: c.Int(FlagMaxLocalExeJobs), + MaxLocalPostJobs: c.Int(FlagMaxLocalPostJobs), + HookPreloadLibPath: c.String(FlagHookPreloadLib), + HookConfigPath: c.String(FlagHookConfig), + HookMode: c.Bool(FlagHook), + NoLocal: c.Bool(FlagNoLocal), + Local: c.Bool(FlagLocal), + WorkerSideCache: c.Bool(FlagWorkerSideCache), + LocalRecord: c.Bool(FlagLocalRecord), + Degraded: c.Bool(FlagDegraded), + ExecutorLogLevel: c.String(FlagExecutorLog), + Environments: make(map[string]string), + Bazel: c.Bool(FlagBazel), + BazelPlus: c.Bool(FlagBazelPlus), + Bazel4Plus: c.Bool(FlagBazel4Plus), + BazelNoLauncher: c.Bool(FlagBazelNoLauncher), + Launcher: c.Bool(FlagLauncher) || c.Bool(FlagBazelPlus) || c.Bool(FlagBazel4Plus), + AdditionFiles: c.StringSlice(FlagAdditionFile), + WorkerList: c.StringSlice(FlagWorkerList), + CheckMd5: c.Bool(FlagCheckMd5), + OutputEnvJSONFile: c.StringSlice(FlagOutputEnvJSONFile), + OutputEnvSourceFile: c.StringSlice(FlagOutputEnvSourceFile), + CommitSuicide: c.Bool(FlagCommitSuicide), + ToolChainJSONFile: c.String(FlagToolChainJSONFile), + SupportDirectives: c.Bool(FlagDirectives), + GlobalSlots: c.Bool(FlagGlobalSlots), + IOTimeoutSecs: c.Int(FlagIOTimeoutSecs), + Pump: c.Bool(FlagPump), + PumpDisableMacro: c.Bool(FlagPumpDisableMacro), + PumpIncludeSysHeader: c.Bool(FlagPumpIncludeSysHeader), + PumpCheck: c.Bool(FlagPumpCheck), + PumpCache: c.Bool(FlagPumpCache), + PumpCacheDir: c.String(FlagPumpCacheDir), + PumpCacheSizeMaxMB: pumpCacheSizeMaxMB, + PumpCacheRemoveAll: c.Bool(FlagPumpCacheRemoveAll), + PumpBlackList: c.StringSlice(FlagPumpBlackList), + PumpMinActionNum: int32(pumpMinActionNum), + PumpDisableStatCache: c.Bool(FlagPumpDisableStatCache), + PumpSearchLink: c.Bool(FlagPumpSearchLink), + PumpSearchLinkFile: c.String(FlagPumpSearchLinkFile), + PumpSearchLinkDir: c.StringSlice(FlagPumpSearchLinkDir), + PumpLstatByDir: c.Bool(FlagPumpLstatByDir), + PumpCorrectCap: c.Bool(FlagPumpCorrectCap), + ForceLocalList: c.StringSlice(FlagForceLocalList), + NoWork: c.Bool(FlagNoWork), + WriteMemroy: c.Bool(FlagWriteMemroMemroy), + IdleKeepSecs: c.Int(FlagIdleKeepSecs), + CleanTmpFilesDayAgo: cleanTmpFilesDayAgo, + SearchToolchain: c.Bool(FlagSearchToolchain), + IgnoreHttpStatus: c.Bool(FlagIgnoreHttpStatus), + ResultCacheType: c.Int(FlagResultCacheType), + ResultCacheTriggleSecs: resultCacheTriggleSecs, }, Transport: dcType.BoosterTransport{ diff --git a/src/backend/booster/bk_dist/booster/pkg/booster.go b/src/backend/booster/bk_dist/booster/pkg/booster.go index 112abf6e..21a54142 100644 --- a/src/backend/booster/bk_dist/booster/pkg/booster.go +++ b/src/backend/booster/bk_dist/booster/pkg/booster.go @@ -368,6 +368,7 @@ func (b *Booster) getWorkersEnv() map[string]string { } requiredEnv[env.KeyExecutorResultCacheType] = strconv.Itoa(b.config.Works.ResultCacheType) + requiredEnv[env.KeyExecutorResultCacheTriggleSecs] = strconv.Itoa(b.config.Works.ResultCacheTriggleSecs) resultEnv := make(map[string]string, 10) for k, v := range requiredEnv { diff --git a/src/backend/booster/bk_dist/common/env/env.go b/src/backend/booster/bk_dist/common/env/env.go index 8a25cf82..ce9f82ca 100644 --- a/src/backend/booster/bk_dist/common/env/env.go +++ b/src/backend/booster/bk_dist/common/env/env.go @@ -77,6 +77,7 @@ const ( KeyExecutorUEShaderLocalCPUWeight = "UE_SHADER_LOCAL_CPU_WEIGHT" KeyExecutorIgnoreHttpStatus = "IGNORE_HTTP_STATUS" KeyExecutorResultCacheType = "RESULT_CACHE_TYPE" + KeyExecutorResultCacheTriggleSecs = "RESULT_CACHE_TRIGGLE_SECS" KeyExecutorHasResultIndex = "HAS_RESULT_INDEX" KeyUserDefinedLogLevel = "USER_DEFINED_LOG_LEVEL" diff --git a/src/backend/booster/bk_dist/common/resultcache/cache.go b/src/backend/booster/bk_dist/common/resultcache/cache.go index 10e69937..64598b24 100644 --- a/src/backend/booster/bk_dist/common/resultcache/cache.go +++ b/src/backend/booster/bk_dist/common/resultcache/cache.go @@ -586,8 +586,12 @@ func (t *Table) GetRecordGroup(key string) ([]byte, error) { return rg.ToBytes() } +const ( + indexSuffix = ".txt" +) + func (t *Table) Save(rg *RecordGroup) error { - filename := fmt.Sprintf("%s.txt", rg.Key) + filename := fmt.Sprintf("%s%s", rg.Key, indexSuffix) fullpath := filepath.Join(t.IndexDir, filename) data, err := rg.ToBytes() @@ -641,6 +645,12 @@ func (t *Table) Load() error { f.Close() for _, fi := range fis { + suffix := filepath.Ext(fi.Name()) + if suffix != indexSuffix { + blog.Infof("resultcache: ignore file:%s", fi.Name()) + continue + } + fullpath := filepath.Join(t.IndexDir, fi.Name()) file, err := os.Open(fullpath) if err != nil { diff --git a/src/backend/booster/bk_dist/common/types/booster.go b/src/backend/booster/bk_dist/common/types/booster.go index b0e93e7e..c0a0a6e3 100644 --- a/src/backend/booster/bk_dist/common/types/booster.go +++ b/src/backend/booster/bk_dist/common/types/booster.go @@ -136,7 +136,8 @@ type BoosterWorks struct { SearchToolchain bool IgnoreHttpStatus bool - ResultCacheType int + ResultCacheType int + ResultCacheTriggleSecs int } // BoosterTransport describe the transport data to controller diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go index 9520f958..80a5f173 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor.go @@ -116,6 +116,7 @@ type executor struct { // for result cache cacheType int + remoteTriggleSecs int cacheGroupKey string commandKey string preprocessResultKey string diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor4resultcache.go b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor4resultcache.go index 62a4d83a..a95b9099 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/local/executor4resultcache.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/local/executor4resultcache.go @@ -25,7 +25,7 @@ import ( ) const ( - LongExecutionSeconds = 300 + DefaultTriggleSeconds = 300 ) func (e *executor) initResultCacheInfo() { @@ -36,6 +36,14 @@ func (e *executor) initResultCacheInfo() { e.cacheGroupKey = str } + e.remoteTriggleSecs = DefaultTriggleSeconds + if str := e.sandbox.Env.GetEnv(env.KeyExecutorResultCacheTriggleSecs); str != "" { + intv, err := strconv.Atoi(str) + if err == nil && intv >= 0 { + e.remoteTriggleSecs = intv + } + } + e.commandKey = strings.Join(e.req.Commands, " ") record := resultcache.Record{ @@ -49,9 +57,11 @@ func (e *executor) initResultCacheInfo() { } blog.Infof("executor: got cache type:%d,cache group key:%s,command:[%s],"+ - "haslocalindex:%v,hasremoteindex:%v", + "haslocalindex:%v,hasremoteindex:%v"+ + "remoteTriggleSecs:%d", e.cacheType, e.cacheGroupKey, e.commandKey, - e.hasLocalIndex, e.hasRemoteIndex) + e.hasLocalIndex, e.hasRemoteIndex, + e.remoteTriggleSecs) } func (e *executor) localCacheEnabled() bool { @@ -292,7 +302,7 @@ func (e *executor) putCacheResult(r *dcSDK.BKDistResult, stat *dcSDK.ControllerJ blog.Infof("executor: remote executed %d seconds for this command", e.remoteExecuteSecs) } if e.remoteExecuteSecs > 0 { - remoteTooLong = e.remoteExecuteSecs >= LongExecutionSeconds + remoteTooLong = e.remoteExecuteSecs >= e.remoteTriggleSecs } record := resultcache.Record{} diff --git a/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go b/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go index 2cda78d4..fef4efa1 100644 --- a/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go +++ b/src/backend/booster/bk_dist/controller/pkg/manager/local/mgr.go @@ -272,6 +272,10 @@ func (m *Mgr) ExecuteTask( // 如果cache查询到了，则按预期的结果列表保存 cacheresult := e.getCacheResult(c) if cacheresult != nil { + blog.Infof("local: success to execute task by query cache for work(%s)"+ + " from pid(%d) in env(%v) dir(%s)", + m.work.ID(), + req.Pid, req.Environments, req.Dir) return cacheresult, nil } diff --git a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go index b4d960a9..141d4476 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cc/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cc/handler.go @@ -12,6 +12,7 @@ package cc import ( // "bytes" + "bytes" "fmt" "io/ioutil" "os" @@ -27,12 +28,14 @@ import ( dcFile "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/file" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/protocol" dcPump "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/pump" + "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/resultcache" dcSDK "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/sdk" dcSyscall "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/syscall" dcType "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/types" dcUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/common/util" commonUtil "github.com/TencentBlueKing/bk-turbo/src/backend/booster/bk_dist/handler/common" "github.com/TencentBlueKing/bk-turbo/src/backend/booster/common/blog" + "github.com/cespare/xxhash" ) const ( @@ -68,6 +71,7 @@ type TaskCC struct { rewriteCrossArgs []string preProcessArgs []string serverSideArgs []string + resultCacheArgs []string pumpArgs []string // file names @@ -100,7 +104,8 @@ type TaskCC struct { pchFileDesc *dcSDK.FileDesc // for /showIncludes - showinclude bool + showinclude bool + preprocessedErrorBuf string ForceLocalResponseFileKeys []string ForceLocalCppFileKeys []string @@ -122,6 +127,11 @@ func NewTaskCC() *TaskCC { } } +// GetPreprocessedBuf return preprocessedErrorBuf +func (cc *TaskCC) GetPreprocessedBuf() string { + return cc.preprocessedErrorBuf +} + // InitSandbox set sandbox to task-cc func (cc *TaskCC) InitSandbox(sandbox *dcSyscall.Sandbox) { cc.sandbox = sandbox @@ -954,22 +964,24 @@ func (cc *TaskCC) preExecute(command []string) (*dcSDK.BKDistCommand, dcType.BKD cc.originArgs = command // ++ try with pump,only support windows now - if !cc.pumpremotefailed && dcPump.SupportPump(cc.sandbox.Env) && cc.workerSupportAbsPath() { - if satisfied, _ := cc.isPumpActionNumSatisfied(); satisfied { - req, err, notifyerr := cc.trypump(command) - if err != nil { - if notifyerr == ErrorNotSupportRemote { - blog.Warnf("cc: pre execute failed to try pump %v: %v", command, err) - return nil, dcType.BKDistCommonError{ - Code: dcType.UnknowCode, - Error: err, + if !cc.hasResultIndex() { + if !cc.pumpremotefailed && dcPump.SupportPump(cc.sandbox.Env) && cc.workerSupportAbsPath() { + if satisfied, _ := cc.isPumpActionNumSatisfied(); satisfied { + req, err, notifyerr := cc.trypump(command) + if err != nil { + if notifyerr == ErrorNotSupportRemote { + blog.Warnf("cc: pre execute failed to try pump %v: %v", command, err) + return nil, dcType.BKDistCommonError{ + Code: dcType.UnknowCode, + Error: err, + } } + } else { + // for debug + blog.Debugf("cc: after try pump, req: %+v", *req) + cc.pumpremote = true + return req, dcType.ErrorNone } - } else { - // for debug - blog.Debugf("cc: after try pump, req: %+v", *req) - cc.pumpremote = true - return req, dcType.ErrorNone } } } @@ -1405,6 +1417,17 @@ func (cc *TaskCC) preBuild(args []string) error { cc.serverSideArgs = serverSideArgs + if cc.SupportResultCache(args) != resultcache.CacheTypeNone { + cc.resultCacheArgs = make([]string, 0, len(cc.serverSideArgs)) + copy(cc.resultCacheArgs, cc.serverSideArgs) + for index := range cc.resultCacheArgs { + if cc.resultCacheArgs[index] == cc.preprocessedFile { + cc.resultCacheArgs[index] = cc.inputFile + break + } + } + } + // debugRecordFileName(fmt.Sprintf("cc: pre-build finished")) blog.Infof("cc: pre-build success for enter args: %v", args) @@ -1484,11 +1507,15 @@ func (cc *TaskCC) doPreProcess(args []string, inputFile string) (string, error) sandbox := cc.sandbox.Fork() sandbox.Stdout = output + var errBuf bytes.Buffer + sandbox.Stderr = &errBuf + if _, err = sandbox.ExecCommand(execName, execArgs...); err != nil { blog.Errorf("cc: failed to do pre-process %s %s: %v", execName, strings.Join(execArgs, " "), err) return "", err } blog.Infof("cc: success to execute pre-process and get %s: %s", outputFile, strings.Join(newArgs, " ")) + cc.preprocessedErrorBuf = errBuf.String() return outputFile, nil } @@ -1562,9 +1589,61 @@ func (cc *TaskCC) needSaveResponseFile(args []string) (bool, string, error) { // SupportResultCache check whether this command support result cache func (cc *TaskCC) SupportResultCache(command []string) int { + if cc.sandbox != nil { + if str := cc.sandbox.Env.GetEnv(dcEnv.KeyExecutorResultCacheType); str != "" { + i, err := strconv.Atoi(str) + if err == nil { + return i + } + } + } + return 0 } +// hasResultIndex check whether the env of hasresultindex set +func (cc *TaskCC) hasResultIndex() bool { + return cc.sandbox.Env.GetEnv(dcEnv.KeyExecutorHasResultIndex) != "" +} + func (cc *TaskCC) GetResultCacheKey(command []string) string { - return "" + if !dcFile.Stat(cc.preprocessedFile).Exist() { + blog.Warnf("cc: cc.preprocessedFile %s not existed when get result cache key", cc.preprocessedFile) + return "" + } + + // ext from cc.preprocessedFile + ext := filepath.Ext(cc.preprocessedFile) + + // cc_mtime cc_name from compile tool + cchash, err := dcUtil.HashFile(command[0]) + if err != nil { + blog.Warnf("cc: hash file %s with error: %v", err) + return "" + } + + // LANG and LC_ALL from env , ignore in windows now + // cwd from work dir , ignore now + + // arg from cc.resultCacheArgs + argstring := strings.Join(cc.resultCacheArgs, " ") + arghash := xxhash.Sum64([]byte(argstring)) + + // cpp content from cl.preprocessedFile + cpphash, err := dcUtil.HashFile(cc.preprocessedFile) + if err != nil { + blog.Warnf("cc: hash file %s with error: %v", err) + return "" + } + + // cppstderr from cc.preprocessedErrorBuf + cppstderrhash := xxhash.Sum64([]byte(cc.preprocessedErrorBuf)) + + fullstring := fmt.Sprintf("%s_%x_%x_%x_%x", ext, cchash, arghash, cpphash, cppstderrhash) + fullstringhash := xxhash.Sum64([]byte(fullstring)) + + blog.Infof("cc: got hash key %x for string[%s] cmd:[%s]", + fullstringhash, fullstring, strings.Join(command, " ")) + + return fmt.Sprintf("%x", fullstringhash) } diff --git a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go index 2d4e6958..4cda5fbc 100644 --- a/src/backend/booster/bk_dist/handler/ue4/cl/handler.go +++ b/src/backend/booster/bk_dist/handler/ue4/cl/handler.go @@ -1377,9 +1377,10 @@ func (cl *TaskCL) doPreProcess(args []string, inputFile string) (string, []byte, return "", nil, err } blog.Infof("cl: success to execute pre-process and get %s: %s", outputFile, strings.Join(newArgs, " ")) - if cl.showinclude || cl.forcedepend { - cl.preprocessedErrorBuf = errBuf.String() - } + + // if cl.showinclude || cl.forcedepend { + cl.preprocessedErrorBuf = errBuf.String() + // } if !savetomemroy { return outputFile, nil, nil @@ -1548,7 +1549,7 @@ func (cl *TaskCL) hasResultIndex() bool { func (cl *TaskCL) GetResultCacheKey(command []string) string { if !dcFile.Stat(cl.preprocessedFile).Exist() { - blog.Warnf("cl: cl.preprocessedFile %s not existed when get result cache key", cl.sourcedependfile) + blog.Warnf("cl: cl.preprocessedFile %s not existed when get result cache key", cl.preprocessedFile) return "" }