feat:support result cache, issue: TencentBlueKing#315

src/backend/booster/bk_dist/booster/command/command.go

@@ -23,97 +23,98 @@ import (
 
 // define const vars
 const (
-	FlagProjectID            = "project_id"
-	FlagBuildID              = "build_id"
-	FlagArgs                 = "args"
-	FlagGccVersion           = "gcc_version"
-	FlagBazel                = "bazel"
-	FlagBazelPlus            = "bazel_plus"
-	FlagBazel4Plus           = "bazel4_plus"
-	FlagBazelNoLauncher      = "bazel_no_launcher"
-	FlagLauncher             = "launcher"
-	FlagLog                  = "log"
-	FlagLogDir               = "log_dir"
-	FlagLogToConsole         = "log_to_console"
-	FlagExecutorLog          = "executor_log"
-	FlagTest                 = "test"
-	FlagCommandPath          = "command_path"
-	FlagLimit                = "limit"
-	FlagNoLocal              = "no_local"
-	FlagLocal                = "local"
-	FlagDegraded             = "degraded"
-	FlagSaveCode             = "save_code"
-	FlagJobs                 = "jobs"
-	FlagMaxJobs              = "max_jobs"
-	FlagPresetJobs           = "preset_jobs"
-	FlagMaxDegradedJobs      = "max_degraded_jobs"
-	FlagMaxLocalTotalJobs    = "max_local_total_jobs"
-	FlagMaxLocalExeJobs      = "max_local_exe_jobs"
-	FlagMaxLocalPreJobs      = "max_local_pre_jobs"
-	FlagMaxLocalPostJobs     = "max_local_post_jobs"
-	FlagHook                 = "hook"
-	FlagBoosterType          = "booster_type"
-	FlagHookConfig           = "hook_config"
-	FlagHookPreloadLib       = "hook_preload_library"
-	FlagAdditionFile         = "addition_file"
-	FlagDashboard            = "dashboard"
-	FlagWorkerList           = "worker_list"
-	FlagCheckMd5             = "check_md5"
-	FlagOutputEnvJSONFile    = "output_env_json_file"
-	FlagOutputEnvSourceFile  = "output_env_source_file"
-	FlagCommitSuicide        = "commit_suicide"
-	FlagToolChainJSONFile    = "tool_chain_json_file"
-	FlagSearchToolchain      = "search_toolchain"
-	FlagBatchMode            = "batch_mode"
-	FlagDirectives           = "directives"
-	FlagGlobalSlots          = "global_slots"
-	FlagSudoController       = "sudo_controller"
-	FlagIOTimeoutSecs        = "io_timeout_secs"
-	FlagPump                 = "pump"
-	FlagPumpDisableMacro     = "pump_disable_macro"
-	FlagPumpIncludeSysHeader = "pump_include_sys_header"
-	FlagPumpCheck            = "pump_check"
-	FlagPumpCache            = "pump_cache"
-	FlagPumpCacheDir         = "pump_cache_dir"
-	FlagPumpCacheSizeMaxMB   = "pump_cache_size_max_MB"
-	FlagPumpCacheRemoveAll   = "pump_cache_remove_all"
-	FlagPumpBlackList        = "pump_black_list"
-	FlagPumpMinActionNum     = "pump_min_action_num"
-	FlagPumpDisableStatCache = "pump_disable_stat_cache"
-	FlagPumpSearchLink       = "pump_search_link"
-	FlagPumpSearchLinkFile   = "pump_search_link_file"
-	FlagPumpSearchLinkDir    = "pump_search_link_dir"
-	FlagPumpLstatByDir       = "pump_lstat_by_dir"
-	FlagPumpCorrectCap       = "pump_correct_cap"
-	FlagForceLocalList       = "force_local_list"
-	FlagNoWork               = "no_work"
-	FlagControllerNoWait     = "controller_no_wait"
-	FlagControllerRemainTime = "controller_remain_time"
-	FlagServer               = "server"
-	FlagCacheServer          = "cache_server"
-	FlagWorkerSideCache      = "worker_side_cache"
-	FlagLocalRecord          = "local_record"
-	FlagWriteMemroMemroy     = "write_memory"
-	FlagIdleKeepSecs         = "idle_keep_secs"
-	FlagResourceTimeoutSecs  = "resource_timeout_secs"
-	FlagLocalIdleCPUPercent  = "use_local_cpu_percent"
-	FlagDisableFileLock      = "disable_file_lock"
-	FlagAutoResourceMgr      = "auto_resource_mgr"
-	FlagResIdleSecsForFree   = "res_idle_secs_for_free"
-	FlagSendCork             = "send_cork"
-	FlagSendFileMemoryLimit  = "send_file_memory_limit"
-	FlagSendMemoryCache      = "send_memory_cache"
-	FlagNetErrorLimit        = "net_error_limit"
-	FlagRemoteRetryTimes     = "remote_retry_times"
-	FlagEnableLink           = "enable_link"
-	FlagEnableLib            = "enable_lib"
-	FlagLongTCP              = "long_tcp"
-	FlagUseDefaultWorker     = "use_default_worker"
-	FlagDynamicPort          = "dynamic_port"
-	FlagWorkerOfferSlot      = "worker_offer_slot"
-	FlagCleanTmpFilesDayAgo  = "clean_tmp_files_day_ago"
-	FlagIgnoreHttpStatus     = "ignore_http_status"
-	FlagResultCacheType      = "result_cache_type"
+	FlagProjectID              = "project_id"
+	FlagBuildID                = "build_id"
+	FlagArgs                   = "args"
+	FlagGccVersion             = "gcc_version"
+	FlagBazel                  = "bazel"
+	FlagBazelPlus              = "bazel_plus"
+	FlagBazel4Plus             = "bazel4_plus"
+	FlagBazelNoLauncher        = "bazel_no_launcher"
+	FlagLauncher               = "launcher"
+	FlagLog                    = "log"
+	FlagLogDir                 = "log_dir"
+	FlagLogToConsole           = "log_to_console"
+	FlagExecutorLog            = "executor_log"
+	FlagTest                   = "test"
+	FlagCommandPath            = "command_path"
+	FlagLimit                  = "limit"
+	FlagNoLocal                = "no_local"
+	FlagLocal                  = "local"
+	FlagDegraded               = "degraded"
+	FlagSaveCode               = "save_code"
+	FlagJobs                   = "jobs"
+	FlagMaxJobs                = "max_jobs"
+	FlagPresetJobs             = "preset_jobs"
+	FlagMaxDegradedJobs        = "max_degraded_jobs"
+	FlagMaxLocalTotalJobs      = "max_local_total_jobs"
+	FlagMaxLocalExeJobs        = "max_local_exe_jobs"
+	FlagMaxLocalPreJobs        = "max_local_pre_jobs"
+	FlagMaxLocalPostJobs       = "max_local_post_jobs"
+	FlagHook                   = "hook"
+	FlagBoosterType            = "booster_type"
+	FlagHookConfig             = "hook_config"
+	FlagHookPreloadLib         = "hook_preload_library"
+	FlagAdditionFile           = "addition_file"
+	FlagDashboard              = "dashboard"
+	FlagWorkerList             = "worker_list"
+	FlagCheckMd5               = "check_md5"
+	FlagOutputEnvJSONFile      = "output_env_json_file"
+	FlagOutputEnvSourceFile    = "output_env_source_file"
+	FlagCommitSuicide          = "commit_suicide"
+	FlagToolChainJSONFile      = "tool_chain_json_file"
+	FlagSearchToolchain        = "search_toolchain"
+	FlagBatchMode              = "batch_mode"
+	FlagDirectives             = "directives"
+	FlagGlobalSlots            = "global_slots"
+	FlagSudoController         = "sudo_controller"
+	FlagIOTimeoutSecs          = "io_timeout_secs"
+	FlagPump                   = "pump"
+	FlagPumpDisableMacro       = "pump_disable_macro"
+	FlagPumpIncludeSysHeader   = "pump_include_sys_header"
+	FlagPumpCheck              = "pump_check"
+	FlagPumpCache              = "pump_cache"
+	FlagPumpCacheDir           = "pump_cache_dir"
+	FlagPumpCacheSizeMaxMB     = "pump_cache_size_max_MB"
+	FlagPumpCacheRemoveAll     = "pump_cache_remove_all"
+	FlagPumpBlackList          = "pump_black_list"
+	FlagPumpMinActionNum       = "pump_min_action_num"
+	FlagPumpDisableStatCache   = "pump_disable_stat_cache"
+	FlagPumpSearchLink         = "pump_search_link"
+	FlagPumpSearchLinkFile     = "pump_search_link_file"
+	FlagPumpSearchLinkDir      = "pump_search_link_dir"
+	FlagPumpLstatByDir         = "pump_lstat_by_dir"
+	FlagPumpCorrectCap         = "pump_correct_cap"
+	FlagForceLocalList         = "force_local_list"
+	FlagNoWork                 = "no_work"
+	FlagControllerNoWait       = "controller_no_wait"
+	FlagControllerRemainTime   = "controller_remain_time"
+	FlagServer                 = "server"
+	FlagCacheServer            = "cache_server"
+	FlagWorkerSideCache        = "worker_side_cache"
+	FlagLocalRecord            = "local_record"
+	FlagWriteMemroMemroy       = "write_memory"
+	FlagIdleKeepSecs           = "idle_keep_secs"
+	FlagResourceTimeoutSecs    = "resource_timeout_secs"
+	FlagLocalIdleCPUPercent    = "use_local_cpu_percent"
+	FlagDisableFileLock        = "disable_file_lock"
+	FlagAutoResourceMgr        = "auto_resource_mgr"
+	FlagResIdleSecsForFree     = "res_idle_secs_for_free"
+	FlagSendCork               = "send_cork"
+	FlagSendFileMemoryLimit    = "send_file_memory_limit"
+	FlagSendMemoryCache        = "send_memory_cache"
+	FlagNetErrorLimit          = "net_error_limit"
+	FlagRemoteRetryTimes       = "remote_retry_times"
+	FlagEnableLink             = "enable_link"
+	FlagEnableLib              = "enable_lib"
+	FlagLongTCP                = "long_tcp"
+	FlagUseDefaultWorker       = "use_default_worker"
+	FlagDynamicPort            = "dynamic_port"
+	FlagWorkerOfferSlot        = "worker_offer_slot"
+	FlagCleanTmpFilesDayAgo    = "clean_tmp_files_day_ago"
+	FlagIgnoreHttpStatus       = "ignore_http_status"
+	FlagResultCacheType        = "result_cache_type"
+	FlagResultCacheTriggleSecs = "result_cache_triggle_secs"
 
 	EnvBuildIDOld  = "TURBO_PLAN_BUILD_ID"
 	EnvBuildID     = "TBS_BUILD_ID"
@@ -487,6 +488,10 @@ var (
 			Name:  "result_cache_type",
 			Usage: "use which result cache type,0:no cache,1:local cache,2:remote cache,3:both",
 		},
+		commandCli.IntFlag{
+			Name:  "result_cache_triggle_secs",
+			Usage: "only remote execute time(seconds) great than this will triggle result cache",
+		},
 	}
 )
 

src/backend/booster/bk_dist/booster/command/process.go

@@ -249,6 +249,11 @@ func newBooster(c *commandCli.Context) (*pkg.Booster, error) {
 
 	cache_server, _ := getCacheServerHost(c)
 
+	resultCacheTriggleSecs := -1
+	if c.IsSet(FlagResultCacheTriggleSecs) && c.Int(FlagResultCacheTriggleSecs) >= 0 {
+		resultCacheTriggleSecs = c.Int(FlagResultCacheTriggleSecs)
+	}
+
 	// generate a new booster.
 	cmdConfig := dcType.BoosterConfig{
 		Type:      dcType.GetBoosterType(bt),
@@ -258,68 +263,69 @@ func newBooster(c *commandCli.Context) (*pkg.Booster, error) {
 		Args:      c.String(FlagArgs),
 		Cmd:       strings.Join(os.Args, " "),
 		Works: dcType.BoosterWorks{
-			Stdout:               os.Stdout,
-			Stderr:               os.Stderr,
-			RunDir:               runDir,
-			User:                 usr.Username,
-			LimitPerWorker:       c.Int(FlagLimit),
-			Jobs:                 c.Int(FlagJobs),
-			MaxJobs:              c.Int(FlagMaxJobs),
-			Presetjobs:           c.Int(FlagPresetJobs),
-			MaxDegradedJobs:      c.Int(FlagMaxDegradedJobs),
-			MaxLocalTotalJobs:    defaultCPULimit(c.Int(FlagMaxLocalTotalJobs)),
-			MaxLocalPreJobs:      c.Int(FlagMaxLocalPreJobs),
-			MaxLocalExeJobs:      c.Int(FlagMaxLocalExeJobs),
-			MaxLocalPostJobs:     c.Int(FlagMaxLocalPostJobs),
-			HookPreloadLibPath:   c.String(FlagHookPreloadLib),
-			HookConfigPath:       c.String(FlagHookConfig),
-			HookMode:             c.Bool(FlagHook),
-			NoLocal:              c.Bool(FlagNoLocal),
-			Local:                c.Bool(FlagLocal),
-			WorkerSideCache:      c.Bool(FlagWorkerSideCache),
-			LocalRecord:          c.Bool(FlagLocalRecord),
-			Degraded:             c.Bool(FlagDegraded),
-			ExecutorLogLevel:     c.String(FlagExecutorLog),
-			Environments:         make(map[string]string),
-			Bazel:                c.Bool(FlagBazel),
-			BazelPlus:            c.Bool(FlagBazelPlus),
-			Bazel4Plus:           c.Bool(FlagBazel4Plus),
-			BazelNoLauncher:      c.Bool(FlagBazelNoLauncher),
-			Launcher:             c.Bool(FlagLauncher) || c.Bool(FlagBazelPlus) || c.Bool(FlagBazel4Plus),
-			AdditionFiles:        c.StringSlice(FlagAdditionFile),
-			WorkerList:           c.StringSlice(FlagWorkerList),
-			CheckMd5:             c.Bool(FlagCheckMd5),
-			OutputEnvJSONFile:    c.StringSlice(FlagOutputEnvJSONFile),
-			OutputEnvSourceFile:  c.StringSlice(FlagOutputEnvSourceFile),
-			CommitSuicide:        c.Bool(FlagCommitSuicide),
-			ToolChainJSONFile:    c.String(FlagToolChainJSONFile),
-			SupportDirectives:    c.Bool(FlagDirectives),
-			GlobalSlots:          c.Bool(FlagGlobalSlots),
-			IOTimeoutSecs:        c.Int(FlagIOTimeoutSecs),
-			Pump:                 c.Bool(FlagPump),
-			PumpDisableMacro:     c.Bool(FlagPumpDisableMacro),
-			PumpIncludeSysHeader: c.Bool(FlagPumpIncludeSysHeader),
-			PumpCheck:            c.Bool(FlagPumpCheck),
-			PumpCache:            c.Bool(FlagPumpCache),
-			PumpCacheDir:         c.String(FlagPumpCacheDir),
-			PumpCacheSizeMaxMB:   pumpCacheSizeMaxMB,
-			PumpCacheRemoveAll:   c.Bool(FlagPumpCacheRemoveAll),
-			PumpBlackList:        c.StringSlice(FlagPumpBlackList),
-			PumpMinActionNum:     int32(pumpMinActionNum),
-			PumpDisableStatCache: c.Bool(FlagPumpDisableStatCache),
-			PumpSearchLink:       c.Bool(FlagPumpSearchLink),
-			PumpSearchLinkFile:   c.String(FlagPumpSearchLinkFile),
-			PumpSearchLinkDir:    c.StringSlice(FlagPumpSearchLinkDir),
-			PumpLstatByDir:       c.Bool(FlagPumpLstatByDir),
-			PumpCorrectCap:       c.Bool(FlagPumpCorrectCap),
-			ForceLocalList:       c.StringSlice(FlagForceLocalList),
-			NoWork:               c.Bool(FlagNoWork),
-			WriteMemroy:          c.Bool(FlagWriteMemroMemroy),
-			IdleKeepSecs:         c.Int(FlagIdleKeepSecs),
-			CleanTmpFilesDayAgo:  cleanTmpFilesDayAgo,
-			SearchToolchain:      c.Bool(FlagSearchToolchain),
-			IgnoreHttpStatus:     c.Bool(FlagIgnoreHttpStatus),
-			ResultCacheType:      c.Int(FlagResultCacheType),
+			Stdout:                 os.Stdout,
+			Stderr:                 os.Stderr,
+			RunDir:                 runDir,
+			User:                   usr.Username,
+			LimitPerWorker:         c.Int(FlagLimit),
+			Jobs:                   c.Int(FlagJobs),
+			MaxJobs:                c.Int(FlagMaxJobs),
+			Presetjobs:             c.Int(FlagPresetJobs),
+			MaxDegradedJobs:        c.Int(FlagMaxDegradedJobs),
+			MaxLocalTotalJobs:      defaultCPULimit(c.Int(FlagMaxLocalTotalJobs)),
+			MaxLocalPreJobs:        c.Int(FlagMaxLocalPreJobs),
+			MaxLocalExeJobs:        c.Int(FlagMaxLocalExeJobs),
+			MaxLocalPostJobs:       c.Int(FlagMaxLocalPostJobs),
+			HookPreloadLibPath:     c.String(FlagHookPreloadLib),
+			HookConfigPath:         c.String(FlagHookConfig),
+			HookMode:               c.Bool(FlagHook),
+			NoLocal:                c.Bool(FlagNoLocal),
+			Local:                  c.Bool(FlagLocal),
+			WorkerSideCache:        c.Bool(FlagWorkerSideCache),
+			LocalRecord:            c.Bool(FlagLocalRecord),
+			Degraded:               c.Bool(FlagDegraded),
+			ExecutorLogLevel:       c.String(FlagExecutorLog),
+			Environments:           make(map[string]string),
+			Bazel:                  c.Bool(FlagBazel),
+			BazelPlus:              c.Bool(FlagBazelPlus),
+			Bazel4Plus:             c.Bool(FlagBazel4Plus),
+			BazelNoLauncher:        c.Bool(FlagBazelNoLauncher),
+			Launcher:               c.Bool(FlagLauncher) || c.Bool(FlagBazelPlus) || c.Bool(FlagBazel4Plus),
+			AdditionFiles:          c.StringSlice(FlagAdditionFile),
+			WorkerList:             c.StringSlice(FlagWorkerList),
+			CheckMd5:               c.Bool(FlagCheckMd5),
+			OutputEnvJSONFile:      c.StringSlice(FlagOutputEnvJSONFile),
+			OutputEnvSourceFile:    c.StringSlice(FlagOutputEnvSourceFile),
+			CommitSuicide:          c.Bool(FlagCommitSuicide),
+			ToolChainJSONFile:      c.String(FlagToolChainJSONFile),
+			SupportDirectives:      c.Bool(FlagDirectives),
+			GlobalSlots:            c.Bool(FlagGlobalSlots),
+			IOTimeoutSecs:          c.Int(FlagIOTimeoutSecs),
+			Pump:                   c.Bool(FlagPump),
+			PumpDisableMacro:       c.Bool(FlagPumpDisableMacro),
+			PumpIncludeSysHeader:   c.Bool(FlagPumpIncludeSysHeader),
+			PumpCheck:              c.Bool(FlagPumpCheck),
+			PumpCache:              c.Bool(FlagPumpCache),
+			PumpCacheDir:           c.String(FlagPumpCacheDir),
+			PumpCacheSizeMaxMB:     pumpCacheSizeMaxMB,
+			PumpCacheRemoveAll:     c.Bool(FlagPumpCacheRemoveAll),
+			PumpBlackList:          c.StringSlice(FlagPumpBlackList),
+			PumpMinActionNum:       int32(pumpMinActionNum),
+			PumpDisableStatCache:   c.Bool(FlagPumpDisableStatCache),
+			PumpSearchLink:         c.Bool(FlagPumpSearchLink),
+			PumpSearchLinkFile:     c.String(FlagPumpSearchLinkFile),
+			PumpSearchLinkDir:      c.StringSlice(FlagPumpSearchLinkDir),
+			PumpLstatByDir:         c.Bool(FlagPumpLstatByDir),
+			PumpCorrectCap:         c.Bool(FlagPumpCorrectCap),
+			ForceLocalList:         c.StringSlice(FlagForceLocalList),
+			NoWork:                 c.Bool(FlagNoWork),
+			WriteMemroy:            c.Bool(FlagWriteMemroMemroy),
+			IdleKeepSecs:           c.Int(FlagIdleKeepSecs),
+			CleanTmpFilesDayAgo:    cleanTmpFilesDayAgo,
+			SearchToolchain:        c.Bool(FlagSearchToolchain),
+			IgnoreHttpStatus:       c.Bool(FlagIgnoreHttpStatus),
+			ResultCacheType:        c.Int(FlagResultCacheType),
+			ResultCacheTriggleSecs: resultCacheTriggleSecs,
 		},
 
 		Transport: dcType.BoosterTransport{

src/backend/booster/bk_dist/booster/pkg/booster.go

@@ -368,6 +368,7 @@ func (b *Booster) getWorkersEnv() map[string]string {
 	}
 
 	requiredEnv[env.KeyExecutorResultCacheType] = strconv.Itoa(b.config.Works.ResultCacheType)
+	requiredEnv[env.KeyExecutorResultCacheTriggleSecs] = strconv.Itoa(b.config.Works.ResultCacheTriggleSecs)
 
 	resultEnv := make(map[string]string, 10)
 	for k, v := range requiredEnv {

src/backend/booster/bk_dist/common/env/env.go

@@ -77,6 +77,7 @@ const (
 	KeyExecutorUEShaderLocalCPUWeight  = "UE_SHADER_LOCAL_CPU_WEIGHT"
 	KeyExecutorIgnoreHttpStatus        = "IGNORE_HTTP_STATUS"
 	KeyExecutorResultCacheType         = "RESULT_CACHE_TYPE"
+	KeyExecutorResultCacheTriggleSecs  = "RESULT_CACHE_TRIGGLE_SECS"
 	KeyExecutorHasResultIndex          = "HAS_RESULT_INDEX"
 
 	KeyUserDefinedLogLevel         = "USER_DEFINED_LOG_LEVEL"

src/backend/booster/bk_dist/common/resultcache/cache.go

@@ -586,8 +586,12 @@ func (t *Table) GetRecordGroup(key string) ([]byte, error) {
 	return rg.ToBytes()
 }
 
+const (
+	indexSuffix = ".txt"
+)
+
 func (t *Table) Save(rg *RecordGroup) error {
-	filename := fmt.Sprintf("%s.txt", rg.Key)
+	filename := fmt.Sprintf("%s%s", rg.Key, indexSuffix)
 	fullpath := filepath.Join(t.IndexDir, filename)
 
 	data, err := rg.ToBytes()
@@ -641,6 +645,12 @@ func (t *Table) Load() error {
 	f.Close()
 
 	for _, fi := range fis {
+		suffix := filepath.Ext(fi.Name())
+		if suffix != indexSuffix {
+			blog.Infof("resultcache: ignore file:%s", fi.Name())
+			continue
+		}
+
 		fullpath := filepath.Join(t.IndexDir, fi.Name())
 		file, err := os.Open(fullpath)
 		if err != nil {

src/backend/booster/bk_dist/common/types/booster.go

@@ -136,7 +136,8 @@ type BoosterWorks struct {
 	SearchToolchain  bool
 	IgnoreHttpStatus bool
 
-	ResultCacheType int
+	ResultCacheType        int
+	ResultCacheTriggleSecs int
 }
 
 // BoosterTransport describe the transport data to controller