[MGMT-14453](https://issues.redhat.com//browse/MGMT-14453): Fix bugs …

…in the installer cache This PR is for the purpose of resolving multiple bugs within the installer cache, due to the poor condition of the current cache, it makes sense to fix this in a single PR. * https://issues.redhat.com/browse/MGMT-14452 Installer cache removes in-used cached image when out of space * https://issues.redhat.com/browse/MGMT-14453 INSTALLER_CACHE_CAPACITY small value cause to assisted-service crash * https://issues.redhat.com/browse/MGMT-14457 Installer cache - fails to install when running parallel with same version * Additionally, the cache did not respect limits, so this has been addressed here. Fixes: I have implemented fixes for each of the following issues. * Mutex was ineffective as not instantiated corrctly, leading to [MGMT-14452](https://issues.redhat.com//browse/MGMT-14452), [MGMT-14453](https://issues.redhat.com//browse/MGMT-14453). * Naming convention for hardlinks changed to be UUID based to resolve [MGMT-14457](https://issues.redhat.com//browse/MGMT-14457). * Any time we either extract or use a release, the modified time must be updated, not only for cached releases. This was causing premature pruning of hardlinks. * LRU cache order updated to be based on microseconds instead of seconds. * Eviction checks updated to consider max release size and also cache threshold. * We now check there is enough space before writing. * During eviction - releases without hard links will be evicted before releases with hard links.
openshift · Jan 30, 2025 · 396a0e0 · 396a0e0
1 parent ef9f0c8
commit 396a0e0
Show file tree

Hide file tree

Showing 10 changed files with 813 additions and 198 deletions.
diff --git a/api/config/webhook/manifests.yaml b/api/config/webhook/manifests.yaml
@@ -0,0 +1,66 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: validating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-agent-install-openshift-io-v1beta1-agent
+  failurePolicy: Fail
+  name: vagent.kb.io
+  rules:
+  - apiGroups:
+    - agent-install.openshift.io
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - agents
+  sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-agent-install-openshift-io-v1beta1-agentclassification
+  failurePolicy: Fail
+  name: vagentclassification.kb.io
+  rules:
+  - apiGroups:
+    - agent-install.openshift.io
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - agentclassifications
+  sideEffects: None
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-agent-install-openshift-io-v1beta1-infraenv
+  failurePolicy: Fail
+  name: vinfraenv.kb.io
+  rules:
+  - apiGroups:
+    - agent-install.openshift.io
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - infraenvs
+  sideEffects: None
diff --git a/cmd/main.go b/cmd/main.go
@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/NYTimes/gziphandler"
+	"github.com/alecthomas/units"
 	"github.com/go-openapi/runtime"
 	"github.com/go-openapi/strfmt"
 	"github.com/go-openapi/swag"
@@ -39,6 +40,7 @@ import (
 	"github.com/openshift/assisted-service/internal/ignition"
 	"github.com/openshift/assisted-service/internal/infraenv"
 	installcfg "github.com/openshift/assisted-service/internal/installcfg/builder"
+	"github.com/openshift/assisted-service/internal/installercache"
 	internaljson "github.com/openshift/assisted-service/internal/json"
 	"github.com/openshift/assisted-service/internal/manifests"
 	"github.com/openshift/assisted-service/internal/metrics"
@@ -172,6 +174,18 @@ var Options struct {
 
 	// EnableXattrFallback is a boolean flag to enable en emulated fallback methoid of xattr on systems that do not support xattr.
 	EnableXattrFallback bool `envconfig:"ENABLE_XATTR_FALLBACK" default:"true"`
+
+	// InstallerCacheCapacityGiB is the capacity of the installer cache in GiB
+	InstallerCacheCapacityGiB uint `envconfig:"INSTALLER_CACHE_CAPACITY_GIB" default:"0"`
+
+	// InstallerCacheMaxReleaseSizeGiB is the expected maximum size of a single release in GiB
+	InstallerCacheMaxReleaseSizeGiB uint `envconfig:"INSTALLER_CACHE_MAX_RELEASE_SIZE_GIB" default:"2"`
+
+	// InstallerCacheEvictionThresholdPercent is the percentage of capacity at which the cache will start to evict releases.
+	InstallerCacheEvictionThresholdPercent uint `envconfig:"INSTALLER_CACHE_EVICTION_THRESHOLD_PERCENT" default:"80"`
+
+	// ReleaseFetchRetryIntervalSeconds is the number of seconds that the cache should wait before retrying the fetch of a release if unable to do so for capacity reasons.
+	ReleaseFetchRetryIntervalSeconds uint `envconfig:"INSTALLER_CACHE_RELEASE_FETCH_RETRY_INTERVAL_SECONDS" default:"30"`
 }
 
 func InitLogs(logLevel, logFormat string) *logrus.Logger {
@@ -315,7 +329,8 @@ func main() {
 	metricsManagerConfig := &metrics.MetricsManagerConfig{
 		DirectoryUsageMonitorConfig: metrics.DirectoryUsageMonitorConfig{
 			Directories: []string{Options.WorkDir}}}
-	metricsManager := metrics.NewMetricsManager(prometheusRegistry, eventsHandler, metrics.NewOSDiskStatsHelper(), metricsManagerConfig, log)
+	diskStatsHelper := metrics.NewOSDiskStatsHelper(logrus.New())
+	metricsManager := metrics.NewMetricsManager(prometheusRegistry, eventsHandler, diskStatsHelper, metricsManagerConfig, log)
 	if ocmClient != nil {
 		//inject the metric server to the ocm client for purpose of
 		//performance monitoring the calls to ACM. This could not be done
@@ -488,7 +503,18 @@ func main() {
 	failOnError(err, "failed to create valid bm config S3 endpoint URL from %s", Options.BMConfig.S3EndpointURL)
 	Options.BMConfig.S3EndpointURL = newUrl
 
-	generator := generator.New(log, objectHandler, Options.GeneratorConfig, Options.WorkDir, providerRegistry, manifestsApi, eventsHandler)
+	installGeneratorDirectoryConfig := generator.InstallGeneratorDirectoryConfig{WorkDir: Options.WorkDir}
+	installerCacheConfig := installercache.InstallerCacheConfig{
+		CacheDir:                        filepath.Join(installGeneratorDirectoryConfig.GetWorkingDirectory(), "installercache"),
+		MaxCapacity:                     int64(Options.InstallerCacheCapacityGiB) * int64(units.GiB),
+		MaxReleaseSize:                  int64(Options.InstallerCacheMaxReleaseSizeGiB) * int64(units.GiB),
+		ReleaseFetchRetryInterval:       time.Duration(Options.ReleaseFetchRetryIntervalSeconds) * time.Second,
+		InstallerCacheEvictionThreshold: float64(Options.InstallerCacheEvictionThresholdPercent) / 100,
+	}
+	installerCache, err := installercache.New(installerCacheConfig, eventsHandler, diskStatsHelper, log)
+	failOnError(err, "failed to instantiate installercache")
+
+	generator := generator.New(installGeneratorDirectoryConfig, log, objectHandler, Options.GeneratorConfig, providerRegistry, manifestsApi, eventsHandler, installerCache)
 	var crdUtils bminventory.CRDUtils
 	if ctrlMgr != nil {
 		crdUtils = controllers.NewCRDUtils(ctrlMgr.GetClient(), hostApi)

diff --git a/internal/ignition/installmanifests.go b/internal/ignition/installmanifests.go
@@ -85,7 +85,6 @@ type installerGenerator struct {
 	cluster                       *common.Cluster
 	releaseImage                  string
 	releaseImageMirror            string
-	installerDir                  string
 	serviceCACert                 string
 	encodedDhcpFileContents       string
 	s3Client                      s3wrapper.API
@@ -110,24 +109,23 @@ var fileNames = [...]string{
 }
 
 // NewGenerator returns a generator that can generate ignition files
-func NewGenerator(workDir string, installerDir string, cluster *common.Cluster, releaseImage string, releaseImageMirror string,
+func NewGenerator(workDir string, cluster *common.Cluster, releaseImage string, releaseImageMirror string,
 	serviceCACert string, installInvoker string, s3Client s3wrapper.API, log logrus.FieldLogger, providerRegistry registry.ProviderRegistry,
-	installerReleaseImageOverride, clusterTLSCertOverrideDir string, storageCapacityLimit int64, manifestApi manifestsapi.ManifestsAPI, eventsHandler eventsapi.Handler) Generator {
+	installerReleaseImageOverride, clusterTLSCertOverrideDir string, manifestApi manifestsapi.ManifestsAPI, eventsHandler eventsapi.Handler, installerCache *installercache.Installers) Generator {
 	return &installerGenerator{
 		cluster:                       cluster,
 		log:                           log,
 		releaseImage:                  releaseImage,
 		releaseImageMirror:            releaseImageMirror,
 		workDir:                       workDir,
-		installerDir:                  installerDir,
 		serviceCACert:                 serviceCACert,
 		s3Client:                      s3Client,
 		enableMetal3Provisioning:      true,
 		installInvoker:                installInvoker,
 		providerRegistry:              providerRegistry,
 		installerReleaseImageOverride: installerReleaseImageOverride,
 		clusterTLSCertOverrideDir:     clusterTLSCertOverrideDir,
-		installerCache:                installercache.New(installerDir, storageCapacityLimit, eventsHandler, log),
+		installerCache:                installerCache,
 		manifestApi:                   manifestApi,
 	}
 }