feat(cve): implement CVE scanning as background tasks (#1833)

1. Move existing CVE DB download generator/task login under the cve package
2. Add a new CVE scanner task generator and task type to run in the background, as well as tests for it
3. Move the CVE cache in its own package
4. Add a CVE scanner methods to check if an entry is present in the cache, and to retreive the results
5. Modify the FilterTags MetaDB method to not exit on first error
This is needed in order to pass all tags to the generator,
instead of the generator stopping at the first set of invalid data
6. Integrate the new scanning task generator with the existing zot code.
7. Fix an issue where the CVE scan results for multiarch images was not cached
8. Rewrite some of the older CVE tests to use the new image-utils test package
9. Use the CVE scanner as attribute of the controller instead of CveInfo.
Remove functionality of CVE DB update from CveInfo, it is now responsible,
as the name states, only for providing CVE information.
10. The logic to get maximum severity and cve count for image sumaries now uses only the scanner cache.
11. Removed the GetCVESummaryForImage method from CveInfo as it was only used in tests

Signed-off-by: Andrei Aaron <[email protected]>

pkg/api/config/config.go

@@ -348,6 +348,10 @@ func (c *Config) IsSearchEnabled() bool {
 	return c.Extensions != nil && c.Extensions.Search != nil && *c.Extensions.Search.Enable
 }
 
+func (c *Config) IsCveScanningEnabled() bool {
+	return c.IsSearchEnabled() && c.Extensions.Search.CVE != nil
+}
+
 func (c *Config) IsUIEnabled() bool {
 	return c.Extensions != nil && c.Extensions.UI != nil && *c.Extensions.UI.Enable
 }

pkg/api/controller.go

@@ -46,7 +46,7 @@ type Controller struct {
 	Audit           *log.Logger
 	Server          *http.Server
 	Metrics         monitoring.MetricServer
-	CveInfo         ext.CveInfo
+	CveScanner      ext.CveScanner
 	SyncOnDemand    SyncOnDemand
 	RelyingParties  map[string]rp.RelyingParty
 	CookieStore     sessions.Store
@@ -241,7 +241,7 @@ func (c *Controller) Init(reloadCtx context.Context) error {
 func (c *Controller) InitCVEInfo() {
 	// Enable CVE extension if extension config is provided
 	if c.Config != nil && c.Config.Extensions != nil {
-		c.CveInfo = ext.GetCVEInfo(c.Config, c.StoreController, c.MetaDB, c.Log)
+		c.CveScanner = ext.GetCveScanner(c.Config, c.StoreController, c.MetaDB, c.Log)
 	}
 }
 
@@ -347,7 +347,7 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
 	// Enable extensions if extension config is provided for DefaultStore
 	if c.Config != nil && c.Config.Extensions != nil {
 		ext.EnableMetricsExtension(c.Config, c.Log, c.Config.Storage.RootDirectory)
-		ext.EnableSearchExtension(c.Config, c.StoreController, c.MetaDB, taskScheduler, c.CveInfo, c.Log)
+		ext.EnableSearchExtension(c.Config, c.StoreController, c.MetaDB, taskScheduler, c.CveScanner, c.Log)
 	}
 
 	if c.Config.Storage.SubPaths != nil {

pkg/api/routes.go

@@ -184,7 +184,7 @@ func (rh *RouteHandler) SetupRoutes() {
 
 	// Preconditions for enabling the actual extension routes are part of extensions themselves
 	ext.SetupMetricsRoutes(rh.c.Config, rh.c.Router, authHandler, rh.c.Log, rh.c.Metrics)
-	ext.SetupSearchRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.MetaDB, rh.c.CveInfo,
+	ext.SetupSearchRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.MetaDB, rh.c.CveScanner,
 		rh.c.Log)
 	ext.SetupImageTrustRoutes(rh.c.Config, prefixedRouter, rh.c.MetaDB, rh.c.Log)
 	ext.SetupMgmtRoutes(rh.c.Config, prefixedRouter, rh.c.Log)

pkg/cli/client/cve_cmd_internal_test.go

@@ -531,7 +531,7 @@ func TestNegativeServerResponse(t *testing.T) {
 			panic(err)
 		}
 
-		ctlr.CveInfo = getMockCveInfo(ctlr.MetaDB, ctlr.Log)
+		ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB)
 
 		go func() {
 			if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) {
@@ -606,7 +606,7 @@ func TestServerCVEResponse(t *testing.T) {
 		panic(err)
 	}
 
-	ctlr.CveInfo = getMockCveInfo(ctlr.MetaDB, ctlr.Log)
+	ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB)
 
 	go func() {
 		if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) {
@@ -947,39 +947,35 @@ func TestCVESort(t *testing.T) {
 		panic(err)
 	}
 
-	ctlr.CveInfo = cveinfo.BaseCveInfo{
-		Log:    ctlr.Log,
-		MetaDB: mocks.MetaDBMock{},
-		Scanner: mocks.CveScannerMock{
-			ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) {
-				return map[string]cvemodel.CVE{
-					"CVE-2023-1255": {
-						ID:       "CVE-2023-1255",
-						Severity: "LOW",
-						Title:    "Input buffer over-read in AES-XTS implementation and testing",
-					},
-					"CVE-2023-2650": {
-						ID:       "CVE-2023-2650",
-						Severity: "MEDIUM",
-						Title:    "Possible DoS translating ASN.1 object identifier and executer",
-					},
-					"CVE-2023-2975": {
-						ID:       "CVE-2023-2975",
-						Severity: "HIGH",
-						Title:    "AES-SIV cipher implementation contains a bug that can break",
-					},
-					"CVE-2023-3446": {
-						ID:       "CVE-2023-3446",
-						Severity: "CRITICAL",
-						Title:    "Excessive time spent checking DH keys and parenthesis",
-					},
-					"CVE-2023-3817": {
-						ID:       "CVE-2023-3817",
-						Severity: "MEDIUM",
-						Title:    "Excessive time spent checking DH q parameter and arguments",
-					},
-				}, nil
-			},
+	ctlr.CveScanner = mocks.CveScannerMock{
+		ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) {
+			return map[string]cvemodel.CVE{
+				"CVE-2023-1255": {
+					ID:       "CVE-2023-1255",
+					Severity: "LOW",
+					Title:    "Input buffer over-read in AES-XTS implementation and testing",
+				},
+				"CVE-2023-2650": {
+					ID:       "CVE-2023-2650",
+					Severity: "MEDIUM",
+					Title:    "Possible DoS translating ASN.1 object identifier and executer",
+				},
+				"CVE-2023-2975": {
+					ID:       "CVE-2023-2975",
+					Severity: "HIGH",
+					Title:    "AES-SIV cipher implementation contains a bug that can break",
+				},
+				"CVE-2023-3446": {
+					ID:       "CVE-2023-3446",
+					Severity: "CRITICAL",
+					Title:    "Excessive time spent checking DH keys and parenthesis",
+				},
+				"CVE-2023-3817": {
+					ID:       "CVE-2023-3817",
+					Severity: "MEDIUM",
+					Title:    "Excessive time spent checking DH q parameter and arguments",
+				},
+			}, nil
 		},
 	}
 
@@ -1373,7 +1369,7 @@ func TestCVECommandErrors(t *testing.T) {
 	})
 }
 
-func getMockCveInfo(metaDB mTypes.MetaDB, log log.Logger) cveinfo.CveInfo {
+func getMockCveScanner(metaDB mTypes.MetaDB) cveinfo.Scanner {
 	// MetaDB loaded with initial data now mock the scanner
 	// Setup test CVE data in mock scanner
 	scanner := mocks.CveScannerMock{
@@ -1472,11 +1468,7 @@ func getMockCveInfo(metaDB mTypes.MetaDB, log log.Logger) cveinfo.CveInfo {
 		},
 	}
 
-	return &cveinfo.BaseCveInfo{
-		Log:     log,
-		Scanner: scanner,
-		MetaDB:  metaDB,
-	}
+	return &scanner
 }
 
 type mockServiceForRetry struct {

pkg/extensions/extension_search.go

@@ -4,9 +4,7 @@
 package extensions
 
 import (
-	"context"
 	"net/http"
-	"sync"
 	"time"
 
 	gqlHandler "github.com/99designs/gqlgen/graphql/handler"
@@ -24,145 +22,58 @@ import (
 	"zotregistry.io/zot/pkg/storage"
 )
 
-type (
-	CveInfo cveinfo.CveInfo
-	state   int
-)
+const scanInterval = 15 * time.Minute
 
-const (
-	pending state = iota
-	running
-	done
-)
+type CveScanner cveinfo.Scanner
 
 func IsBuiltWithSearchExtension() bool {
 	return true
 }
 
-func GetCVEInfo(config *config.Config, storeController storage.StoreController,
+func GetCveScanner(conf *config.Config, storeController storage.StoreController,
 	metaDB mTypes.MetaDB, log log.Logger,
-) CveInfo {
-	if config.Extensions.Search == nil || !*config.Extensions.Search.Enable || config.Extensions.Search.CVE == nil {
+) CveScanner {
+	if !conf.IsCveScanningEnabled() {
 		return nil
 	}
 
-	dbRepository := config.Extensions.Search.CVE.Trivy.DBRepository
-	javaDBRepository := config.Extensions.Search.CVE.Trivy.JavaDBRepository
+	dbRepository := conf.Extensions.Search.CVE.Trivy.DBRepository
+	javaDBRepository := conf.Extensions.Search.CVE.Trivy.JavaDBRepository
 
-	return cveinfo.NewCVEInfo(storeController, metaDB, dbRepository, javaDBRepository, log)
+	return cveinfo.NewScanner(storeController, metaDB, dbRepository, javaDBRepository, log)
 }
 
-func EnableSearchExtension(config *config.Config, storeController storage.StoreController,
-	metaDB mTypes.MetaDB, taskScheduler *scheduler.Scheduler, cveInfo CveInfo, log log.Logger,
+func EnableSearchExtension(conf *config.Config, storeController storage.StoreController,
+	metaDB mTypes.MetaDB, taskScheduler *scheduler.Scheduler, cveScanner CveScanner, log log.Logger,
 ) {
-	if config.Extensions.Search != nil && *config.Extensions.Search.Enable && config.Extensions.Search.CVE != nil {
-		updateInterval := config.Extensions.Search.CVE.UpdateInterval
+	if conf.IsCveScanningEnabled() {
+		updateInterval := conf.Extensions.Search.CVE.UpdateInterval
 
-		downloadTrivyDB(updateInterval, taskScheduler, cveInfo, log)
+		downloadTrivyDB(updateInterval, taskScheduler, cveScanner, log)
+		startScanner(scanInterval, metaDB, taskScheduler, cveScanner, log)
 	} else {
 		log.Info().Msg("CVE config not provided, skipping CVE update")
 	}
 }
 
-func downloadTrivyDB(interval time.Duration, sch *scheduler.Scheduler, cveInfo CveInfo, log log.Logger) {
-	generator := NewTrivyTaskGenerator(interval, cveInfo, log)
+func downloadTrivyDB(interval time.Duration, sch *scheduler.Scheduler, cveScanner CveScanner, log log.Logger) {
+	generator := cveinfo.NewDBUpdateTaskGenerator(interval, cveScanner, log)
 
 	log.Info().Msg("Submitting CVE DB update scheduler")
 	sch.SubmitGenerator(generator, interval, scheduler.HighPriority)
 }
 
-func NewTrivyTaskGenerator(interval time.Duration, cveInfo CveInfo, log log.Logger) *TrivyTaskGenerator {
-	generator := &TrivyTaskGenerator{interval, cveInfo, log, pending, 0, time.Now(), &sync.Mutex{}}
-
-	return generator
-}
-
-type TrivyTaskGenerator struct {
-	interval     time.Duration
-	cveInfo      CveInfo
-	log          log.Logger
-	status       state
-	waitTime     time.Duration
-	lastTaskTime time.Time
-	lock         *sync.Mutex
-}
-
-func (gen *TrivyTaskGenerator) Next() (scheduler.Task, error) {
-	var newTask scheduler.Task
-
-	gen.lock.Lock()
-
-	if gen.status == pending && time.Since(gen.lastTaskTime) >= gen.waitTime {
-		newTask = newTrivyTask(gen.interval, gen.cveInfo, gen, gen.log)
-		gen.status = running
-	}
-	gen.lock.Unlock()
-
-	return newTask, nil
-}
-
-func (gen *TrivyTaskGenerator) IsDone() bool {
-	gen.lock.Lock()
-	status := gen.status
-	gen.lock.Unlock()
-
-	return status == done
-}
-
-func (gen *TrivyTaskGenerator) IsReady() bool {
-	return true
-}
-
-func (gen *TrivyTaskGenerator) Reset() {
-	gen.lock.Lock()
-	gen.status = pending
-	gen.waitTime = 0
-	gen.lock.Unlock()
-}
-
-type trivyTask struct {
-	interval  time.Duration
-	cveInfo   cveinfo.CveInfo
-	generator *TrivyTaskGenerator
-	log       log.Logger
-}
-
-func newTrivyTask(interval time.Duration, cveInfo cveinfo.CveInfo,
-	generator *TrivyTaskGenerator, log log.Logger,
-) *trivyTask {
-	return &trivyTask{interval, cveInfo, generator, log}
-}
-
-func (trivyT *trivyTask) DoWork(ctx context.Context) error {
-	trivyT.log.Info().Msg("updating the CVE database")
-
-	err := trivyT.cveInfo.UpdateDB()
-	if err != nil {
-		trivyT.generator.lock.Lock()
-		trivyT.generator.status = pending
-
-		if trivyT.generator.waitTime == 0 {
-			trivyT.generator.waitTime = time.Second
-		}
-
-		trivyT.generator.waitTime *= 2
-		trivyT.generator.lastTaskTime = time.Now()
-		trivyT.generator.lock.Unlock()
-
-		return err
-	}
-
-	trivyT.generator.lock.Lock()
-	trivyT.generator.lastTaskTime = time.Now()
-	trivyT.generator.status = done
-	trivyT.generator.lock.Unlock()
-	trivyT.log.Info().Str("DB update completed, next update scheduled after", trivyT.interval.String()).Msg("")
+func startScanner(interval time.Duration, metaDB mTypes.MetaDB, sch *scheduler.Scheduler,
+	cveScanner CveScanner, log log.Logger,
+) {
+	generator := cveinfo.NewScanTaskGenerator(metaDB, cveScanner, log)
 
-	return nil
+	log.Info().Msg("Submitting CVE scan scheduler")
+	sch.SubmitGenerator(generator, interval, scheduler.MediumPriority)
 }
 
 func SetupSearchRoutes(conf *config.Config, router *mux.Router, storeController storage.StoreController,
-	metaDB mTypes.MetaDB, cveInfo CveInfo, log log.Logger,
+	metaDB mTypes.MetaDB, cveScanner CveScanner, log log.Logger,
 ) {
 	if !conf.IsSearchEnabled() {
 		log.Info().Msg("skip enabling the search route as the config prerequisites are not met")
@@ -172,6 +83,13 @@ func SetupSearchRoutes(conf *config.Config, router *mux.Router, storeController
 
 	log.Info().Msg("setting up search routes")
 
+	var cveInfo cveinfo.CveInfo
+	if conf.IsCveScanningEnabled() {
+		cveInfo = cveinfo.NewCVEInfo(cveScanner, metaDB, log)
+	} else {
+		cveInfo = nil
+	}
+
 	resConfig := search.GetResolverConfig(log, storeController, metaDB, cveInfo)
 
 	allowedMethods := zcommon.AllowedMethods(http.MethodGet, http.MethodPost)

pkg/extensions/extension_search_disabled.go

@@ -13,11 +13,11 @@ import (
 	"zotregistry.io/zot/pkg/storage"
 )
 
-type CveInfo interface{}
+type CveScanner interface{}
 
-func GetCVEInfo(config *config.Config, storeController storage.StoreController,
+func GetCveScanner(config *config.Config, storeController storage.StoreController,
 	metaDB mTypes.MetaDB, log log.Logger,
-) CveInfo {
+) CveScanner {
 	return nil
 }
 
@@ -27,15 +27,15 @@ func IsBuiltWithSearchExtension() bool {
 
 // EnableSearchExtension ...
 func EnableSearchExtension(config *config.Config, storeController storage.StoreController,
-	metaDB mTypes.MetaDB, scheduler *scheduler.Scheduler, cveInfo CveInfo, log log.Logger,
+	metaDB mTypes.MetaDB, scheduler *scheduler.Scheduler, cveScanner CveScanner, log log.Logger,
 ) {
 	log.Warn().Msg("skipping enabling search extension because given zot binary doesn't include this feature," +
 		"please build a binary that does so")
 }
 
 // SetupSearchRoutes ...
 func SetupSearchRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController,
-	metaDB mTypes.MetaDB, cveInfo CveInfo, log log.Logger,
+	metaDB mTypes.MetaDB, cveScanner CveScanner, log log.Logger,
 ) {
 	log.Warn().Msg("skipping setting up search routes because given zot binary doesn't include this feature," +
 		"please build a binary that does so")

pkg/extensions/search/cve/trivy/cache.go → pkg/extensions/search/cve/cache/cache.go

@@ -1,4 +1,4 @@
-package trivy
+package cache
 
 import (
 	lru "github.com/hashicorp/golang-lru/v2"
@@ -22,6 +22,10 @@ func (cveCache *CveCache) Add(image string, cveMap map[string]cvemodel.CVE) {
 	cveCache.cache.Add(image, cveMap)
 }
 
+func (cveCache *CveCache) Contains(image string) bool {
+	return cveCache.cache.Contains(image)
+}
+
 func (cveCache *CveCache) Get(image string) map[string]cvemodel.CVE {
 	cveMap, ok := cveCache.cache.Get(image)
 	if !ok {