feat(cve): implement CVE scanning as background tasks

pkg/api/config/config.go

@@ -348,6 +348,10 @@ func (c *Config) IsSearchEnabled() bool {
 	return c.Extensions != nil && c.Extensions.Search != nil && *c.Extensions.Search.Enable
 }
 
+func (c *Config) IsCveScanningEnabled() bool {
+	return c.IsSearchEnabled() && c.Extensions.Search.CVE != nil
+}
+
 func (c *Config) IsUIEnabled() bool {
 	return c.Extensions != nil && c.Extensions.UI != nil && *c.Extensions.UI.Enable
 }

pkg/api/controller.go

@@ -46,7 +46,7 @@ type Controller struct {
 	Audit           *log.Logger
 	Server          *http.Server
 	Metrics         monitoring.MetricServer
-	CveInfo         ext.CveInfo
+	CveScanner      ext.CveScanner
 	SyncOnDemand    SyncOnDemand
 	RelyingParties  map[string]rp.RelyingParty
 	CookieStore     sessions.Store
@@ -241,7 +241,7 @@ func (c *Controller) Init(reloadCtx context.Context) error {
 func (c *Controller) InitCVEInfo() {
 	// Enable CVE extension if extension config is provided
 	if c.Config != nil && c.Config.Extensions != nil {
-		c.CveInfo = ext.GetCVEInfo(c.Config, c.StoreController, c.MetaDB, c.Log)
+		c.CveScanner = ext.GetCveScanner(c.Config, c.StoreController, c.MetaDB, c.Log)
 	}
 }
 
@@ -347,7 +347,7 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
 	// Enable extensions if extension config is provided for DefaultStore
 	if c.Config != nil && c.Config.Extensions != nil {
 		ext.EnableMetricsExtension(c.Config, c.Log, c.Config.Storage.RootDirectory)
-		ext.EnableSearchExtension(c.Config, c.StoreController, c.MetaDB, taskScheduler, c.CveInfo, c.Log)
+		ext.EnableSearchExtension(c.Config, c.StoreController, c.MetaDB, taskScheduler, c.CveScanner, c.Log)
 	}
 
 	if c.Config.Storage.SubPaths != nil {

pkg/api/routes.go

@@ -184,7 +184,7 @@ func (rh *RouteHandler) SetupRoutes() {
 
 	// Preconditions for enabling the actual extension routes are part of extensions themselves
 	ext.SetupMetricsRoutes(rh.c.Config, rh.c.Router, authHandler, rh.c.Log, rh.c.Metrics)
-	ext.SetupSearchRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.MetaDB, rh.c.CveInfo,
+	ext.SetupSearchRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.MetaDB, rh.c.CveScanner,
 		rh.c.Log)
 	ext.SetupImageTrustRoutes(rh.c.Config, prefixedRouter, rh.c.MetaDB, rh.c.Log)
 	ext.SetupMgmtRoutes(rh.c.Config, prefixedRouter, rh.c.Log)

pkg/cli/client/cve_cmd_internal_test.go

@@ -531,7 +531,7 @@ func TestNegativeServerResponse(t *testing.T) {
 			panic(err)
 		}
 
-		ctlr.CveInfo = getMockCveInfo(ctlr.MetaDB, ctlr.Log)
+		ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB)
 
 		go func() {
 			if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) {
@@ -606,7 +606,7 @@ func TestServerCVEResponse(t *testing.T) {
 		panic(err)
 	}
 
-	ctlr.CveInfo = getMockCveInfo(ctlr.MetaDB, ctlr.Log)
+	ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB)
 
 	go func() {
 		if err := ctlr.Run(ctx); !errors.Is(err, http.ErrServerClosed) {
@@ -947,39 +947,35 @@ func TestCVESort(t *testing.T) {
 		panic(err)
 	}
 
-	ctlr.CveInfo = cveinfo.BaseCveInfo{
-		Log:    ctlr.Log,
-		MetaDB: mocks.MetaDBMock{},
-		Scanner: mocks.CveScannerMock{
-			ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) {
-				return map[string]cvemodel.CVE{
-					"CVE-2023-1255": {
-						ID:       "CVE-2023-1255",
-						Severity: "LOW",
-						Title:    "Input buffer over-read in AES-XTS implementation and testing",
-					},
-					"CVE-2023-2650": {
-						ID:       "CVE-2023-2650",
-						Severity: "MEDIUM",
-						Title:    "Possible DoS translating ASN.1 object identifier and executer",
-					},
-					"CVE-2023-2975": {
-						ID:       "CVE-2023-2975",
-						Severity: "HIGH",
-						Title:    "AES-SIV cipher implementation contains a bug that can break",
-					},
-					"CVE-2023-3446": {
-						ID:       "CVE-2023-3446",
-						Severity: "CRITICAL",
-						Title:    "Excessive time spent checking DH keys and parenthesis",
-					},
-					"CVE-2023-3817": {
-						ID:       "CVE-2023-3817",
-						Severity: "MEDIUM",
-						Title:    "Excessive time spent checking DH q parameter and arguments",
-					},
-				}, nil
-			},
+	ctlr.CveScanner = mocks.CveScannerMock{
+		ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) {
+			return map[string]cvemodel.CVE{
+				"CVE-2023-1255": {
+					ID:       "CVE-2023-1255",
+					Severity: "LOW",
+					Title:    "Input buffer over-read in AES-XTS implementation and testing",
+				},
+				"CVE-2023-2650": {
+					ID:       "CVE-2023-2650",
+					Severity: "MEDIUM",
+					Title:    "Possible DoS translating ASN.1 object identifier and executer",
+				},
+				"CVE-2023-2975": {
+					ID:       "CVE-2023-2975",
+					Severity: "HIGH",
+					Title:    "AES-SIV cipher implementation contains a bug that can break",
+				},
+				"CVE-2023-3446": {
+					ID:       "CVE-2023-3446",
+					Severity: "CRITICAL",
+					Title:    "Excessive time spent checking DH keys and parenthesis",
+				},
+				"CVE-2023-3817": {
+					ID:       "CVE-2023-3817",
+					Severity: "MEDIUM",
+					Title:    "Excessive time spent checking DH q parameter and arguments",
+				},
+			}, nil
 		},
 	}
 
@@ -1373,7 +1369,7 @@ func TestCVECommandErrors(t *testing.T) {
 	})
 }
 
-func getMockCveInfo(metaDB mTypes.MetaDB, log log.Logger) cveinfo.CveInfo {
+func getMockCveScanner(metaDB mTypes.MetaDB) cveinfo.Scanner {
 	// MetaDB loaded with initial data now mock the scanner
 	// Setup test CVE data in mock scanner
 	scanner := mocks.CveScannerMock{
@@ -1472,11 +1468,7 @@ func getMockCveInfo(metaDB mTypes.MetaDB, log log.Logger) cveinfo.CveInfo {
 		},
 	}
 
-	return &cveinfo.BaseCveInfo{
-		Log:     log,
-		Scanner: scanner,
-		MetaDB:  metaDB,
-	}
+	return &scanner
 }
 
 type mockServiceForRetry struct {

pkg/extensions/extension_search.go

@@ -4,9 +4,7 @@
 package extensions
 
 import (
-	"context"
 	"net/http"
-	"sync"
 	"time"
 
 	gqlHandler "github.com/99designs/gqlgen/graphql/handler"
@@ -24,145 +22,58 @@ import (
 	"zotregistry.io/zot/pkg/storage"
 )
 
-type (
-	CveInfo cveinfo.CveInfo
-	state   int
-)
+const scanInterval = 15 * time.Minute
 
-const (
-	pending state = iota
-	running
-	done
-)
+type CveScanner cveinfo.Scanner
 
 func IsBuiltWithSearchExtension() bool {
 	return true
 }
 
-func GetCVEInfo(config *config.Config, storeController storage.StoreController,
+func GetCveScanner(conf *config.Config, storeController storage.StoreController,
 	metaDB mTypes.MetaDB, log log.Logger,
-) CveInfo {
-	if config.Extensions.Search == nil || !*config.Extensions.Search.Enable || config.Extensions.Search.CVE == nil {
+) CveScanner {
+	if !conf.IsCveScanningEnabled() {
 		return nil
 	}
 
-	dbRepository := config.Extensions.Search.CVE.Trivy.DBRepository
-	javaDBRepository := config.Extensions.Search.CVE.Trivy.JavaDBRepository
+	dbRepository := conf.Extensions.Search.CVE.Trivy.DBRepository
+	javaDBRepository := conf.Extensions.Search.CVE.Trivy.JavaDBRepository
 
-	return cveinfo.NewCVEInfo(storeController, metaDB, dbRepository, javaDBRepository, log)
+	return cveinfo.NewScanner(storeController, metaDB, dbRepository, javaDBRepository, log)
 }
 
-func EnableSearchExtension(config *config.Config, storeController storage.StoreController,
-	metaDB mTypes.MetaDB, taskScheduler *scheduler.Scheduler, cveInfo CveInfo, log log.Logger,
+func EnableSearchExtension(conf *config.Config, storeController storage.StoreController,
+	metaDB mTypes.MetaDB, taskScheduler *scheduler.Scheduler, cveScanner CveScanner, log log.Logger,
 ) {
-	if config.Extensions.Search != nil && *config.Extensions.Search.Enable && config.Extensions.Search.CVE != nil {
-		updateInterval := config.Extensions.Search.CVE.UpdateInterval
+	if conf.IsCveScanningEnabled() {
+		updateInterval := conf.Extensions.Search.CVE.UpdateInterval
 
-		downloadTrivyDB(updateInterval, taskScheduler, cveInfo, log)
+		downloadTrivyDB(updateInterval, taskScheduler, cveScanner, log)
+		startScanner(scanInterval, metaDB, taskScheduler, cveScanner, log)
 	} else {
 		log.Info().Msg("CVE config not provided, skipping CVE update")
 	}
 }
 
-func downloadTrivyDB(interval time.Duration, sch *scheduler.Scheduler, cveInfo CveInfo, log log.Logger) {
-	generator := NewTrivyTaskGenerator(interval, cveInfo, log)
+func downloadTrivyDB(interval time.Duration, sch *scheduler.Scheduler, cveScanner CveScanner, log log.Logger) {
+	generator := cveinfo.NewDBUpdateTaskGenerator(interval, cveScanner, log)
 
 	log.Info().Msg("Submitting CVE DB update scheduler")
 	sch.SubmitGenerator(generator, interval, scheduler.HighPriority)
 }
 
-func NewTrivyTaskGenerator(interval time.Duration, cveInfo CveInfo, log log.Logger) *TrivyTaskGenerator {
-	generator := &TrivyTaskGenerator{interval, cveInfo, log, pending, 0, time.Now(), &sync.Mutex{}}
-
-	return generator
-}
-
-type TrivyTaskGenerator struct {
-	interval     time.Duration
-	cveInfo      CveInfo
-	log          log.Logger
-	status       state
-	waitTime     time.Duration
-	lastTaskTime time.Time
-	lock         *sync.Mutex
-}
-
-func (gen *TrivyTaskGenerator) Next() (scheduler.Task, error) {
-	var newTask scheduler.Task
-
-	gen.lock.Lock()
-
-	if gen.status == pending && time.Since(gen.lastTaskTime) >= gen.waitTime {
-		newTask = newTrivyTask(gen.interval, gen.cveInfo, gen, gen.log)
-		gen.status = running
-	}
-	gen.lock.Unlock()
-
-	return newTask, nil
-}
-
-func (gen *TrivyTaskGenerator) IsDone() bool {
-	gen.lock.Lock()
-	status := gen.status
-	gen.lock.Unlock()
-
-	return status == done
-}
-
-func (gen *TrivyTaskGenerator) IsReady() bool {
-	return true
-}
-
-func (gen *TrivyTaskGenerator) Reset() {
-	gen.lock.Lock()
-	gen.status = pending
-	gen.waitTime = 0
-	gen.lock.Unlock()
-}
-
-type trivyTask struct {
-	interval  time.Duration
-	cveInfo   cveinfo.CveInfo
-	generator *TrivyTaskGenerator
-	log       log.Logger
-}
-
-func newTrivyTask(interval time.Duration, cveInfo cveinfo.CveInfo,
-	generator *TrivyTaskGenerator, log log.Logger,
-) *trivyTask {
-	return &trivyTask{interval, cveInfo, generator, log}
-}
-
-func (trivyT *trivyTask) DoWork(ctx context.Context) error {
-	trivyT.log.Info().Msg("updating the CVE database")
-
-	err := trivyT.cveInfo.UpdateDB()
-	if err != nil {
-		trivyT.generator.lock.Lock()
-		trivyT.generator.status = pending
-
-		if trivyT.generator.waitTime == 0 {
-			trivyT.generator.waitTime = time.Second
-		}
-
-		trivyT.generator.waitTime *= 2
-		trivyT.generator.lastTaskTime = time.Now()
-		trivyT.generator.lock.Unlock()
-
-		return err
-	}
-
-	trivyT.generator.lock.Lock()
-	trivyT.generator.lastTaskTime = time.Now()
-	trivyT.generator.status = done
-	trivyT.generator.lock.Unlock()
-	trivyT.log.Info().Str("DB update completed, next update scheduled after", trivyT.interval.String()).Msg("")
+func startScanner(interval time.Duration, metaDB mTypes.MetaDB, sch *scheduler.Scheduler,
+	cveScanner CveScanner, log log.Logger,
+) {
+	generator := cveinfo.NewScanTaskGenerator(metaDB, cveScanner, log)
 
-	return nil
+	log.Info().Msg("Submitting CVE scan scheduler")
+	sch.SubmitGenerator(generator, interval, scheduler.MediumPriority)
 }
 
 func SetupSearchRoutes(conf *config.Config, router *mux.Router, storeController storage.StoreController,
-	metaDB mTypes.MetaDB, cveInfo CveInfo, log log.Logger,
+	metaDB mTypes.MetaDB, cveScanner CveScanner, log log.Logger,
 ) {
 	if !conf.IsSearchEnabled() {
 		log.Info().Msg("skip enabling the search route as the config prerequisites are not met")
@@ -172,6 +83,13 @@ func SetupSearchRoutes(conf *config.Config, router *mux.Router, storeController
 
 	log.Info().Msg("setting up search routes")
 
+	var cveInfo cveinfo.CveInfo
+	if conf.IsCveScanningEnabled() {
+		cveInfo = cveinfo.NewCVEInfo(cveScanner, metaDB, log)
+	} else {
+		cveInfo = nil
+	}
+
 	resConfig := search.GetResolverConfig(log, storeController, metaDB, cveInfo)
 
 	allowedMethods := zcommon.AllowedMethods(http.MethodGet, http.MethodPost)

pkg/extensions/extension_search_disabled.go

@@ -13,11 +13,11 @@ import (
 	"zotregistry.io/zot/pkg/storage"
 )
 
-type CveInfo interface{}
+type CveScanner interface{}
 
-func GetCVEInfo(config *config.Config, storeController storage.StoreController,
+func GetCveScanner(config *config.Config, storeController storage.StoreController,
 	metaDB mTypes.MetaDB, log log.Logger,
-) CveInfo {
+) CveScanner {
 	return nil
 }
 
@@ -27,15 +27,15 @@ func IsBuiltWithSearchExtension() bool {
 
 // EnableSearchExtension ...
 func EnableSearchExtension(config *config.Config, storeController storage.StoreController,
-	metaDB mTypes.MetaDB, scheduler *scheduler.Scheduler, cveInfo CveInfo, log log.Logger,
+	metaDB mTypes.MetaDB, scheduler *scheduler.Scheduler, cveScanner CveScanner, log log.Logger,
 ) {
 	log.Warn().Msg("skipping enabling search extension because given zot binary doesn't include this feature," +
 		"please build a binary that does so")
 }
 
 // SetupSearchRoutes ...
 func SetupSearchRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController,
-	metaDB mTypes.MetaDB, cveInfo CveInfo, log log.Logger,
+	metaDB mTypes.MetaDB, cveScanner CveScanner, log log.Logger,
 ) {
 	log.Warn().Msg("skipping setting up search routes because given zot binary doesn't include this feature," +
 		"please build a binary that does so")

pkg/extensions/search/cve/trivy/cache.go → pkg/extensions/search/cve/cache/cache.go

@@ -1,4 +1,4 @@
-package trivy
+package cache
 
 import (
 	lru "github.com/hashicorp/golang-lru/v2"
@@ -22,6 +22,10 @@ func (cveCache *CveCache) Add(image string, cveMap map[string]cvemodel.CVE) {
 	cveCache.cache.Add(image, cveMap)
 }
 
+func (cveCache *CveCache) Contains(image string) bool {
+	return cveCache.cache.Contains(image)
+}
+
 func (cveCache *CveCache) Get(image string) map[string]cvemodel.CVE {
 	cveMap, ok := cveCache.cache.Get(image)
 	if !ok {