test: validate CycloneDX with the JSON schema (#4956)

* test: validate CycloneDX with the JSON schema

* fix(sbom): move licenses to `name` field in Cyclonedx format (#4941)

* use license.Name instead of Expression

* update tests

* test: add uuid package

* test: compare UUID

---------

Co-authored-by: DmitriyLewen <[email protected]>

go.mod

@@ -91,6 +91,7 @@ require (
 	github.com/testcontainers/testcontainers-go/modules/localstack v0.21.0
 	github.com/tetratelabs/wazero v1.2.1
 	github.com/twitchtv/twirp v8.1.2+incompatible
+	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/xlab/treeprint v1.1.0
 	go.etcd.io/bbolt v1.3.7
 	go.uber.org/zap v1.24.0
@@ -344,7 +345,6 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
-	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/yashtewari/glob-intersection v0.1.0 // indirect
 	github.com/yuin/gopher-lua v1.1.0 // indirect
 	github.com/zclconf/go-cty v1.10.0 // indirect

integration/client_server_test.go

@@ -4,23 +4,21 @@ package integration
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
-	cdx "github.com/CycloneDX/cyclonedx-go"
 	"github.com/docker/go-connections/nat"
-	"github.com/samber/lo"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	testcontainers "github.com/testcontainers/testcontainers-go"
 
 	"github.com/aquasecurity/trivy/pkg/clock"
 	"github.com/aquasecurity/trivy/pkg/report"
+	"github.com/aquasecurity/trivy/pkg/uuid"
 )
 
 type csArgs struct {
@@ -402,45 +400,34 @@ func TestClientServerWithFormat(t *testing.T) {
 }
 
 func TestClientServerWithCycloneDX(t *testing.T) {
-	if *update {
-		t.Skipf("This test doesn't use golden files")
-	}
 	tests := []struct {
-		name                  string
-		args                  csArgs
-		wantComponentsCount   int
-		wantDependenciesCount int
+		name   string
+		args   csArgs
+		golden string
 	}{
 		{
 			name: "fluentd with RubyGems with CycloneDX format",
 			args: csArgs{
 				Format: "cyclonedx",
 				Input:  "testdata/fixtures/images/fluentd-multiple-lockfiles.tar.gz",
 			},
-			wantComponentsCount:   161,
-			wantDependenciesCount: 162,
+			golden: "testdata/fluentd-multiple-lockfiles.cdx.json.golden",
 		},
 	}
 
 	addr, cacheDir := setup(t, setupOptions{})
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			osArgs, outputFile := setupClient(t, tt.args, addr, cacheDir, "")
+			clock.SetFakeTime(t, time.Date(2020, 9, 10, 14, 20, 30, 5, time.UTC))
+			uuid.SetFakeUUID(t, "3ff14136-e09f-4df9-80ea-%012d")
+
+			osArgs, outputFile := setupClient(t, tt.args, addr, cacheDir, tt.golden)
 
 			// Run Trivy client
 			err := execute(osArgs)
 			require.NoError(t, err)
 
-			f, err := os.Open(outputFile)
-			require.NoError(t, err)
-			defer f.Close()
-
-			var got cdx.BOM
-			err = json.NewDecoder(f).Decode(&got)
-			require.NoError(t, err)
-
-			assert.EqualValues(t, tt.wantComponentsCount, len(lo.FromPtr(got.Components)))
-			assert.EqualValues(t, tt.wantDependenciesCount, len(lo.FromPtr(got.Dependencies)))
+			compareCycloneDX(t, tt.golden, outputFile)
 		})
 	}
 }

integration/integration_test.go

@@ -11,18 +11,20 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"testing"
 	"time"
 
 	cdx "github.com/CycloneDX/cyclonedx-go"
+	"github.com/samber/lo"
 	spdxjson "github.com/spdx/tools-golang/json"
 	"github.com/spdx/tools-golang/spdx"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"github.com/xeipuuv/gojsonschema"
 
 	"github.com/aquasecurity/trivy-db/pkg/db"
 	"github.com/aquasecurity/trivy-db/pkg/metadata"
-
 	"github.com/aquasecurity/trivy/pkg/commands"
 	"github.com/aquasecurity/trivy/pkg/dbtest"
 	"github.com/aquasecurity/trivy/pkg/types"
@@ -138,10 +140,7 @@ func readCycloneDX(t *testing.T, filePath string) *cdx.BOM {
 	err = decoder.Decode(bom)
 	require.NoError(t, err)
 
-	// We don't compare values which change each time an SBOM is generated
-	bom.Metadata.Timestamp = ""
-	bom.Metadata.Component.BOMRef = ""
-	bom.SerialNumber = ""
+	// Sort components
 	if bom.Components != nil {
 		sort.Slice(*bom.Components, func(i, j int) bool {
 			return (*bom.Components)[i].Name < (*bom.Components)[j].Name
@@ -153,12 +152,6 @@ func readCycloneDX(t *testing.T, filePath string) *cdx.BOM {
 			})
 		}
 	}
-	if bom.Dependencies != nil {
-		for j := range *bom.Dependencies {
-			(*bom.Dependencies)[j].Ref = ""
-			(*bom.Dependencies)[j].Dependencies = nil
-		}
-	}
 
 	return bom
 }
@@ -212,6 +205,20 @@ func compareCycloneDX(t *testing.T, wantFile, gotFile string) {
 	want := readCycloneDX(t, wantFile)
 	got := readCycloneDX(t, gotFile)
 	assert.Equal(t, want, got)
+
+	// Validate CycloneDX output against the JSON schema
+	schemaLoader := gojsonschema.NewReferenceLoader(got.JSONSchema)
+	documentLoader := gojsonschema.NewGoLoader(got)
+
+	result, err := gojsonschema.Validate(schemaLoader, documentLoader)
+	require.NoError(t, err)
+
+	if valid := result.Valid(); !valid {
+		errs := lo.Map(result.Errors(), func(err gojsonschema.ResultError, _ int) string {
+			return err.String()
+		})
+		assert.True(t, valid, strings.Join(errs, "\n"))
+	}
 }
 
 func compareSpdxJson(t *testing.T, wantFile, gotFile string) {

integration/repo_test.go

@@ -7,12 +7,15 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/aquasecurity/trivy/pkg/clock"
 	ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
 	"github.com/aquasecurity/trivy/pkg/types"
+	"github.com/aquasecurity/trivy/pkg/uuid"
 )
 
 // TestRepository tests `trivy repo` with the local code repositories
@@ -31,7 +34,7 @@ func TestRepository(t *testing.T) {
 		skipFiles      []string
 		skipDirs       []string
 		command        string
-		format         string
+		format         types.Format
 		includeDevDeps bool
 	}
 	tests := []struct {
@@ -367,7 +370,7 @@ func TestRepository(t *testing.T) {
 				command = tt.args.command
 			}
 
-			format := "json"
+			format := types.FormatJSON
 			if tt.args.format != "" {
 				format = tt.args.format
 			}
@@ -380,7 +383,7 @@ func TestRepository(t *testing.T) {
 				"--skip-db-update",
 				"--skip-policy-update",
 				"--format",
-				format,
+				string(format),
 				"--offline-scan",
 			}
 
@@ -458,17 +461,20 @@ func TestRepository(t *testing.T) {
 			osArgs = append(osArgs, "--output", outputFile)
 			osArgs = append(osArgs, tt.args.input)
 
+			clock.SetFakeTime(t, time.Date(2020, 9, 10, 14, 20, 30, 5, time.UTC))
+			uuid.SetFakeUUID(t, "3ff14136-e09f-4df9-80ea-%012d")
+
 			// Run "trivy repo"
 			err := execute(osArgs)
 			require.NoError(t, err)
 
 			// Compare want and got
 			switch format {
-			case "cyclonedx":
+			case types.FormatCycloneDX:
 				compareCycloneDX(t, tt.golden, outputFile)
-			case "spdx-json":
+			case types.FormatSPDXJSON:
 				compareSpdxJson(t, tt.golden, outputFile)
-			case "json":
+			case types.FormatJSON:
 				compareReports(t, tt.golden, outputFile, tt.override)
 			default:
 				require.Fail(t, "invalid format", "format: %s", format)

integration/sbom_test.go

@@ -56,7 +56,7 @@ func TestSBOM(t *testing.T) {
 				format:       "json",
 				artifactType: "cyclonedx",
 			},
-			golden: "testdata/fluentd-multiple-lockfiles-cyclonedx.json.golden",
+			golden: "testdata/fluentd-multiple-lockfiles.json.golden",
 		},
 		{
 			name: "centos7 in in-toto attestation",

integration/testdata/conda-cyclonedx.json.golden

@@ -2,10 +2,10 @@
   "$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json",
   "bomFormat": "CycloneDX",
   "specVersion": "1.5",
-  "serialNumber": "urn:uuid:e7d2faf4-1d5f-4cd7-a792-8b9b5f6fe2d7",
+  "serialNumber": "urn:uuid:3ff14136-e09f-4df9-80ea-000000000001",
   "version": 1,
   "metadata": {
-    "timestamp": "2023-08-04T05:57:22+00:00",
+    "timestamp": "2020-09-10T14:20:30+00:00",
     "tools": [
       {
         "vendor": "aquasecurity",
@@ -14,7 +14,7 @@
       }
     ],
     "component": {
-      "bom-ref": "a80bd6fc-91e4-4e42-9941-eafc2423d031",
+      "bom-ref": "3ff14136-e09f-4df9-80ea-000000000002",
       "type": "application",
       "name": "testdata/fixtures/repo/conda",
       "properties": [
@@ -77,7 +77,7 @@
   ],
   "dependencies": [
     {
-      "ref": "a80bd6fc-91e4-4e42-9941-eafc2423d031",
+      "ref": "3ff14136-e09f-4df9-80ea-000000000002",
       "dependsOn": [
         "pkg:conda/[email protected]?file_path=miniconda3%2Fenvs%2Ftestenv%2Fconda-meta%2Fopenssl-1.1.1q-h7f8727e_0.json",
         "pkg:conda/[email protected]?file_path=miniconda3%2Fenvs%2Ftestenv%2Fconda-meta%2Fpip-22.2.2-py38h06a4308_0.json"