From e0d9061bc182a8b3d7e98f004bd272766847dfc2 Mon Sep 17 00:00:00 2001 From: ppandrate Date: Wed, 22 Jan 2025 16:32:45 -0800 Subject: [PATCH 01/15] update UT to use spdx 3.0 generator --- src/Microsoft.Sbom.Api/Utils/Constants.cs | 6 +++ .../Constants.cs | 8 ++++ .../Generator.cs | 41 ++++++++++++++++++- .../Microsoft.Sbom.Api.Tests.csproj | 1 + .../ManifestGenerationWorkflowTests.cs | 30 ++++++++++---- 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Sbom.Api/Utils/Constants.cs b/src/Microsoft.Sbom.Api/Utils/Constants.cs index 5771039fc..ce99fc81f 100644 --- a/src/Microsoft.Sbom.Api/Utils/Constants.cs +++ b/src/Microsoft.Sbom.Api/Utils/Constants.cs @@ -19,6 +19,12 @@ public static class Constants Version = "2.2" }; + public static ManifestInfo SPDX30ManifestInfo = new ManifestInfo + { + Name = "SPDX", + Version = "3.0" + }; + public static SbomSpecification SPDX22Specification = SPDX22ManifestInfo.ToSBOMSpecification(); // TODO: move to test csproj diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs index 76d545e23..eb164eb88 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs @@ -19,6 +19,14 @@ internal static class Constants internal const string SPDXContextHeaderName = "@context"; internal const string SPDXGraphHeaderName = "@graph"; + internal const string SPDXVersionHeaderName = "spdxVersion"; + internal const string DataLicenseHeaderName = "dataLicense"; + internal const string SPDXIDHeaderName = "SPDXID"; + internal const string DocumentNameHeaderName = "name"; + internal const string DocumentNamespaceHeaderName = "documentNamespace"; + internal const string CreationInfoHeaderName = "creationInfo"; + internal const string DocumentDescribesHeaderName = "documentDescribes"; + /// /// Use if SPDX creator /// - made an attempt to retrieve the info but cannot determine correct values. diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs index f46ccb225..b21846179 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs @@ -615,5 +615,44 @@ private PackageVerificationCode GetPackageVerificationCode(IInternalMetadataProv public ManifestInfo RegisterManifest() => Constants.Spdx30ManifestInfo; - IDictionary IManifestGenerator.GetMetadataDictionary(IInternalMetadataProvider internalMetadataProvider) => throw new NotSupportedException(); + public IDictionary GetMetadataDictionary(IInternalMetadataProvider internalMetadataProvider) + { + if (internalMetadataProvider is null) + { + throw new ArgumentNullException(nameof(internalMetadataProvider)); + } + + var generationData = internalMetadataProvider.GetGenerationData(Constants.Spdx30ManifestInfo); + + var sbomToolName = internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolName); + var sbomToolVersion = internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolVersion); + var packageName = internalMetadataProvider.GetPackageName(); + var packageVersion = internalMetadataProvider.GetPackageVersion(); + + var documentName = string.Format(Constants.SPDXDocumentNameFormatString, packageName, packageVersion); + + var creationInfo = new CreationInfo + { + Created = internalMetadataProvider.GetGenerationTimestamp(), + CreatedBy = new List + { + $"{internalMetadataProvider.GetPackageSupplier()}", + }, + CreatedUsing = new List + { + $"{sbomToolName}-{sbomToolVersion}" + } + }; + + return new Dictionary + { + { Constants.SPDXVersionHeaderName, Version }, + { Constants.DataLicenseHeaderName, Constants.DataLicenceValue }, + { Constants.SPDXIDHeaderName, Constants.SPDXDocumentIdValue }, + { Constants.DocumentNameHeaderName, documentName }, + { Constants.DocumentNamespaceHeaderName, internalMetadataProvider.GetDocumentNamespace() }, + { Constants.CreationInfoHeaderName, creationInfo }, + { Constants.DocumentDescribesHeaderName, new string[] { generationData.RootPackageId } } + }; + } } diff --git a/test/Microsoft.Sbom.Api.Tests/Microsoft.Sbom.Api.Tests.csproj b/test/Microsoft.Sbom.Api.Tests/Microsoft.Sbom.Api.Tests.csproj index 5fa6a6af1..b97d5ee44 100644 --- a/test/Microsoft.Sbom.Api.Tests/Microsoft.Sbom.Api.Tests.csproj +++ b/test/Microsoft.Sbom.Api.Tests/Microsoft.Sbom.Api.Tests.csproj @@ -19,6 +19,7 @@ + diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs index dbfacf17d..352cd1091 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs @@ -19,6 +19,7 @@ using Microsoft.Sbom.Api.Hashing; using Microsoft.Sbom.Api.Manifest; using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Metadata; using Microsoft.Sbom.Api.Output; using Microsoft.Sbom.Api.Output.Telemetry; using Microsoft.Sbom.Api.PackageDetails; @@ -36,12 +37,16 @@ using Microsoft.Sbom.Contracts.Enums; using Microsoft.Sbom.Extensions; using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Parsers.Spdx22SbomParser; +using Microsoft.Sbom.Parsers.Spdx30SbomParser; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Newtonsoft.Json.Linq; using Serilog.Events; +using Spectre.Console; using Checksum = Microsoft.Sbom.Contracts.Checksum; using Constants = Microsoft.Sbom.Api.Utils.Constants; +using Generator = Microsoft.Sbom.Parsers.Spdx30SbomParser.Generator; using IComponentDetector = Microsoft.Sbom.Api.Utils.IComponentDetector; using ILogger = Serilog.ILogger; @@ -82,19 +87,19 @@ public void Setup() [DataRow(false, true)] public async Task ManifestGenerationWorkflowTests_Succeeds(bool deleteExistingManifestDir, bool isDefaultSourceManifestDirPath) { - var manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new TestManifestGenerator() }); + var manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new Generator() }); manifestGeneratorProvider.Init(); var metadataBuilder = new MetadataBuilder( mockLogger.Object, manifestGeneratorProvider, - Constants.TestManifestInfo, + Constants.SPDX30ManifestInfo, recorderMock.Object); var jsonFilePath = "/root/_manifest/manifest.json"; ISbomConfig sbomConfig = new SbomConfig(fileSystemMock.Object) { - ManifestInfo = Constants.TestManifestInfo, + ManifestInfo = Constants.SPDX30ManifestInfo, ManifestJsonDirPath = "/root/_manifest", ManifestJsonFilePath = jsonFilePath, MetadataBuilder = metadataBuilder, @@ -106,13 +111,18 @@ public async Task ManifestGenerationWorkflowTests_Succeeds(bool deleteExistingMa { { MetadataKey.Build_BuildId, 12 }, { MetadataKey.Build_DefinitionName, "test" }, + { MetadataKey.SBOMToolName, "testMicrosoft" }, + { MetadataKey.SBOMToolVersion, "Tool.Version" }, + { MetadataKey.PackageVersion, "Package.Version" }, + { MetadataKey.PackageSupplier, "testPackageSupplier" } }); + var localMetadataProvider = new LocalMetadataProvider(configurationMock.Object); var sbomConfigs = new SbomConfigProvider( - new IManifestConfigHandler[] { mockConfigHandler.Object }, - new IMetadataProvider[] { mockMetadataProvider.Object }, - mockLogger.Object, - recorderMock.Object); + manifestConfigHandlers: new IManifestConfigHandler[] { mockConfigHandler.Object }, + metadataProviders: new IMetadataProvider[] { localMetadataProvider, mockMetadataProvider.Object }, + logger: mockLogger.Object, + recorder: recorderMock.Object); using var manifestStream = new MemoryStream(); using var manifestWriter = new StreamWriter(manifestStream); @@ -144,6 +154,12 @@ public async Task ManifestGenerationWorkflowTests_Succeeds(bool deleteExistingMa configurationMock.SetupGet(c => c.BuildComponentPath).Returns(new ConfigurationSetting { Value = "/root" }); configurationMock.SetupGet(c => c.FollowSymlinks).Returns(new ConfigurationSetting { Value = true }); + // Added config settings + configurationMock.SetupGet(c => c.PackageName).Returns(new ConfigurationSetting("the-package-name")); + configurationMock.SetupGet(c => c.PackageVersion).Returns(new ConfigurationSetting("the-package-version")); + configurationMock.SetupGet(c => c.NamespaceUriUniquePart).Returns(new ConfigurationSetting("some-custom-value-here")); + configurationMock.SetupGet(c => c.NamespaceUriBase).Returns(new ConfigurationSetting("http://sbom.microsoft")); + fileSystemMock .Setup(f => f.CreateDirectory( It.Is(d => d == "/root/_manifest"))) From 414a3fb21c11ae475608dc344b178ca0d71f33f1 Mon Sep 17 00:00:00 2001 From: ppandrate Date: Fri, 31 Jan 2025 12:56:47 -0800 Subject: [PATCH 02/15] generator api changes + restructuring classes --- .../Executors/GenerateResult.cs | 28 ++++ .../Executors/IJsonSerializationStrategy.cs | 32 ++++ .../JsonSerializationStrategyFactory.cs | 19 +++ .../Executors/Spdx2SerializationStrategy.cs | 140 ++++++++++++++++++ .../Executors/Spdx3SerializationStrategy.cs | 91 ++++++++++++ .../Output/ManifestToolJsonSerializer.cs | 5 + .../ExternalDocumentReferenceGenerator.cs | 45 +++--- .../Workflows/Helpers/FileArrayGenerator.cs | 44 +++--- .../Workflows/Helpers/IJsonArrayGenerator.cs | 7 +- .../Helpers/PackageArrayGenerator.cs | 52 ++++--- .../Helpers/RelationshipsArrayGenerator.cs | 117 ++++++++------- .../Workflows/SBOMGenerationWorkflow.cs | 36 +++-- .../IManifestToolJsonSerializer.cs | 7 + .../Generator.cs | 20 ++- .../SBOMGeneratorTest.cs | 10 +- .../RelationshipsArrayGeneratorTest.cs | 12 +- .../ManifestGenerationWorkflowTests.cs | 90 +++++++---- 17 files changed, 569 insertions(+), 186 deletions(-) create mode 100644 src/Microsoft.Sbom.Api/Executors/GenerateResult.cs create mode 100644 src/Microsoft.Sbom.Api/Executors/IJsonSerializationStrategy.cs create mode 100644 src/Microsoft.Sbom.Api/Executors/JsonSerializationStrategyFactory.cs create mode 100644 src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs create mode 100644 src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs diff --git a/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs b/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs new file mode 100644 index 000000000..8839a2f19 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Workflows.Helpers; + +/// +/// Result from GenerateAsync +/// +public class GenerateResult + { + public List Errors { get; set; } + + public Dictionary> SerializerToJsonDocuments { get; set; } + + public GenerateResult(List errors, Dictionary> serializerToJsonDocuments) + { + Errors = errors; + SerializerToJsonDocuments = serializerToJsonDocuments; + } + + // TODO: add an optional header name? so that you don't have to hardcode it in the Spdx2SerializationStrategy +} diff --git a/src/Microsoft.Sbom.Api/Executors/IJsonSerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/IJsonSerializationStrategy.cs new file mode 100644 index 000000000..4751dfbec --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/IJsonSerializationStrategy.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Workflows.Helpers; + +public interface IJsonSerializationStrategy +{ + public void AddToFilesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config); + + public void AddToPackagesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config); + + public bool AddToRelationshipsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config); + + public void AddToExternalDocRefsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config); + + public void AddHeadersToSbom(ISbomConfigProvider sbomConfigs) + { + } + + public Task> WriteJsonObjectsToSbomAsync( + ISbomConfig sbomConfig, + string spdxManifestVersion, + IJsonArrayGenerator fileArrayGenerator, + IJsonArrayGenerator packageArrayGenerator, + IJsonArrayGenerator relationshipsArrayGenerator, + IJsonArrayGenerator externalDocumentReferenceGenerator); +} diff --git a/src/Microsoft.Sbom.Api/Executors/JsonSerializationStrategyFactory.cs b/src/Microsoft.Sbom.Api/Executors/JsonSerializationStrategyFactory.cs new file mode 100644 index 000000000..6b359137b --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/JsonSerializationStrategyFactory.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Api.Workflows.Helpers; + +public static class JsonSerializationStrategyFactory +{ + public static IJsonSerializationStrategy GetStrategy(string manifestInfoSpdxVersion) + { + if (manifestInfoSpdxVersion == "3.0") + { + return new Spdx3SerializationStrategy(); + } + else + { + return new Spdx2SerializationStrategy(); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs new file mode 100644 index 000000000..6a78887bb --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs @@ -0,0 +1,140 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Workflows.Helpers; + +/// +/// Serialization methods for SPDX 2.2. +/// +public class Spdx2SerializationStrategy : IJsonSerializationStrategy +{ + public void AddToFilesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + { + if (config.MetadataBuilder.TryGetFilesArrayHeaderName(out var headerName)) + { + elementsSupportingConfigs.Add(config); + } + } + + public void AddToPackagesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + { + if (config.MetadataBuilder.TryGetPackageArrayHeaderName(out var headerName)) + { + elementsSupportingConfigs.Add(config); + } + } + + public bool AddToRelationshipsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + { + if (config.MetadataBuilder.TryGetRelationshipsHeaderName(out var relationshipArrayHeaderName)) + { + return true; + } + + return false; + } + + public void AddToExternalDocRefsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + { + if (config.MetadataBuilder.TryGetExternalRefArrayHeaderName(out var headerName)) + { + elementsSupportingConfigs.Add(config); + } + } + + public void AddHeadersToSbom(ISbomConfigProvider sbomConfigs) + { + sbomConfigs.ApplyToEachConfig(config => + config.JsonSerializer.WriteJsonString( + config.MetadataBuilder.GetHeaderJsonString(sbomConfigs))); + } + + public async Task> WriteJsonObjectsToSbomAsync( + ISbomConfig sbomConfig, + string spdxManifestVersion, + IJsonArrayGenerator fileArrayGenerator, + IJsonArrayGenerator packageArrayGenerator, + IJsonArrayGenerator relationshipsArrayGenerator, + IJsonArrayGenerator externalDocumentReferenceGenerator) + { + fileArrayGenerator.SbomConfig = sbomConfig; + packageArrayGenerator.SbomConfig = sbomConfig; + relationshipsArrayGenerator.SbomConfig = sbomConfig; + externalDocumentReferenceGenerator.SbomConfig = sbomConfig; + + fileArrayGenerator.SpdxManifestVersion = spdxManifestVersion; + packageArrayGenerator.SpdxManifestVersion = spdxManifestVersion; + relationshipsArrayGenerator.SpdxManifestVersion = spdxManifestVersion; + externalDocumentReferenceGenerator.SpdxManifestVersion = spdxManifestVersion; + + var errors = new List(); + + // Files section + var filesGenerateResult = await fileArrayGenerator.GenerateAsync(); + filesGenerateResult.Errors.AddRange(filesGenerateResult.Errors); + + foreach (var serializer in filesGenerateResult.SerializerToJsonDocuments.Keys) + { + serializer.StartJsonArray("files"); + foreach (var jsonDocument in filesGenerateResult.SerializerToJsonDocuments[serializer]) + { + serializer.Write(jsonDocument); + } + + serializer.EndJsonArray(); + } + + // Packages section + var packagesGenerateResult = await packageArrayGenerator.GenerateAsync(); + packagesGenerateResult.Errors.AddRange(packagesGenerateResult.Errors); + + foreach (var serializer in packagesGenerateResult.SerializerToJsonDocuments.Keys) + { + serializer.StartJsonArray("packages"); + foreach (var jsonDocument in packagesGenerateResult.SerializerToJsonDocuments[serializer]) + { + serializer.Write(jsonDocument); + } + + serializer.EndJsonArray(); + } + + // External Document Reference section + var externalDocumentReferenceGenerateResult = await externalDocumentReferenceGenerator.GenerateAsync(); + externalDocumentReferenceGenerateResult.Errors.AddRange(externalDocumentReferenceGenerateResult.Errors); + + foreach (var serializer in externalDocumentReferenceGenerateResult.SerializerToJsonDocuments.Keys) + { + serializer.StartJsonArray("externalDocumentRefs"); + foreach (var jsonDocument in externalDocumentReferenceGenerateResult.SerializerToJsonDocuments[serializer]) + { + serializer.Write(jsonDocument); + } + + serializer.EndJsonArray(); + } + + // Relationships section + var relationshipGenerateResult = await relationshipsArrayGenerator.GenerateAsync(); + relationshipGenerateResult.Errors.AddRange(relationshipGenerateResult.Errors); + + foreach (var serializer in relationshipGenerateResult.SerializerToJsonDocuments.Keys) + { + serializer.StartJsonArray("relationships"); + foreach (var jsonDocument in relationshipGenerateResult.SerializerToJsonDocuments[serializer]) + { + serializer.Write(jsonDocument); + } + + serializer.EndJsonArray(); + } + + return errors; + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs new file mode 100644 index 000000000..d9e292cf9 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Workflows.Helpers; + +/// +/// Serialization methods for SPDX 3.0. +/// +public class Spdx3SerializationStrategy : IJsonSerializationStrategy +{ + public void AddToFilesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + { + elementsSupportingConfigs.Add(config); + } + + public void AddToPackagesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + { + elementsSupportingConfigs.Add(config); + } + + public bool AddToRelationshipsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + { + return true; + } + + public void AddToExternalDocRefsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + { + elementsSupportingConfigs.Add(config); + } + + public async Task> WriteJsonObjectsToSbomAsync( + ISbomConfig sbomConfig, + string spdxManifestVersion, + IJsonArrayGenerator fileArrayGenerator, + IJsonArrayGenerator packageArrayGenerator, + IJsonArrayGenerator relationshipsArrayGenerator, + IJsonArrayGenerator externalDocumentReferenceGenerator) + { + fileArrayGenerator.SbomConfig = sbomConfig; + packageArrayGenerator.SbomConfig = sbomConfig; + relationshipsArrayGenerator.SbomConfig = sbomConfig; + externalDocumentReferenceGenerator.SbomConfig = sbomConfig; + + fileArrayGenerator.SpdxManifestVersion = spdxManifestVersion; + packageArrayGenerator.SpdxManifestVersion = spdxManifestVersion; + relationshipsArrayGenerator.SpdxManifestVersion = spdxManifestVersion; + externalDocumentReferenceGenerator.SpdxManifestVersion = spdxManifestVersion; + + var generateResult = await fileArrayGenerator.GenerateAsync(); + + // Packages section + var packagesGenerateResult = await packageArrayGenerator.GenerateAsync(); + generateResult.Errors.AddRange(packagesGenerateResult.Errors); + + // External Document Reference section + var externalDocumentReferenceGenerateResult = await externalDocumentReferenceGenerator.GenerateAsync(); + generateResult.Errors.AddRange(externalDocumentReferenceGenerateResult.Errors); + + // Relationships section + var relationshipGenerateResult = await relationshipsArrayGenerator.GenerateAsync(); + generateResult.Errors.AddRange(relationshipGenerateResult.Errors); + + // Write the JSON objects to the SBOM + // TODO: avoid this for loop + // TODO: can add deduplication here + foreach (var serializer in generateResult.SerializerToJsonDocuments.Keys) + { + serializer.StartJsonArray("@graph"); + + var jsonDocuments = generateResult.SerializerToJsonDocuments[serializer]; + foreach (var jsonDocument in jsonDocuments) + { + foreach (var element in jsonDocument.RootElement.EnumerateArray()) + { + serializer.Write(element); + } + } + + serializer.EndJsonArray(); + } + + return generateResult.Errors; + } +} diff --git a/src/Microsoft.Sbom.Api/Output/ManifestToolJsonSerializer.cs b/src/Microsoft.Sbom.Api/Output/ManifestToolJsonSerializer.cs index 709540008..e080c3fb4 100644 --- a/src/Microsoft.Sbom.Api/Output/ManifestToolJsonSerializer.cs +++ b/src/Microsoft.Sbom.Api/Output/ManifestToolJsonSerializer.cs @@ -71,6 +71,11 @@ public void Write(JsonDocument jsonDocument) } } + public void Write(JsonElement jsonElement) + { + jsonElement.WriteTo(jsonWriter); + } + /// /// Write a json string object. This usually is some metadata about the document /// that is generated. diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs index 997bf8ad4..4033b4ca1 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Manifest.Configuration; @@ -11,6 +12,7 @@ using Microsoft.Sbom.Api.Providers; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; using Serilog; namespace Microsoft.Sbom.Api.Workflows.Helpers; @@ -22,49 +24,43 @@ public class ExternalDocumentReferenceGenerator : IJsonArrayGenerator sourcesProviders; private readonly IRecorder recorder; + public ISbomConfig SbomConfig { get; set; } + + public string SpdxManifestVersion { get; set; } + public ExternalDocumentReferenceGenerator( ILogger log, - ISbomConfigProvider sbomConfigs, IEnumerable sourcesProviders, IRecorder recorder) { this.log = log ?? throw new ArgumentNullException(nameof(log)); - this.sbomConfigs = sbomConfigs ?? throw new ArgumentNullException(nameof(sbomConfigs)); this.sourcesProviders = sourcesProviders ?? throw new ArgumentNullException(nameof(sourcesProviders)); this.recorder = recorder ?? throw new ArgumentNullException(nameof(recorder)); } - public async Task> GenerateAsync() + public async Task GenerateAsync() { using (recorder.TraceEvent(Events.ExternalDocumentReferenceGeneration)) { - IList totalErrors = new List(); + var totalErrors = new List(); + var serializersToJsonDocs = new Dictionary>(); var sourcesProviders = this.sourcesProviders .Where(s => s.IsSupported(ProviderType.ExternalDocumentReference)); if (!sourcesProviders.Any()) { log.Debug($"No source providers found for {ProviderType.ExternalDocumentReference}"); - return totalErrors; + return new GenerateResult(totalErrors, serializersToJsonDocs); } // Write the start of the array, if supported. IList externalRefArraySupportingConfigs = new List(); - foreach (var manifestInfo in sbomConfigs.GetManifestInfos()) - { - var config = sbomConfigs.Get(manifestInfo); - if (config.MetadataBuilder.TryGetExternalRefArrayHeaderName(out var externalRefArrayHeaderName)) - { - externalRefArraySupportingConfigs.Add(config); - config.JsonSerializer.StartJsonArray(externalRefArrayHeaderName); - } - } + var serializationStrategy = JsonSerializationStrategyFactory.GetStrategy(SpdxManifestVersion); + serializationStrategy.AddToFilesSupportingConfig(ref externalRefArraySupportingConfigs, this.SbomConfig); foreach (var sourcesProvider in sourcesProviders) { @@ -75,23 +71,24 @@ public async Task> GenerateAsync() await foreach (var jsonResults in jsonDocResults.ReadAllAsync()) { - jsonResults.Serializer.Write(jsonResults.Document); + if (!serializersToJsonDocs.ContainsKey(jsonResults.Serializer)) + { + serializersToJsonDocs[jsonResults.Serializer] = new List(); + } + + serializersToJsonDocs[jsonResults.Serializer].Add(jsonResults.Document); totalJsonDocumentsWritten++; } + log.Debug($"Wrote {totalJsonDocumentsWritten} ExternalDocumentReference elements in the SBOM."); + await foreach (var error in errors.ReadAllAsync()) { totalErrors.Add(error); } } - // Write the end of the array. - foreach (SbomConfig config in externalRefArraySupportingConfigs) - { - config.JsonSerializer.EndJsonArray(); - } - - return totalErrors; + return new GenerateResult(totalErrors, serializersToJsonDocs); } } } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs index 7ea4754eb..cc4789eab 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs @@ -4,12 +4,15 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Manifest.Configuration; using Microsoft.Sbom.Api.Output.Telemetry; using Microsoft.Sbom.Api.Providers; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; using ILogger = Serilog.ILogger; namespace Microsoft.Sbom.Api.Workflows.Helpers; @@ -19,21 +22,21 @@ namespace Microsoft.Sbom.Api.Workflows.Helpers; /// public class FileArrayGenerator : IJsonArrayGenerator { - private readonly ISbomConfigProvider sbomConfigs; - private readonly IEnumerable sourcesProviders; private readonly IRecorder recorder; private readonly ILogger logger; + public ISbomConfig SbomConfig { get; set; } + + public string SpdxManifestVersion { get; set; } + public FileArrayGenerator( - ISbomConfigProvider sbomConfigs, IEnumerable sourcesProviders, IRecorder recorder, ILogger logger) { - this.sbomConfigs = sbomConfigs ?? throw new ArgumentNullException(nameof(sbomConfigs)); this.sourcesProviders = sourcesProviders ?? throw new ArgumentNullException(nameof(sourcesProviders)); this.recorder = recorder ?? throw new ArgumentNullException(nameof(recorder)); this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -47,36 +50,35 @@ public FileArrayGenerator( /// The serializer used to write the SBOM. /// The header key for the file array object. /// - public async Task> GenerateAsync() + public async Task GenerateAsync() { using (recorder.TraceEvent(Events.FilesGeneration)) { - IList totalErrors = new List(); + var totalErrors = new List(); var sourcesProviders = this.sourcesProviders .Where(s => s.IsSupported(ProviderType.Files)); // Write the start of the array, if supported. IList filesArraySupportingSBOMs = new List(); - foreach (var manifestInfo in sbomConfigs.GetManifestInfos()) - { - var config = sbomConfigs.Get(manifestInfo); + var serializationStrategy = JsonSerializationStrategyFactory.GetStrategy(SpdxManifestVersion); + serializationStrategy.AddToFilesSupportingConfig(ref filesArraySupportingSBOMs, this.SbomConfig); - if (config.MetadataBuilder.TryGetFilesArrayHeaderName(out var filesArrayHeaderName)) - { - config.JsonSerializer.StartJsonArray(filesArrayHeaderName); - filesArraySupportingSBOMs.Add(config); - this.logger.Verbose("Started writing files array for {configFile}.", config.ManifestJsonFilePath); - } - } + this.logger.Verbose("Started writing files array for {configFile}.", this.SbomConfig.ManifestJsonFilePath); + var serializersToJsonDocs = new Dictionary>(); foreach (var sourcesProvider in sourcesProviders) { var (jsondDocResults, errors) = sourcesProvider.Get(filesArraySupportingSBOMs); await foreach (var jsonResults in jsondDocResults.ReadAllAsync()) { - jsonResults.Serializer.Write(jsonResults.Document); + if (!serializersToJsonDocs.ContainsKey(jsonResults.Serializer)) + { + serializersToJsonDocs[jsonResults.Serializer] = new List(); + } + + serializersToJsonDocs[jsonResults.Serializer].Add(jsonResults.Document); } await foreach (var error in errors.ReadAllAsync()) @@ -89,13 +91,7 @@ public async Task> GenerateAsync() } } - // Write the end of the array. - foreach (var config in filesArraySupportingSBOMs) - { - config.JsonSerializer.EndJsonArray(); - } - - return totalErrors; + return new GenerateResult(totalErrors, serializersToJsonDocs); } } } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs index 07d001d5e..3f63a9619 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Extensions; namespace Microsoft.Sbom.Api.Workflows.Helpers; @@ -13,10 +14,14 @@ namespace Microsoft.Sbom.Api.Workflows.Helpers; public interface IJsonArrayGenerator where T : IJsonArrayGenerator { + ISbomConfig SbomConfig { get; set; } + + string SpdxManifestVersion { get; set; } + /// /// Generates an array in the json serializer with the headerName and writes all elements of the /// specific type into the array. /// /// The list of failures. - Task> GenerateAsync(); + Task GenerateAsync(); } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs index 9d71dd862..9989b3db1 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Output.Telemetry; @@ -22,53 +23,56 @@ public class PackageArrayGenerator : IJsonArrayGenerator { private readonly ILogger log; - private readonly ISbomConfigProvider sbomConfigs; - private readonly IEnumerable sourcesProviders; private readonly IRecorder recorder; + private readonly ISbomConfigProvider sbomConfigs; + + public ISbomConfig SbomConfig { get; set; } + + public string SpdxManifestVersion { get; set; } + public PackageArrayGenerator( ILogger log, - ISbomConfigProvider sbomConfigs, IEnumerable sourcesProviders, - IRecorder recorder) + IRecorder recorder, + ISbomConfigProvider sbomConfigs) { this.log = log ?? throw new ArgumentNullException(nameof(log)); - this.sbomConfigs = sbomConfigs ?? throw new ArgumentNullException(nameof(sbomConfigs)); this.sourcesProviders = sourcesProviders ?? throw new ArgumentNullException(nameof(sourcesProviders)); this.recorder = recorder ?? throw new ArgumentNullException(nameof(recorder)); + this.sbomConfigs = sbomConfigs ?? throw new ArgumentNullException(nameof(sbomConfigs)); } - public async Task> GenerateAsync() + public async Task GenerateAsync() { using (recorder.TraceEvent(Events.PackagesGeneration)) { - IList totalErrors = new List(); + var totalErrors = new List(); var sourcesProvider = this.sourcesProviders .FirstOrDefault(s => s.IsSupported(ProviderType.Packages)); // Write the start of the array, if supported. IList packagesArraySupportingConfigs = new List(); - foreach (var manifestInfo in sbomConfigs.GetManifestInfos()) - { - var config = sbomConfigs.Get(manifestInfo); - if (config.MetadataBuilder.TryGetPackageArrayHeaderName(out var packagesArrayHeaderName)) - { - packagesArraySupportingConfigs.Add(config); - config.JsonSerializer.StartJsonArray(packagesArrayHeaderName); - } - } + var serializationStrategy = JsonSerializationStrategyFactory.GetStrategy(SpdxManifestVersion); + serializationStrategy.AddToPackagesSupportingConfig(ref packagesArraySupportingConfigs, this.SbomConfig); var (jsonDocResults, errors) = sourcesProvider.Get(packagesArraySupportingConfigs); - // 6. Collect all the json elements and write to the serializer. + // 6. Collect all the json elements to be written to the serializer. var totalJsonDocumentsWritten = 0; + var serializersToJsonDocs = new Dictionary>(); await foreach (var jsonDocResult in jsonDocResults.ReadAllAsync()) { - jsonDocResult.Serializer.Write(jsonDocResult.Document); + if (!serializersToJsonDocs.ContainsKey(jsonDocResult.Serializer)) + { + serializersToJsonDocs[jsonDocResult.Serializer] = new List(); + } + + serializersToJsonDocs[jsonDocResult.Serializer].Add(jsonDocResult.Document); totalJsonDocumentsWritten++; } @@ -91,16 +95,18 @@ public async Task> GenerateAsync() // Write the root package information to the packages array. if (sbomConfig.MetadataBuilder.TryGetRootPackageJson(sbomConfigs, out var generationResult)) { - sbomConfig.JsonSerializer.Write(generationResult?.Document); + if (!serializersToJsonDocs.ContainsKey(sbomConfig.JsonSerializer)) + { + serializersToJsonDocs[sbomConfig.JsonSerializer] = new List(); + } + + serializersToJsonDocs[sbomConfig.JsonSerializer].Add(generationResult?.Document); sbomConfig.Recorder.RecordRootPackageId(generationResult?.ResultMetadata?.EntityId); sbomConfig.Recorder.RecordDocumentId(generationResult?.ResultMetadata?.DocumentId); } - - // Write the end of the array. - sbomConfig.JsonSerializer.EndJsonArray(); } - return totalErrors; + return new GenerateResult(totalErrors, serializersToJsonDocs); } } } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs index 846e43282..c7924ef90 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest.Configuration; using Microsoft.Sbom.Api.Output.Telemetry; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Extensions; @@ -26,92 +27,92 @@ public class RelationshipsArrayGenerator : IJsonArrayGenerator> GenerateAsync() + public async Task GenerateAsync() { using (recorder.TraceEvent(Events.RelationshipsGeneration)) { - IList totalErrors = new List(); + var totalErrors = new List(); + var serializersToJsonDocs = new Dictionary>(); + + IList relationshipsArraySupportingConfigs = new List(); + var serializationStrategy = JsonSerializationStrategyFactory.GetStrategy(SpdxManifestVersion); // Write the relationship array only if supported - foreach (var manifestInfo in sbomConfigs.GetManifestInfos()) + if (serializationStrategy.AddToRelationshipsSupportingConfig(ref relationshipsArraySupportingConfigs, this.SbomConfig)) { - var sbomConfig = sbomConfigs.Get(manifestInfo); - if (sbomConfig.MetadataBuilder.TryGetRelationshipsHeaderName(out var relationshipArrayHeaderName)) + // Get generation data + var generationData = this.SbomConfig.Recorder.GetGenerationData(); + + var jsonChannelsArray = new ChannelReader[] { - sbomConfig.JsonSerializer.StartJsonArray(relationshipArrayHeaderName); + // Packages relationships + generator.Run( + GetRelationships( + RelationshipType.DEPENDS_ON, + generationData), + this.SbomConfig.ManifestInfo), + + // Root package relationship + generator.Run( + GetRelationships( + RelationshipType.DESCRIBES, + generationData.DocumentId, + new string[] { generationData.RootPackageId }), + this.SbomConfig.ManifestInfo), + + // External reference relationship + generator.Run( + GetRelationships( + RelationshipType.PREREQUISITE_FOR, + generationData.RootPackageId, + generationData.ExternalDocumentReferenceIDs), + this.SbomConfig.ManifestInfo), + + // External reference file relationship + generator.Run( + GetRelationships( + RelationshipType.DESCRIBED_BY, + generationData.SPDXFileIds, + generationData.DocumentId), + this.SbomConfig.ManifestInfo), + }; - // Get generation data - var generationData = sbomConfig.Recorder.GetGenerationData(); + // Collect all the json elements and write to the serializer. + var count = 0; - var jsonChannelsArray = new ChannelReader[] - { - // Packages relationships - generator.Run( - GetRelationships( - RelationshipType.DEPENDS_ON, - generationData), - sbomConfig.ManifestInfo), - - // Root package relationship - generator.Run( - GetRelationships( - RelationshipType.DESCRIBES, - generationData.DocumentId, - new string[] { generationData.RootPackageId }), - sbomConfig.ManifestInfo), - - // External reference relationship - generator.Run( - GetRelationships( - RelationshipType.PREREQUISITE_FOR, - generationData.RootPackageId, - generationData.ExternalDocumentReferenceIDs), - sbomConfig.ManifestInfo), - - // External reference file relationship - generator.Run( - GetRelationships( - RelationshipType.DESCRIBED_BY, - generationData.SPDXFileIds, - generationData.DocumentId), - sbomConfig.ManifestInfo), - }; - - // Collect all the json elements and write to the serializer. - var count = 0; - - await foreach (var jsonDoc in channelUtils.Merge(jsonChannelsArray).ReadAllAsync()) + await foreach (var jsonDoc in channelUtils.Merge(jsonChannelsArray).ReadAllAsync()) + { + count++; + if (!serializersToJsonDocs.ContainsKey(this.SbomConfig.JsonSerializer)) { - count++; - sbomConfig.JsonSerializer.Write(jsonDoc); + serializersToJsonDocs[this.SbomConfig.JsonSerializer] = new List(); } - log.Debug($"Wrote {count} relationship elements in the SBOM."); - - // Write the end of the array. - sbomConfig.JsonSerializer.EndJsonArray(); + serializersToJsonDocs[this.SbomConfig.JsonSerializer].Add(jsonDoc); } + + log.Debug($"Wrote {count} relationship elements in the SBOM."); } - return totalErrors; + return new GenerateResult(totalErrors, serializersToJsonDocs); } } @@ -179,4 +180,6 @@ private IEnumerator GetRelationships(RelationshipType relationship } } } + + Task IJsonArrayGenerator.GenerateAsync() => throw new System.NotImplementedException(); } diff --git a/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs index c970120db..4a45f5703 100644 --- a/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs +++ b/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs @@ -16,6 +16,7 @@ using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Extensions; using PowerArgs; +using PowerArgs.Cli; using Serilog; using Constants = Microsoft.Sbom.Api.Utils.Constants; @@ -98,22 +99,25 @@ public virtual async Task RunAsync() { sbomConfigs.ApplyToEachConfig(config => config.JsonSerializer.StartJsonObject()); - // Files section - validErrors = await fileArrayGenerator.GenerateAsync(); - - // Packages section - validErrors.Concat(await packageArrayGenerator.GenerateAsync()); - - // External Document Reference section - validErrors.Concat(await externalDocumentReferenceGenerator.GenerateAsync()); - - // Relationships section - validErrors.Concat(await relationshipsArrayGenerator.GenerateAsync()); - - // Write headers - sbomConfigs.ApplyToEachConfig(config => - config.JsonSerializer.WriteJsonString( - config.MetadataBuilder.GetHeaderJsonString(sbomConfigs))); + // Use the WriteJsonObjectsToSbomAsync method based on the SPDX version in manifest info + foreach (var manifestInfo in sbomConfigs.GetManifestInfos()) + { + var config = sbomConfigs.Get(manifestInfo); + + // Get the appropriate strategy + var serializationStrategy = JsonSerializationStrategyFactory.GetStrategy(manifestInfo.Version); + validErrors = await serializationStrategy.WriteJsonObjectsToSbomAsync( + config, + manifestInfo.Version, + fileArrayGenerator, + packageArrayGenerator, + relationshipsArrayGenerator, + externalDocumentReferenceGenerator). + ConfigureAwait(false); + + // Write headers + serializationStrategy.AddHeadersToSbom(sbomConfigs); + } // Finalize JSON sbomConfigs.ApplyToEachConfig(config => config.JsonSerializer.FinalizeJsonObject()); diff --git a/src/Microsoft.Sbom.Extensions/IManifestToolJsonSerializer.cs b/src/Microsoft.Sbom.Extensions/IManifestToolJsonSerializer.cs index d60092438..d20e11436 100644 --- a/src/Microsoft.Sbom.Extensions/IManifestToolJsonSerializer.cs +++ b/src/Microsoft.Sbom.Extensions/IManifestToolJsonSerializer.cs @@ -44,6 +44,13 @@ public interface IManifestToolJsonSerializer : IAsyncDisposable, IDisposable /// The json document. public void Write(JsonDocument jsonDocument); + /// + /// This writes a json element to the underlying stream. + /// We also call dispose on the JsonDocument once we finish writing. + /// + /// The json document. + void Write(JsonElement jsonElement); + /// /// Write a json string object. This usually is some metadata about the document /// that is generated. diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs index b21846179..1c8603a57 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs @@ -123,17 +123,22 @@ public GenerationResult GenerateJsonDocument(SbomPackage packageInfo) ExternalIdentifierType = "purl", Identifier = packageInfo.PackageUrl }; - } - spdxExternalIdentifier.AddSpdxId(); - spdxPackage.ExternalIdentifier = new List { spdxExternalIdentifier.SpdxId }; + spdxExternalIdentifier.AddSpdxId(); + spdxPackage.ExternalIdentifier = new List { spdxExternalIdentifier.SpdxId }; + } var spdxElementsRelatedToPackageInfo = new List { spdxSupplier, spdxPackage, - spdxExternalIdentifier, }; + + if (spdxExternalIdentifier != null) + { + spdxElementsRelatedToPackageInfo.Add(spdxExternalIdentifier); + } + spdxElementsRelatedToPackageInfo.AddRange(spdxRelationshipAndLicensesFromSbomPackage); var dependOnId = packageInfo.DependOn; @@ -485,7 +490,12 @@ private List GetSpdxRelationshipsFromSbomFile(Element spdxFileElement, spdxRelationshipLicenseConcludedElement.AddSpdxId(); spdxRelationshipAndLicenseElementsToAddToSBOM.Add(spdxRelationshipLicenseConcludedElement); - // Convert licenseDeclared to SPDX license elements and add Relationship elements for them + // If they exist, convert licenseDeclared to SPDX license elements and add Relationship elements for them + if (fileInfo.LicenseInfoInFiles == null || !fileInfo.LicenseInfoInFiles.Any()) + { + return spdxRelationshipAndLicenseElementsToAddToSBOM; + } + var toRelationships = new List(); foreach (var licenseInfoInOneFile in fileInfo.LicenseInfoInFiles) { diff --git a/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs b/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs index a1794d259..46a7bc5c9 100644 --- a/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs +++ b/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs @@ -8,16 +8,20 @@ using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Hashing; using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Manifest.Configuration; using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Recorder; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Api.Workflows; using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config.Validators; using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Contracts.Entities; +using Microsoft.Sbom.Extensions; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using EntityErrorType = Microsoft.Sbom.Contracts.Enums.ErrorType; +using Generator = Microsoft.Sbom.Parsers.Spdx30SbomParser.Generator; namespace Microsoft.Sbom.Api.Tests; @@ -87,14 +91,16 @@ public async Task When_GenerateSbomAsync_WithNoRecordedErrors_Then_EmptyEntityEr { mockRecorder.Setup(c => c.Errors).Returns(new List()).Verifiable(); mockWorkflow.Setup(c => c.RunAsync()).Returns(Task.FromResult(true)).Verifiable(); + var manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new Generator() }); + manifestGeneratorProvider.Init(); var metadata = new SBOMMetadata() { PackageSupplier = "Contoso" }; - generator = new SbomGenerator(mockWorkflow.Object, mockGeneratorProvider.Object, mockRecorder.Object, new List(), mockSanitizer.Object); - var result = await generator.GenerateSbomAsync("rootPath", "compPath", metadata, runtimeConfiguration: runtimeConfiguration); + generator = new SbomGenerator(mockWorkflow.Object, manifestGeneratorProvider, mockRecorder.Object, new List(), mockSanitizer.Object); + var result = await generator.GenerateSbomAsync(".", "compPath", metadata, runtimeConfiguration: runtimeConfiguration); Assert.AreEqual(0, result.Errors.Count); mockRecorder.Verify(); diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs index 935721afd..8defd003b 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs @@ -64,7 +64,7 @@ public void Setup() } }); relationshipGeneratorMock.CallBase = true; - relationshipsArrayGenerator = new RelationshipsArrayGenerator(relationshipGeneratorMock.Object, new ChannelUtils(), loggerMock.Object, sbomConfigsMock.Object, recorderMock.Object); + relationshipsArrayGenerator = new RelationshipsArrayGenerator(relationshipGeneratorMock.Object, new ChannelUtils(), loggerMock.Object, recorderMock.Object); manifestGeneratorProvider.Init(); metadataBuilder = new MetadataBuilder( mockLogger.Object, @@ -96,7 +96,7 @@ public async Task When_BaseGenerationDataExist_DescribesRelationshipsAreGenerate recorder.RecordRootPackageId(RootPackageId); var results = await relationshipsArrayGenerator.GenerateAsync(); - Assert.AreEqual(0, results.Count); + Assert.AreEqual(0, results.Errors.Count); Assert.AreEqual(1, relationships.Count); var describesRelationships = relationships.Where(r => r.RelationshipType == RelationshipType.DESCRIBES); @@ -116,7 +116,7 @@ public async Task When_SPDXFileGenerationDataExist_DescribedByRelationshipsAreGe recorder.RecordSPDXFileId(FileId1); var results = await relationshipsArrayGenerator.GenerateAsync(); - Assert.AreEqual(0, results.Count); + Assert.AreEqual(0, results.Errors.Count); Assert.AreEqual(2, relationships.Count); var describedByRelationships = relationships.Where(r => r.RelationshipType == RelationshipType.DESCRIBED_BY); @@ -134,7 +134,7 @@ public async Task When_ExternalDocRefGenerationDataExist_PreReqRelationshipsAreG recorder.RecordExternalDocumentReferenceIdAndRootElement(ExternalDocRefId1, RootPackageId); var results = await relationshipsArrayGenerator.GenerateAsync(); - Assert.AreEqual(0, results.Count); + Assert.AreEqual(0, results.Errors.Count); Assert.AreEqual(2, relationships.Count); var preReqForRelationships = relationships.Where(r => r.RelationshipType == RelationshipType.PREREQUISITE_FOR); @@ -153,7 +153,7 @@ public async Task When_PackageGenerationDataExist_DependOnRelationshipsAreGenera recorder.RecordPackageId(PackageId1, RootPackageId); var results = await relationshipsArrayGenerator.GenerateAsync(); - Assert.AreEqual(0, results.Count); + Assert.AreEqual(0, results.Errors.Count); Assert.AreEqual(2, relationships.Count); var dependsOnRelationships = relationships.Where(r => r.RelationshipType == RelationshipType.DEPENDS_ON); @@ -168,7 +168,7 @@ public async Task When_NoGenerationDataExist_NoRelationshipsAreGenerated() { var results = await relationshipsArrayGenerator.GenerateAsync(); - Assert.AreEqual(0, results.Count); + Assert.AreEqual(0, results.Errors.Count); Assert.AreEqual(0, relationships.Count); } } diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs index 352cd1091..a3080453d 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs @@ -46,7 +46,7 @@ using Spectre.Console; using Checksum = Microsoft.Sbom.Contracts.Checksum; using Constants = Microsoft.Sbom.Api.Utils.Constants; -using Generator = Microsoft.Sbom.Parsers.Spdx30SbomParser.Generator; +using Generator30 = Microsoft.Sbom.Parsers.Spdx30SbomParser.Generator; using IComponentDetector = Microsoft.Sbom.Api.Utils.IComponentDetector; using ILogger = Serilog.ILogger; @@ -81,25 +81,44 @@ public void Setup() } [TestMethod] - [DataRow(true, true)] - [DataRow(false, false)] - [DataRow(true, false)] - [DataRow(false, true)] - public async Task ManifestGenerationWorkflowTests_Succeeds(bool deleteExistingManifestDir, bool isDefaultSourceManifestDirPath) + //[DataRow("test", true, true)] + //[DataRow("test", false, false)] + //[DataRow("test", true, false)] + //[DataRow("test", false, true)] + [DataRow("3.0", true, true)] + [DataRow("3.0", false, false)] + [DataRow("3.0", true, false)] + [DataRow("3.0", false, true)] + public async Task ManifestGenerationWorkflowTests_Succeeds(string spdxVersionForGenerator, bool deleteExistingManifestDir, bool isDefaultSourceManifestDirPath) { - var manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new Generator() }); + var manifestInfoPerSpdxVersion = new Dictionary + { + { "test", Constants.TestManifestInfo }, + { "3.0", Constants.SPDX30ManifestInfo } + }; + + ManifestGeneratorProvider manifestGeneratorProvider = null; + if (spdxVersionForGenerator == "test") + { + manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new TestManifestGenerator() }); + } + else if (spdxVersionForGenerator == "3.0") + { + manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new Generator30() }); + } + manifestGeneratorProvider.Init(); var metadataBuilder = new MetadataBuilder( mockLogger.Object, manifestGeneratorProvider, - Constants.SPDX30ManifestInfo, + manifestInfoPerSpdxVersion[spdxVersionForGenerator], recorderMock.Object); var jsonFilePath = "/root/_manifest/manifest.json"; ISbomConfig sbomConfig = new SbomConfig(fileSystemMock.Object) { - ManifestInfo = Constants.SPDX30ManifestInfo, + ManifestInfo = manifestInfoPerSpdxVersion[spdxVersionForGenerator], ManifestJsonDirPath = "/root/_manifest", ManifestJsonFilePath = jsonFilePath, MetadataBuilder = metadataBuilder, @@ -154,11 +173,14 @@ public async Task ManifestGenerationWorkflowTests_Succeeds(bool deleteExistingMa configurationMock.SetupGet(c => c.BuildComponentPath).Returns(new ConfigurationSetting { Value = "/root" }); configurationMock.SetupGet(c => c.FollowSymlinks).Returns(new ConfigurationSetting { Value = true }); - // Added config settings - configurationMock.SetupGet(c => c.PackageName).Returns(new ConfigurationSetting("the-package-name")); - configurationMock.SetupGet(c => c.PackageVersion).Returns(new ConfigurationSetting("the-package-version")); - configurationMock.SetupGet(c => c.NamespaceUriUniquePart).Returns(new ConfigurationSetting("some-custom-value-here")); - configurationMock.SetupGet(c => c.NamespaceUriBase).Returns(new ConfigurationSetting("http://sbom.microsoft")); + // Added config settings necessary for 3.0 SBOM generation + if (spdxVersionForGenerator == "3.0") + { + configurationMock.SetupGet(c => c.PackageName).Returns(new ConfigurationSetting("the-package-name")); + configurationMock.SetupGet(c => c.PackageVersion).Returns(new ConfigurationSetting("the-package-version")); + configurationMock.SetupGet(c => c.NamespaceUriUniquePart).Returns(new ConfigurationSetting("some-custom-value-here")); + configurationMock.SetupGet(c => c.NamespaceUriBase).Returns(new ConfigurationSetting("http://sbom.microsoft")); + } fileSystemMock .Setup(f => f.CreateDirectory( @@ -319,15 +341,16 @@ await externalDocumentReferenceChannel.Writer.WriteAsync(new ExternalDocumentRef { externalDocumentReferenceProvider } }; - var fileArrayGenerator = new FileArrayGenerator(sbomConfigs, sourcesProvider, recorderMock.Object, mockLogger.Object); + var fileArrayGenerator = new FileArrayGenerator(sourcesProvider, recorderMock.Object, mockLogger.Object); - var packageArrayGenerator = new PackageArrayGenerator(mockLogger.Object, sbomConfigs, sourcesProvider, recorderMock.Object); + var packageArrayGenerator = new PackageArrayGenerator(mockLogger.Object, sourcesProvider, recorderMock.Object, sbomConfigs); - var externalDocumentReferenceGenerator = new ExternalDocumentReferenceGenerator(mockLogger.Object, sbomConfigs, sourcesProvider, recorderMock.Object); + var externalDocumentReferenceGenerator = new ExternalDocumentReferenceGenerator(mockLogger.Object, sourcesProvider, recorderMock.Object); + var generateResult = new GenerateResult(new List(), new Dictionary>()); relationshipArrayGenerator .Setup(r => r.GenerateAsync()) - .ReturnsAsync(await Task.FromResult(new List())); + .ReturnsAsync(generateResult); var workflow = new SbomGenerationWorkflow( configurationMock.Object, @@ -346,19 +369,29 @@ await externalDocumentReferenceChannel.Writer.WriteAsync(new ExternalDocumentRef var result = Encoding.UTF8.GetString(manifestStream.ToArray()); var resultJson = JObject.Parse(result); - Assert.AreEqual("1.0.0", resultJson["Version"]); - Assert.AreEqual(12, resultJson["Build"]); - Assert.AreEqual("test", resultJson["Definition"]); + if (spdxVersionForGenerator == "test") + { + Assert.AreEqual("1.0.0", resultJson["Version"]); + Assert.AreEqual(12, resultJson["Build"]); + Assert.AreEqual("test", resultJson["Definition"]); - var outputs = resultJson["Outputs"]; - var sortedOutputs = new JArray(outputs.OrderBy(obj => (string)obj["Source"])); - var expectedSortedOutputs = new JArray(outputs.OrderBy(obj => (string)obj["Source"])); + // TODO: check why outputs is not present by comparing with master + var outputs = resultJson["Outputs"]; + var sortedOutputs = new JArray(outputs.OrderBy(obj => (string)obj["Source"])); + var expectedSortedOutputs = new JArray(outputs.OrderBy(obj => (string)obj["Source"])); + + var packages = resultJson["Packages"]; + Assert.AreEqual(4, packages.Count()); + + Assert.IsTrue(JToken.DeepEquals(sortedOutputs, expectedSortedOutputs)); + } - var packages = resultJson["Packages"]; - Assert.AreEqual(4, packages.Count()); + //else + //{ - Assert.IsTrue(JToken.DeepEquals(sortedOutputs, expectedSortedOutputs)); + //} + // TODO: need to make sure these pass configurationMock.VerifyAll(); fileSystemMock.VerifyAll(); hashCodeGeneratorMock.VerifyAll(); @@ -409,7 +442,8 @@ public async Task ManifestGenerationWorkflowTests_SBOMDir_NotDefault_NotDeleted( fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); fileSystemMock.Setup(f => f.DeleteDir(It.IsAny(), true)).Verifiable(); var fileArrayGeneratorMock = new Mock>(); - fileArrayGeneratorMock.Setup(f => f.GenerateAsync()).ReturnsAsync(new List { new FileValidationResult() }); + var generateResult = new GenerateResult(new List(), new Dictionary>()); + fileArrayGeneratorMock.Setup(f => f.GenerateAsync()).ReturnsAsync(generateResult); var workflow = new SbomGenerationWorkflow( configurationMock.Object, From bbdd7513e637e992abdb3b36521ba71593d9f2ed Mon Sep 17 00:00:00 2001 From: ppandrate Date: Mon, 3 Feb 2025 11:13:10 -0800 Subject: [PATCH 03/15] wokring generator api changes with UT --- .../Executors/GenerateResult.cs | 1 - .../Executors/Spdx2SerializationStrategy.cs | 11 ++++--- .../Executors/Spdx3SerializationStrategy.cs | 14 ++++++--- .../Output/MetadataBuilder.cs | 22 ++++++++++++++ .../ExternalDocumentReferenceGenerator.cs | 4 +-- .../Workflows/Helpers/FileArrayGenerator.cs | 2 -- .../Workflows/Helpers/IJsonArrayGenerator.cs | 2 -- .../Helpers/PackageArrayGenerator.cs | 14 ++++++++- .../Helpers/RelationshipsArrayGenerator.cs | 1 - .../Workflows/SBOMGenerationWorkflow.cs | 1 - .../IManifestGenerator.cs | 13 +++++++++ .../IMetadataBuilder.cs | 5 ++++ .../Generator.cs | 10 +++++++ .../SBOMGeneratorTest.cs | 2 -- .../TestManifestGenerator.cs | 5 ++++ .../ManifestGenerationWorkflowTests.cs | 29 +++++++++---------- 16 files changed, 98 insertions(+), 38 deletions(-) diff --git a/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs b/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs index 8839a2f19..6a7dcd0d6 100644 --- a/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs +++ b/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; using System.Collections.Generic; using System.Text.Json; using Microsoft.Sbom.Api.Entities; diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs index 6a78887bb..d337c9fd3 100644 --- a/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs +++ b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; -using Microsoft.Sbom.Api.Manifest.Configuration; using Microsoft.Sbom.Extensions; namespace Microsoft.Sbom.Api.Workflows.Helpers; @@ -18,6 +17,7 @@ public void AddToFilesSupportingConfig(ref IList elementsSupporting { if (config.MetadataBuilder.TryGetFilesArrayHeaderName(out var headerName)) { + config.JsonSerializer.StartJsonArray(headerName); elementsSupportingConfigs.Add(config); } } @@ -26,14 +26,16 @@ public void AddToPackagesSupportingConfig(ref IList elementsSupport { if (config.MetadataBuilder.TryGetPackageArrayHeaderName(out var headerName)) { + config.JsonSerializer.StartJsonArray(headerName); elementsSupportingConfigs.Add(config); } } public bool AddToRelationshipsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) { - if (config.MetadataBuilder.TryGetRelationshipsHeaderName(out var relationshipArrayHeaderName)) + if (config.MetadataBuilder.TryGetRelationshipsHeaderName(out var headerName)) { + config.JsonSerializer.StartJsonArray(headerName); return true; } @@ -44,6 +46,7 @@ public void AddToExternalDocRefsSupportingConfig(ref IList elements { if (config.MetadataBuilder.TryGetExternalRefArrayHeaderName(out var headerName)) { + config.JsonSerializer.StartJsonArray(headerName); elementsSupportingConfigs.Add(config); } } @@ -81,7 +84,6 @@ public async Task> WriteJsonObjectsToSbomAsync( foreach (var serializer in filesGenerateResult.SerializerToJsonDocuments.Keys) { - serializer.StartJsonArray("files"); foreach (var jsonDocument in filesGenerateResult.SerializerToJsonDocuments[serializer]) { serializer.Write(jsonDocument); @@ -96,7 +98,6 @@ public async Task> WriteJsonObjectsToSbomAsync( foreach (var serializer in packagesGenerateResult.SerializerToJsonDocuments.Keys) { - serializer.StartJsonArray("packages"); foreach (var jsonDocument in packagesGenerateResult.SerializerToJsonDocuments[serializer]) { serializer.Write(jsonDocument); @@ -111,7 +112,6 @@ public async Task> WriteJsonObjectsToSbomAsync( foreach (var serializer in externalDocumentReferenceGenerateResult.SerializerToJsonDocuments.Keys) { - serializer.StartJsonArray("externalDocumentRefs"); foreach (var jsonDocument in externalDocumentReferenceGenerateResult.SerializerToJsonDocuments[serializer]) { serializer.Write(jsonDocument); @@ -126,7 +126,6 @@ public async Task> WriteJsonObjectsToSbomAsync( foreach (var serializer in relationshipGenerateResult.SerializerToJsonDocuments.Keys) { - serializer.StartJsonArray("relationships"); foreach (var jsonDocument in relationshipGenerateResult.SerializerToJsonDocuments[serializer]) { serializer.Write(jsonDocument); diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs index d9e292cf9..c1e2ee5cf 100644 --- a/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs +++ b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs @@ -1,9 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Collections; using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Extensions; @@ -53,20 +51,30 @@ public async Task> WriteJsonObjectsToSbomAsync( relationshipsArrayGenerator.SpdxManifestVersion = spdxManifestVersion; externalDocumentReferenceGenerator.SpdxManifestVersion = spdxManifestVersion; + // Files section var generateResult = await fileArrayGenerator.GenerateAsync(); + WriteElementsToSbom(generateResult); // Packages section var packagesGenerateResult = await packageArrayGenerator.GenerateAsync(); generateResult.Errors.AddRange(packagesGenerateResult.Errors); + WriteElementsToSbom(packagesGenerateResult); // External Document Reference section var externalDocumentReferenceGenerateResult = await externalDocumentReferenceGenerator.GenerateAsync(); generateResult.Errors.AddRange(externalDocumentReferenceGenerateResult.Errors); + WriteElementsToSbom(externalDocumentReferenceGenerateResult); // Relationships section var relationshipGenerateResult = await relationshipsArrayGenerator.GenerateAsync(); generateResult.Errors.AddRange(relationshipGenerateResult.Errors); + WriteElementsToSbom(relationshipGenerateResult); + return generateResult.Errors; + } + + private void WriteElementsToSbom(GenerateResult generateResult) + { // Write the JSON objects to the SBOM // TODO: avoid this for loop // TODO: can add deduplication here @@ -85,7 +93,5 @@ public async Task> WriteJsonObjectsToSbomAsync( serializer.EndJsonArray(); } - - return generateResult.Errors; } } diff --git a/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs b/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs index b6231a60a..1d39a7313 100644 --- a/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs +++ b/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs @@ -117,6 +117,28 @@ public bool TryGetRootPackageJson(IInternalMetadataProvider internalMetadataProv } } + public bool TryGetCreationInfoJson(IInternalMetadataProvider internalMetadataProvider, out GenerationResult generationResult) + { + try + { + generationResult = manifestGenerator + .GenerateJsonDocument(internalMetadataProvider); + + if (generationResult == null) + { + return false; + } + + return true; + } + catch (NotSupportedException) + { + generationResult = null; + logger.Warning("Root package serialization not supported on this SBOM format."); + return false; + } + } + public bool TryGetRelationshipsHeaderName(out string headerName) { try diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs index 4033b4ca1..6e1581938 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs @@ -7,12 +7,10 @@ using System.Text.Json; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; -using Microsoft.Sbom.Api.Manifest.Configuration; using Microsoft.Sbom.Api.Output.Telemetry; using Microsoft.Sbom.Api.Providers; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Extensions; -using Microsoft.Sbom.Extensions.Entities; using Serilog; namespace Microsoft.Sbom.Api.Workflows.Helpers; @@ -60,7 +58,7 @@ public async Task GenerateAsync() // Write the start of the array, if supported. IList externalRefArraySupportingConfigs = new List(); var serializationStrategy = JsonSerializationStrategyFactory.GetStrategy(SpdxManifestVersion); - serializationStrategy.AddToFilesSupportingConfig(ref externalRefArraySupportingConfigs, this.SbomConfig); + serializationStrategy.AddToExternalDocRefsSupportingConfig(ref externalRefArraySupportingConfigs, this.SbomConfig); foreach (var sourcesProvider in sourcesProviders) { diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs index cc4789eab..67e0d34cf 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs @@ -7,12 +7,10 @@ using System.Text.Json; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; -using Microsoft.Sbom.Api.Manifest.Configuration; using Microsoft.Sbom.Api.Output.Telemetry; using Microsoft.Sbom.Api.Providers; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Extensions; -using Microsoft.Sbom.Extensions.Entities; using ILogger = Serilog.ILogger; namespace Microsoft.Sbom.Api.Workflows.Helpers; diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs index 3f63a9619..cb6229059 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs @@ -1,9 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Collections.Generic; using System.Threading.Tasks; -using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Extensions; namespace Microsoft.Sbom.Api.Workflows.Helpers; diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs index 2f0a32ae0..72f69373b 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs @@ -91,7 +91,7 @@ public async Task GenerateAsync() foreach (var sbomConfig in packagesArraySupportingConfigs) { - // Write the root package information to the packages array. + // Write the root package information to SBOM. if (sbomConfig.MetadataBuilder.TryGetRootPackageJson(sbomConfigs, out var generationResult)) { if (!serializersToJsonDocs.ContainsKey(sbomConfig.JsonSerializer)) @@ -100,9 +100,21 @@ public async Task GenerateAsync() } serializersToJsonDocs[sbomConfig.JsonSerializer].Add(generationResult?.Document); + sbomConfig.Recorder.RecordRootPackageId(generationResult?.ResultMetadata?.EntityId); sbomConfig.Recorder.RecordDocumentId(generationResult?.ResultMetadata?.DocumentId); } + + // Write creation info to SBOM. Creation info element is only applicable for SPDX 3.0 and above. + if (sbomConfig.MetadataBuilder.TryGetCreationInfoJson(sbomConfigs, out generationResult)) + { + if (!serializersToJsonDocs.ContainsKey(sbomConfig.JsonSerializer)) + { + serializersToJsonDocs[sbomConfig.JsonSerializer] = new List(); + } + + serializersToJsonDocs[sbomConfig.JsonSerializer].Add(generationResult?.Document); + } } return new GenerateResult(totalErrors, serializersToJsonDocs); diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs index c7924ef90..c6e77a72f 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs @@ -7,7 +7,6 @@ using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Executors; -using Microsoft.Sbom.Api.Manifest.Configuration; using Microsoft.Sbom.Api.Output.Telemetry; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Extensions; diff --git a/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs index 4a45f5703..f3e2a4932 100644 --- a/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs +++ b/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs @@ -16,7 +16,6 @@ using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Extensions; using PowerArgs; -using PowerArgs.Cli; using Serilog; using Constants = Microsoft.Sbom.Api.Utils.Constants; diff --git a/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs b/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs index 2f9ff3448..00ea35cd0 100644 --- a/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs +++ b/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs @@ -80,6 +80,19 @@ public interface IManifestGenerator /// GenerationResult GenerateJsonDocument(ExternalDocumentReferenceInfo externalDocumentReferenceInfo); + /// + /// Generate and return the creationInfo element this SBOM describes. The object can be used + /// to add more information to the final object. + /// This applies to only SPDX 3.0 and above. + /// The JsonDocument implements , the caller of this function + /// has the responsibility to dispose this object, so don't use 'using' or dispose this object + /// in this function. + /// + /// The object provides + /// internal metadata that was generated for this SBOM run. + /// + GenerationResult GenerateJsonDocument(IInternalMetadataProvider internalMetadataProvider); + /// /// Gets an array of hash algorithm names this /// manifest needs to generate for each file. diff --git a/src/Microsoft.Sbom.Extensions/IMetadataBuilder.cs b/src/Microsoft.Sbom.Extensions/IMetadataBuilder.cs index 4a1ed020b..bd53102a9 100644 --- a/src/Microsoft.Sbom.Extensions/IMetadataBuilder.cs +++ b/src/Microsoft.Sbom.Extensions/IMetadataBuilder.cs @@ -41,4 +41,9 @@ public interface IMetadataBuilder /// Gets root package in JSON if supported. /// bool TryGetRootPackageJson(IInternalMetadataProvider internalMetadataProvider, out GenerationResult generationResult); + + /// + /// Gets creation info in JSON if supported. Applies to only SPDX 3.0 and above. + /// + bool TryGetCreationInfoJson(IInternalMetadataProvider internalMetadataProvider, out GenerationResult generationResult); } diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Generator.cs b/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Generator.cs index b93776396..fc25207c5 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Generator.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Generator.cs @@ -352,4 +352,14 @@ private PackageVerificationCode GetPackageVerificationCode(IInternalMetadataProv PackageVerificationCodeExcludedFiles = null // We currently don't ignore any files. }; } + + /// + /// Creation info will not be generated in SPDX 2.2 format. + /// + /// + /// + public GenerationResult GenerateJsonDocument(IInternalMetadataProvider internalMetadataProvider) + { + return null; + } } diff --git a/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs b/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs index 46a7bc5c9..aca9781fc 100644 --- a/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs +++ b/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs @@ -8,9 +8,7 @@ using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Hashing; using Microsoft.Sbom.Api.Manifest; -using Microsoft.Sbom.Api.Manifest.Configuration; using Microsoft.Sbom.Api.Output.Telemetry; -using Microsoft.Sbom.Api.Recorder; using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Api.Workflows; using Microsoft.Sbom.Common; diff --git a/test/Microsoft.Sbom.Api.Tests/TestManifestGenerator.cs b/test/Microsoft.Sbom.Api.Tests/TestManifestGenerator.cs index 280d89a22..ad2e29104 100644 --- a/test/Microsoft.Sbom.Api.Tests/TestManifestGenerator.cs +++ b/test/Microsoft.Sbom.Api.Tests/TestManifestGenerator.cs @@ -110,6 +110,11 @@ public GenerationResult GenerateJsonDocument(ExternalDocumentReferenceInfo exter }; } + public GenerationResult GenerateJsonDocument(IInternalMetadataProvider internalMetadataProvider) + { + return null; + } + [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1313:Parameter names should begin with lower-case letter", Justification = "Discard variable has a _ name")] public GenerationResult GenerateRootPackage(IInternalMetadataProvider _) { diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs index a3080453d..99e289158 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs @@ -37,12 +37,9 @@ using Microsoft.Sbom.Contracts.Enums; using Microsoft.Sbom.Extensions; using Microsoft.Sbom.Extensions.Entities; -using Microsoft.Sbom.Parsers.Spdx22SbomParser; -using Microsoft.Sbom.Parsers.Spdx30SbomParser; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Newtonsoft.Json.Linq; -using Serilog.Events; using Spectre.Console; using Checksum = Microsoft.Sbom.Contracts.Checksum; using Constants = Microsoft.Sbom.Api.Utils.Constants; @@ -81,10 +78,10 @@ public void Setup() } [TestMethod] - //[DataRow("test", true, true)] - //[DataRow("test", false, false)] - //[DataRow("test", true, false)] - //[DataRow("test", false, true)] + [DataRow("test", true, true)] + [DataRow("test", false, false)] + [DataRow("test", true, false)] + [DataRow("test", false, true)] [DataRow("3.0", true, true)] [DataRow("3.0", false, false)] [DataRow("3.0", true, false)] @@ -347,7 +344,7 @@ await externalDocumentReferenceChannel.Writer.WriteAsync(new ExternalDocumentRef var externalDocumentReferenceGenerator = new ExternalDocumentReferenceGenerator(mockLogger.Object, sourcesProvider, recorderMock.Object); - var generateResult = new GenerateResult(new List(), new Dictionary>()); + var generateResult = new GenerateResult(new List(), new Dictionary>()); relationshipArrayGenerator .Setup(r => r.GenerateAsync()) .ReturnsAsync(generateResult); @@ -375,7 +372,6 @@ await externalDocumentReferenceChannel.Writer.WriteAsync(new ExternalDocumentRef Assert.AreEqual(12, resultJson["Build"]); Assert.AreEqual("test", resultJson["Definition"]); - // TODO: check why outputs is not present by comparing with master var outputs = resultJson["Outputs"]; var sortedOutputs = new JArray(outputs.OrderBy(obj => (string)obj["Source"])); var expectedSortedOutputs = new JArray(outputs.OrderBy(obj => (string)obj["Source"])); @@ -385,13 +381,16 @@ await externalDocumentReferenceChannel.Writer.WriteAsync(new ExternalDocumentRef Assert.IsTrue(JToken.DeepEquals(sortedOutputs, expectedSortedOutputs)); } + else + { + var elements = resultJson["@graph"].ToArray(); + var packages = elements.Where(element => element["type"].ToString() == "software_Package").ToList(); + Assert.AreEqual(4, packages.Count); - //else - //{ - - //} + var creationInfo = elements.Where(element => element["type"].ToString() == "CreationInfo").ToList(); + Assert.AreEqual(1, creationInfo.Count); + } - // TODO: need to make sure these pass configurationMock.VerifyAll(); fileSystemMock.VerifyAll(); hashCodeGeneratorMock.VerifyAll(); @@ -442,7 +441,7 @@ public async Task ManifestGenerationWorkflowTests_SBOMDir_NotDefault_NotDeleted( fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); fileSystemMock.Setup(f => f.DeleteDir(It.IsAny(), true)).Verifiable(); var fileArrayGeneratorMock = new Mock>(); - var generateResult = new GenerateResult(new List(), new Dictionary>()); + var generateResult = new GenerateResult(new List(), new Dictionary>()); fileArrayGeneratorMock.Setup(f => f.GenerateAsync()).ReturnsAsync(generateResult); var workflow = new SbomGenerationWorkflow( From 9010c318d9a9cdc12be1f776ab90b256529f40d7 Mon Sep 17 00:00:00 2001 From: ppandrate Date: Thu, 6 Feb 2025 19:57:25 -0800 Subject: [PATCH 04/15] working UT for compliance standard --- docs/sbom-tool-arguments.md | 1 + .../Config/ApiConfigurationBuilder.cs | 4 +- .../Config/Args/ValidationArgs.cs | 7 + src/Microsoft.Sbom.Api/Config/Validator.cs | 3 +- .../ConfigurationProfile.cs | 3 +- .../Executors/ComponentDetectionBaseWalker.cs | 14 +- .../Executors/SBOMFileToFileInfoConverter.cs | 1 + .../Filters/DownloadedRootPathFilter.cs | 41 ++- .../Microsoft.Sbom.Api.csproj | 1 + src/Microsoft.Sbom.Api/Utils/Constants.cs | 15 +- .../Workflows/Helpers/FilesValidator.cs | 112 +++++++ .../SBOMParserBasedValidationWorkflow.cs | 31 ++ .../Config/Configuration.cs | 10 + .../Config/IConfiguration.cs | 5 + .../Config/InputConfiguration.cs | 3 + .../Contracts}/Enums/ComplianceStandard.cs | 4 +- .../ServiceCollectionExtensions.cs | 2 +- .../Constants.cs | 10 + .../Entities/PackageVerificationCode.cs | 3 +- .../Generator.cs | 11 +- .../Parser/ContextsResult.cs | 6 +- .../Parser/ElementsResult.cs | 13 +- .../Parser/SPDX30Parser.cs | 316 ++++++++++++++++-- .../SPDXToSbomFormatConverterExtensions.cs | 56 ++++ .../SbomParserBasedValidationWorkflowTests.cs | 225 +++++++++++++ .../Parser/SbomFileParserTests.cs | 12 +- .../Parser/SbomMetadataParserTests.cs | 6 +- .../Parser/SbomParserTestsBase.cs | 251 +------------- .../Utility/GeneratedSbomValidator.cs | 10 +- 29 files changed, 869 insertions(+), 307 deletions(-) rename src/{Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities => Microsoft.Sbom.Contracts/Contracts}/Enums/ComplianceStandard.cs (83%) create mode 100644 src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs diff --git a/docs/sbom-tool-arguments.md b/docs/sbom-tool-arguments.md index ea5d369f0..6aa8491f7 100644 --- a/docs/sbom-tool-arguments.md +++ b/docs/sbom-tool-arguments.md @@ -41,6 +41,7 @@ Actions TelemetryFilePath (-t) Specify a file where we should write detailed telemetry for the workflow. FollowSymlinks (-F) If set to false, we will not follow symlinks while traversing the build drop folder. Default is set to 'true'. ManifestInfo (-mi) A list of the name and version of the manifest format that we are using. + ComplianceStandard (-cs) The compliance standard to validate against. Generate -options - Generate a SBOM for all the files in the given build drop folder, and the packages in the components path. diff --git a/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs b/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs index 75405b5fa..95dfd2a2b 100644 --- a/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs +++ b/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs @@ -123,9 +123,10 @@ public static InputConfiguration GetConfiguration( throw new ArgumentException($"'{nameof(outputPath)}' cannot be null or whitespace.", nameof(outputPath)); } + // TODO: update to SPDX 3.0 for default. if (specifications is null || specifications.Count == 0) { - specifications = new List() { ApiConstants.SPDX22Specification }; + specifications = ApiConstants.SupportedSbomSpecifications; } var sanitizedRuntimeConfiguration = SanitiseRuntimeConfiguration(runtimeConfiguration); @@ -142,6 +143,7 @@ public static InputConfiguration GetConfiguration( IgnoreMissing = GetConfigurationSetting(ignoreMissing), Parallelism = GetConfigurationSetting(sanitizedRuntimeConfiguration.WorkflowParallelism), ManifestInfo = ConvertSbomSpecificationToManifestInfo(specifications), + ComplianceStandard = GetConfigurationSetting(string.Empty), }; SetVerbosity(sanitizedRuntimeConfiguration, configuration); diff --git a/src/Microsoft.Sbom.Api/Config/Args/ValidationArgs.cs b/src/Microsoft.Sbom.Api/Config/Args/ValidationArgs.cs index cc806fcb0..3e8be3a45 100644 --- a/src/Microsoft.Sbom.Api/Config/Args/ValidationArgs.cs +++ b/src/Microsoft.Sbom.Api/Config/Args/ValidationArgs.cs @@ -81,4 +81,11 @@ public class ValidationArgs : GenerationAndValidationCommonArgs /// [ArgDescription("The Hash algorithm to use while verifying or generating the hash value of a file")] public AlgorithmName HashAlgorithm { get; set; } + + /// + /// The compliance standard to validate against. + /// + [ArgDescription("The compliance standard to validate against")] + [ArgShortcut("cs")] + public ComplianceStandard ComplianceStandard { get; set; } } diff --git a/src/Microsoft.Sbom.Api/Config/Validator.cs b/src/Microsoft.Sbom.Api/Config/Validator.cs index 41ab70cf1..0230a4ea3 100644 --- a/src/Microsoft.Sbom.Api/Config/Validator.cs +++ b/src/Microsoft.Sbom.Api/Config/Validator.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Linq; using System.Threading.Tasks; using Microsoft.Sbom.Api.Config.Args; using Microsoft.Sbom.Api.Exceptions; @@ -35,7 +36,7 @@ public async Task Validate() bool result; try { - if (configuration.ManifestInfo.Value.Contains(Constants.SPDX22ManifestInfo)) + if (configuration.ManifestInfo.Value.Any(Constants.SupportedSpdxManifests.Contains)) { result = await parserValidationWorkflow.RunAsync(); } diff --git a/src/Microsoft.Sbom.Api/ConfigurationProfile.cs b/src/Microsoft.Sbom.Api/ConfigurationProfile.cs index 3c555beba..a5897949b 100644 --- a/src/Microsoft.Sbom.Api/ConfigurationProfile.cs +++ b/src/Microsoft.Sbom.Api/ConfigurationProfile.cs @@ -67,7 +67,8 @@ public ConfigurationProfile() .ForMember(c => c.NamespaceUriUniquePart, o => o.Ignore()) .ForMember(c => c.NamespaceUriBase, o => o.Ignore()) .ForMember(c => c.DeleteManifestDirIfPresent, o => o.Ignore()) - .ForMember(c => c.PackageSupplier, o => o.Ignore()); + .ForMember(c => c.PackageSupplier, o => o.Ignore()) + .ForMember(c => c.ComplianceStandard, o => o.Ignore()); // Create config for the generation args, ignoring other action members CreateMap() diff --git a/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs b/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs index e5c1b9e5b..3d4a40a65 100644 --- a/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs +++ b/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs @@ -79,13 +79,17 @@ public ComponentDetectionBaseWalker( cliArgumentBuilder.AddDetectorArg("SPDX22SBOM", "EnableIfDefaultOff"); cliArgumentBuilder.AddDetectorArg("ConanLock", "EnableIfDefaultOff"); - if (sbomConfigs.TryGet(Constants.SPDX22ManifestInfo, out var spdxSbomConfig)) + // Iterate over all supported SPDX manifests and apply the necessary logic + foreach (var supportedSpdxManifest in Constants.SupportedSpdxManifests) { - var directory = Path.GetDirectoryName(spdxSbomConfig.ManifestJsonFilePath); - directory = fileSystemUtils.GetFullPath(directory); - if (!string.IsNullOrEmpty(directory)) + if (sbomConfigs.TryGet(supportedSpdxManifest, out var spdxSbomConfig)) { - cliArgumentBuilder.AddArg("DirectoryExclusionList", directory); + var directory = Path.GetDirectoryName(spdxSbomConfig.ManifestJsonFilePath); + directory = fileSystemUtils.GetFullPath(directory); + if (!string.IsNullOrEmpty(directory)) + { + cliArgumentBuilder.AddArg("DirectoryExclusionList", directory); + } } } diff --git a/src/Microsoft.Sbom.Api/Executors/SBOMFileToFileInfoConverter.cs b/src/Microsoft.Sbom.Api/Executors/SBOMFileToFileInfoConverter.cs index 103a179dc..4de6bf9ac 100644 --- a/src/Microsoft.Sbom.Api/Executors/SBOMFileToFileInfoConverter.cs +++ b/src/Microsoft.Sbom.Api/Executors/SBOMFileToFileInfoConverter.cs @@ -40,6 +40,7 @@ public SbomFileToFileInfoConverter(IFileTypeUtils fileTypeUtils) { await foreach (var component in componentReader.ReadAllAsync()) { + Console.WriteLine($"Processing {component.Path}"); await Convert(component, output, errors, fileLocation); } diff --git a/src/Microsoft.Sbom.Api/Filters/DownloadedRootPathFilter.cs b/src/Microsoft.Sbom.Api/Filters/DownloadedRootPathFilter.cs index 6ab1526ed..174b269de 100644 --- a/src/Microsoft.Sbom.Api/Filters/DownloadedRootPathFilter.cs +++ b/src/Microsoft.Sbom.Api/Filters/DownloadedRootPathFilter.cs @@ -4,7 +4,7 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; +//using System.Linq; using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config; using Serilog; @@ -69,9 +69,33 @@ public bool IsValid(string filePath) return isValid; } - /// - /// Initializes the root path filters list. - /// + ///// + ///// Initializes the root path filters list. + ///// + //public void Init() + //{ + // logger.Verbose("Adding root path filter valid paths"); + // skipValidation = true; + + // if (configuration.RootPathFilter != null && !string.IsNullOrWhiteSpace(configuration.RootPathFilter.Value)) + // { + // skipValidation = false; + // validPaths = new HashSet(); + // var relativeRootPaths = configuration.RootPathFilter.Value.Split(';'); + + // var r = relativeRootPaths.Select(r => + // new FileInfo(fileSystemUtils.JoinPaths(configuration.BuildDropPath.Value, r)) + // .FullName); + + // validPaths.UnionWith(r); + + // foreach (var validPath in validPaths) + // { + // logger.Verbose($"Added valid path {validPath}"); + // } + // } + //} + public void Init() { logger.Verbose("Adding root path filter valid paths"); @@ -83,9 +107,12 @@ public void Init() validPaths = new HashSet(); var relativeRootPaths = configuration.RootPathFilter.Value.Split(';'); - validPaths.UnionWith(relativeRootPaths.Select(r => - new FileInfo(fileSystemUtils.JoinPaths(configuration.BuildDropPath.Value, r)) - .FullName)); + foreach (var relativePath in relativeRootPaths) + { + var path = fileSystemUtils.JoinPaths(configuration.BuildDropPath.Value, relativePath); + var fullPath = new FileInfo(path).FullName; + validPaths.Add(fullPath); + } foreach (var validPath in validPaths) { diff --git a/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj b/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj index ef27b85a3..82acc4db0 100644 --- a/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj +++ b/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj @@ -53,6 +53,7 @@ + diff --git a/src/Microsoft.Sbom.Api/Utils/Constants.cs b/src/Microsoft.Sbom.Api/Utils/Constants.cs index ce99fc81f..7b86f3102 100644 --- a/src/Microsoft.Sbom.Api/Utils/Constants.cs +++ b/src/Microsoft.Sbom.Api/Utils/Constants.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; +using System.Collections.ObjectModel; using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Contracts.Enums; using Microsoft.Sbom.Extensions.Entities; @@ -25,7 +26,19 @@ public static class Constants Version = "3.0" }; - public static SbomSpecification SPDX22Specification = SPDX22ManifestInfo.ToSBOMSpecification(); + public static Collection SupportedSpdxManifests = new() + { + SPDX22ManifestInfo, + SPDX30ManifestInfo, + }; + + //public static SbomSpecification SPDX22Specification = SPDX22ManifestInfo.ToSBOMSpecification(); + + public static Collection SupportedSbomSpecifications = new() + { + SPDX22ManifestInfo.ToSBOMSpecification(), + SPDX30ManifestInfo.ToSBOMSpecification(), + }; // TODO: move to test csproj public static ManifestInfo TestManifestInfo = new ManifestInfo diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/FilesValidator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/FilesValidator.cs index d8eee4d8e..1d6f76b83 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/FilesValidator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/FilesValidator.cs @@ -12,6 +12,7 @@ using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Entities; using Microsoft.Sbom.Parsers.Spdx22SbomParser.Entities; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; using Microsoft.Sbom.Utils; using Serilog; @@ -127,6 +128,81 @@ public FilesValidator( return (successCount, failures.Values.ToList()); } + public async Task<(int, List)> Validate(IEnumerable files) + { + var errors = new List>(); + var results = new List>(); + var failures = new Dictionary(); + + var (onDiskFileResults, onDiskFileErrors) = GetOnDiskFiles(); + results.AddRange(onDiskFileResults); + errors.AddRange(onDiskFileErrors); + + var workflowErrors = channelUtils.Merge(errors.ToArray()); + await foreach (var error in workflowErrors.ReadAllAsync()) + { + failures.Add(error.Path, error); + } + + var (inSbomFileResults, inSbomFileErrors) = GetInsideSbomFiles(files); + results.AddRange(inSbomFileResults); + errors.AddRange(inSbomFileErrors); + + var successCount = 0; + var resultChannel = channelUtils.Merge(results.ToArray()); + await foreach (var validationResult in resultChannel.ReadAllAsync()) + { + successCount++; + } + + workflowErrors = channelUtils.Merge(errors.ToArray()); + + await foreach (var error in workflowErrors.ReadAllAsync()) + { + failures.Add(error.Path, error); + } + + foreach (var file in fileHashesDictionary.FileHashes) + { + if (failures.ContainsKey(file.Key)) + { + // If we have added a validation error for this file, we don't need to add another one. + continue; + } + + if (file.Value == null) + { + // This generally means that we have case variations in the file names. + failures.Add(file.Key, new FileValidationResult + { + ErrorType = ErrorType.AdditionalFile, + Path = file.Key, + }); + continue; + } + + switch (file.Value.FileLocation) + { + case FileLocation.OnDisk: + failures.Add(file.Key, new FileValidationResult + { + ErrorType = ErrorType.AdditionalFile, + Path = file.Key, + }); + break; + case FileLocation.InSbomFile: + failures.Add(file.Key, new FileValidationResult + { + ErrorType = ErrorType.MissingFile, + Path = file.Key, + }); + break; + } + } + + return (successCount, failures.Values.ToList()); + } + private (List>, List>) GetOnDiskFiles() { var errors = new List>(); @@ -190,4 +266,40 @@ public FilesValidator( return (filesWithHashes, errors); } + + private (List>, List>) GetInsideSbomFiles(IEnumerable files) + { + var errors = new List>(); + var filesWithHashes = new List>(); + + // Enumerate files from SBOM + + var file = files.FirstOrDefault().ToSbomFile(); + Console.WriteLine(file.Path); + + var (sbomFiles, sbomFileErrors) = enumeratorChannel.Enumerate(() => files.Select(f => f.ToSbomFile())); + errors.Add(sbomFileErrors); + + log.Debug($"Splitting the workflow into {configuration.Parallelism.Value} threads."); + var splitFilesChannels = channelUtils.Split(sbomFiles, configuration.Parallelism.Value); + + log.Debug("Waiting for the workflow to finish..."); + foreach (var fileChannel in splitFilesChannels) + { + // Convert files to internal SBOM format. + var (internalSbomFiles, converterErrors) = fileConverter.Convert(fileChannel, FileLocation.InSbomFile); + errors.Add(converterErrors); + + // Filter files. + var (filteredSbomFiles, filterErrors) = spdxFileFilterer.Filter(internalSbomFiles); + //errors.Add(filterErrors); + + var (validationResults, validationErrors) = hashValidator.Validate(filteredSbomFiles); + //errors.Add(validationErrors); + + filesWithHashes.Add(validationResults); + } + + return (filesWithHashes, errors); + } } diff --git a/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs index 7c6dc23d1..17698afad 100644 --- a/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs +++ b/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs @@ -97,9 +97,25 @@ public async Task RunAsync() } } + // Validate compliance standard for SPDX 3.0 parsers and above + if (!string.IsNullOrEmpty(configuration.ComplianceStandard?.Value) && Convert.ToDouble(sbomConfig.ManifestInfo.Version) >= 3.0) + { + var complianceStandard = configuration.ComplianceStandard.Value; + try + { + ((SPDX30Parser)sbomParser).RequiredComplianceStandard = complianceStandard; + } + catch (Exception e) + { + recorder.RecordException(e); + log.Error($"Unable to use the given compliance standard {complianceStandard} to parse the SBOM."); + } + } + var successfullyValidatedFiles = 0; List fileValidationFailures = null; + // This logic is the same as SbomParserTestsBase, however since that logic is part of the test suite we replicate it here ParserStateResult? result = null; do { @@ -126,6 +142,21 @@ public async Task RunAsync() break; case ExternalDocumentReferencesResult externalRefResult: externalRefResult.References.ToList(); + break; + case ContextsResult contextsResult: + contextsResult.Contexts.ToList(); + break; + case ElementsResult elementsResult: + elementsResult.Elements.ToList(); + totalNumberOfPackages = elementsResult.PackagesCount; + + (successfullyValidatedFiles, fileValidationFailures) = await filesValidator.Validate(elementsResult.Files); + invalidInputFiles = fileValidationFailures.Where(f => f.ErrorType == ErrorType.InvalidInputFile).ToList(); + if (invalidInputFiles.Count != 0) + { + throw new InvalidDataException($"Your manifest file is malformed. {invalidInputFiles.First().Path}"); + } + break; default: break; diff --git a/src/Microsoft.Sbom.Common/Config/Configuration.cs b/src/Microsoft.Sbom.Common/Config/Configuration.cs index aa5789bef..1234b2e4f 100644 --- a/src/Microsoft.Sbom.Common/Config/Configuration.cs +++ b/src/Microsoft.Sbom.Common/Config/Configuration.cs @@ -54,6 +54,7 @@ public class Configuration : IConfiguration private static readonly AsyncLocal> verbosity = new(); private static readonly AsyncLocal> sbomPath = new(); private static readonly AsyncLocal> sbomDir = new(); + private static readonly AsyncLocal> complianceStandard = new(); /// [DirectoryExists] @@ -339,4 +340,13 @@ public ConfigurationSetting SbomDir get => sbomDir.Value; set => sbomDir.Value = value; } + + /// + public ConfigurationSetting ComplianceStandard + { + get => complianceStandard.Value; + set => complianceStandard.Value = value; + } + + ConfigurationSetting IConfiguration.ComplianceStandard { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } } diff --git a/src/Microsoft.Sbom.Common/Config/IConfiguration.cs b/src/Microsoft.Sbom.Common/Config/IConfiguration.cs index cd632cb7a..656091077 100644 --- a/src/Microsoft.Sbom.Common/Config/IConfiguration.cs +++ b/src/Microsoft.Sbom.Common/Config/IConfiguration.cs @@ -214,4 +214,9 @@ public interface IConfiguration /// Gets or sets the directory containing the sbom(s) to redact. /// ConfigurationSetting SbomDir { get; set; } + + /// + /// Gets or sets the directory containing the sbom(s) to redact. + /// + ConfigurationSetting ComplianceStandard { get; set; } } diff --git a/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs b/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs index 151d4ca02..45a25490b 100644 --- a/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs +++ b/src/Microsoft.Sbom.Common/Config/InputConfiguration.cs @@ -156,4 +156,7 @@ public class InputConfiguration : IConfiguration /// [Path] public ConfigurationSetting SbomPath { get; set; } + + /// + public ConfigurationSetting ComplianceStandard { get; set; } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/ComplianceStandard.cs b/src/Microsoft.Sbom.Contracts/Contracts/Enums/ComplianceStandard.cs similarity index 83% rename from src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/ComplianceStandard.cs rename to src/Microsoft.Sbom.Contracts/Contracts/Enums/ComplianceStandard.cs index 82684cbd7..6eb607bda 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Enums/ComplianceStandard.cs +++ b/src/Microsoft.Sbom.Contracts/Contracts/Enums/ComplianceStandard.cs @@ -3,7 +3,7 @@ using System.Text.Json.Serialization; -namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums; +namespace Microsoft.Sbom.Contracts.Enums; /// /// Defines the different supported compliance standards. @@ -12,5 +12,5 @@ namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums; public enum ComplianceStandard { NTIA, - None, + None } diff --git a/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs b/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs index fb6df7293..12d7988e7 100644 --- a/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs +++ b/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs @@ -162,7 +162,7 @@ public static IServiceCollection AddSbomTool(this IServiceCollection services, L var manifestData = new ManifestData(); - if (!configuration.ManifestInfo.Value.Contains(Constants.SPDX22ManifestInfo)) + if (!configuration.ManifestInfo.Value.Any(manifestInfo => Constants.SupportedSpdxManifests.Contains(manifestInfo))) { var sbomConfig = sbomConfigs.Get(configuration.ManifestInfo?.Value?.FirstOrDefault()); var parserProvider = x.GetRequiredService(); diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs index eb164eb88..1558ed0aa 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs @@ -2,7 +2,9 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; +using Microsoft.Sbom.Contracts.Enums; using Microsoft.Sbom.Extensions.Entities; +using HashAlgorithm = Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums.HashAlgorithm; namespace Microsoft.Sbom.Parsers.Spdx30SbomParser; @@ -47,4 +49,12 @@ internal static class Constants Name = SPDXName, Version = SPDXVersion }; + + public static readonly Dictionary AlgorithmMap = new() + { + { AlgorithmName.SHA1, HashAlgorithm.sha1 }, + { AlgorithmName.SHA256, HashAlgorithm.sha256 }, + { AlgorithmName.SHA512, HashAlgorithm.sha512 }, + { AlgorithmName.MD5, HashAlgorithm.md5 } + }; } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/PackageVerificationCode.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/PackageVerificationCode.cs index f5f520685..2aa50ab7c 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/PackageVerificationCode.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/PackageVerificationCode.cs @@ -3,7 +3,7 @@ using System.Collections.Generic; using System.Text.Json.Serialization; -using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums; +using HashAlgorithm = Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums.HashAlgorithm; namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; @@ -16,6 +16,7 @@ public class PackageVerificationCode : Element /// /// Gets or sets the algorithm being used to calculate the type of verification. /// + [JsonConverter(typeof(JsonStringEnumConverter))] [JsonRequired] [JsonPropertyName("algorithm")] public HashAlgorithm Algorithm { get; set; } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs index 1c8603a57..c9d6caa6b 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs @@ -15,6 +15,7 @@ using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Exceptions; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Utils; +using HashAlgorithm = Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums.HashAlgorithm; using RelationshipType = Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums.RelationshipType; using SbomEntities = Microsoft.Sbom.Extensions.Entities; using SHA1 = System.Security.Cryptography.SHA1; @@ -27,14 +28,6 @@ namespace Microsoft.Sbom.Parsers.Spdx30SbomParser; /// public class Generator : IManifestGenerator { - private static readonly Dictionary AlgorithmMap = new() - { - { AlgorithmName.SHA1, HashAlgorithm.sha1 }, - { AlgorithmName.SHA256, HashAlgorithm.sha256 }, - { AlgorithmName.SHA512, HashAlgorithm.sha512 }, - { AlgorithmName.MD5, HashAlgorithm.md5 } - }; - public AlgorithmName[] RequiredHashAlgorithms => new[] { AlgorithmName.SHA256, AlgorithmName.SHA1 }; public string Version { get; set; } = string.Join("-", Constants.SPDXName, Constants.SPDXVersion); @@ -445,7 +438,7 @@ private List ConvertSbomFileToSpdxFileAndRelationships(InternalSbomFile { var packageVerificationCode = new PackageVerificationCode { - Algorithm = AlgorithmMap.GetValueOrDefault(checksum.Algorithm), + Algorithm = Constants.AlgorithmMap.GetValueOrDefault(checksum.Algorithm), HashValue = checksum.ChecksumValue.ToLowerInvariant(), }; packageVerificationCode.AddSpdxId(); diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs index a01282409..0e79b51db 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs @@ -2,17 +2,17 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; -using System.Linq; using Microsoft.Sbom.JsonAsynchronousNodeKit; namespace Microsoft.Sbom.Parser; public record ContextsResult : ParserStateResult { - public ContextsResult(ParserStateResult result) + public ContextsResult(ParserStateResult result, List jsonList) : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) { + Contexts = jsonList; } - public IEnumerable Contexts => ((IEnumerable)this.Result!).Select(r => r); + public IEnumerable Contexts { get; set; } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ElementsResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ElementsResult.cs index e9df538ce..26dbdf842 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ElementsResult.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ElementsResult.cs @@ -2,7 +2,6 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; -using System.Linq; using Microsoft.Sbom.JsonAsynchronousNodeKit; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; @@ -15,5 +14,15 @@ public ElementsResult(ParserStateResult result) { } - public IEnumerable Elements => ((IEnumerable)this.Result!).Select(r => (Element)r); + public IEnumerable Elements { get; set; } + + public IEnumerable Files { get; set; } + + public int FilesCount { get; set; } + + public int PackagesCount { get; set; } + + public int ReferencesCount { get; set; } + + public int RelationshipsCount { get; set; } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs index 6992513c0..87868b877 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs @@ -4,12 +4,16 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text.Json; using System.Text.Json.Nodes; +using System.Text.Json.Serialization; using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; using Microsoft.Sbom.Extensions.Entities; using Microsoft.Sbom.JsonAsynchronousNodeKit; using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums; using SPDXConstants = Microsoft.Sbom.Parsers.Spdx30SbomParser.Constants; @@ -33,7 +37,7 @@ public class SPDX30Parser : ISbomParser GraphProperty, }; - public ComplianceStandard? RequiredComplianceStandard; + public string? RequiredComplianceStandard; public IReadOnlyCollection? EntitiesToEnforceComplianceStandardsFor; public SpdxMetadata Metadata = new SpdxMetadata(); private readonly LargeJsonParser parser; @@ -62,7 +66,7 @@ public SPDX30Parser( { this.jsonSerializerOptions = jsonSerializerOptions ?? new JsonSerializerOptions { - Converters = { new ElementSerializer() }, + Converters = { new ElementSerializer(), new JsonStringEnumConverter() }, }; var handlers = new Dictionary @@ -71,7 +75,7 @@ public SPDX30Parser( { GraphProperty, new PropertyHandler(ParameterType.Array) }, }; - if (!string.IsNullOrEmpty(requiredComplianceStandard)) + if (!string.IsNullOrEmpty(this.RequiredComplianceStandard)) { if (!Enum.TryParse(requiredComplianceStandard, true, out var complianceStandardAsEnum)) { @@ -83,7 +87,6 @@ public SPDX30Parser( { case ComplianceStandard.NTIA: this.EntitiesToEnforceComplianceStandardsFor = this.entitiesWithDifferentNTIARequirements; - this.RequiredComplianceStandard = complianceStandardAsEnum; break; default: throw new ParserException($"{requiredComplianceStandard} compliance standard is not supported."); @@ -114,32 +117,36 @@ public SPDX30Parser( /// public ParserStateResult? Next() { - ParserStateResult? result; + ParserStateResult? result = null; do { - result = this.parser.Next(); - if (result is not null) + result = parser.Next(); + + if (result is not null && result.Result is not null) { - this.observedFieldNames.Add(result.FieldName); - if (result.Result is not null) + var fieldName = result.FieldName; + this.observedFieldNames.Add(fieldName); + var jsonElements = result.Result as IEnumerable; + var jsonList = jsonElements?.ToList(); + switch (fieldName) { - switch (result.FieldName) - { - case ContextProperty: - result = new ContextsResult(result); - break; - case GraphProperty: - result = new ElementsResult(result); - break; - default: - throw new InvalidDataException($"Explicit field {result.FieldName} is unhandled."); - } - - return result; + case ContextProperty: + var contextResult = new ContextsResult(result, jsonList); + ValidateContext(contextResult); + result = contextResult; + break; + case GraphProperty: + var elementsResult = ConvertToElements(jsonList, ref result, this.RequiredComplianceStandard, this.EntitiesToEnforceComplianceStandardsFor); + this.Metadata = this.SetMetadata(elementsResult); + result = elementsResult; + break; + default: + throw new InvalidDataException($"Explicit field {result.FieldName} is unhandled."); } + + return result; } - } - while (result is not null); + } while (result is not null); if (this.requiredFieldsCheck) { @@ -157,6 +164,267 @@ public SPDX30Parser( return null; } + private void ValidateContext(ContextsResult result) + { + if (result.Contexts == null || !result.Contexts.Any() || result.Contexts.Count() > 1) + { + throw new ParserException($"The context property is either empty or has more than one string."); + } + } + + /// + /// Converts JSON objects to SPDX elements. + /// + /// + /// + public ElementsResult ConvertToElements(List? jsonList, ref ParserStateResult? result, string? requiredComplianceStandard, IReadOnlyCollection? entitiesWithDifferentNTIARequirements) + { + var elementsResult = new ElementsResult(result); + var elementsList = new List(); + var filesList = new List(); + + if (jsonList is null) + { + elementsResult.FilesCount = 0; + elementsResult.PackagesCount = 0; + elementsResult.ReferencesCount = 0; + elementsResult.RelationshipsCount = 0; + elementsResult.Elements = elementsList; + return elementsResult; + } + + // Parse requiredComplianceStandard into ComplianceStandard enum + var complianceStandardEnum = ComplianceStandard.None; + if (!string.IsNullOrEmpty(requiredComplianceStandard) && !Enum.TryParse(requiredComplianceStandard, true, out complianceStandardEnum)) + { + throw new ParserException($"Unrecognized compliance standard: \"{requiredComplianceStandard}\""); + } + + foreach (JsonObject jsonObject in jsonList) + { + if (jsonObject == null || !jsonObject.Any()) + { + continue; + } + + var entityType = GetEntityType(jsonObject, complianceStandardEnum, entitiesWithDifferentNTIARequirements); + + object? deserializedElement = null; + try + { + deserializedElement = JsonSerializer.Deserialize(jsonObject.ToString(), entityType, jsonSerializerOptions); + } + catch (Exception e) + { + throw new ParserException(e.Message); + } + + if (deserializedElement != null) + { + elementsList.Add((Element)deserializedElement); + + switch (entityType?.Name) + { + case string name when name.Contains("File"): + filesList.Add((Parsers.Spdx30SbomParser.Entities.File)deserializedElement); + elementsResult.FilesCount += 1; + break; + case string name when name.Contains("Package"): + elementsResult.PackagesCount += 1; + break; + case string name when name.Contains("ExternalMap"): + elementsResult.ReferencesCount += 1; + break; + case string name when name.Contains("Relationship"): + elementsResult.RelationshipsCount += 1; + break; + default: + Console.WriteLine($"Unrecognized entity type: {entityType?.Name}"); + break; + } + } + } + + // Validate if elements meet required compliance standards + switch (complianceStandardEnum) + { + case ComplianceStandard.NTIA: + ValidateNTIARequirements(elementsList); + break; + } + + elementsResult.Elements = elementsList; + elementsResult.Files = filesList; + + return elementsResult; + } + + public Type GetEntityType(JsonObject jsonObject, ComplianceStandard? requiredComplianceStandard, IReadOnlyCollection? entitiesWithDifferentNTIARequirements) + { + var assembly = typeof(Element).Assembly; + var typeFromSbom = jsonObject["type"]?.ToString(); + var entityType = typeFromSbom; + + // For these special cases, remove the prefix from the type. + switch (typeFromSbom) + { + case "software_File": + entityType = "File"; + break; + case "software_Package": + entityType = "Package"; + break; + } + + // If the entity type is in the list of entities that require different NTIA requirements, then add the NTIA prefix. + if (entitiesWithDifferentNTIARequirements?.Contains(entityType) == true) + { + switch (requiredComplianceStandard) + { + case ComplianceStandard.NTIA: + entityType = "NTIA" + entityType; + break; + } + } + + var type = assembly.GetType($"Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.{entityType}") ?? throw new ParserException($"Type \"{typeFromSbom} on {jsonObject} is invalid."); + + return type; + } + + public void ValidateNTIARequirements(List elementsList) + { + ValidateSbomDocCreationForNTIA(elementsList); + ValidateSbomFilesForNTIA(elementsList); + ValidateSbomPackagesForNTIA(elementsList); + } + + /// + /// Validate that information about the SBOM document is present. + /// + /// + /// + public void ValidateSbomDocCreationForNTIA(List elementsList) + { + var spdxDocumentElements = elementsList.Where(element => element is SpdxDocument); + if (spdxDocumentElements.Count() != 1) + { + throw new ParserException("SBOM document is not NTIA compliant because it must only contain one SpdxDocument element."); + } + + var spdxDocumentElement = spdxDocumentElements.First(); + + var spdxCreationInfoElement = (CreationInfo?)elementsList. + Where(element => element.Type == nameof(CreationInfo)). + FirstOrDefault(element => ((CreationInfo)element).Id == spdxDocumentElement.CreationInfoDetails) + ?? throw new ParserException($"SBOM document is not NTIA compliant because it must have a creationInfo element with ID of {spdxDocumentElement.CreationInfoDetails}"); + } + + /// + /// Validate that all files have declared and concluded licenses. + /// + /// + /// + public void ValidateSbomFilesForNTIA(List elementsList) + { + var fileElements = elementsList.Where(element => element is NTIAFile); + foreach (var fileElement in fileElements) + { + var fileSpdxId = fileElement.SpdxId; + + var fileHasSha256Hash = fileElement.VerifiedUsing. + Any(packageVerificationCode => packageVerificationCode.Algorithm == + HashAlgorithm.sha256); + + if (!fileHasSha256Hash) + { + throw new ParserException($"SBOM document is not NTIA compliant because file with SPDX ID {fileSpdxId} does not have a SHA256 hash."); + } + } + } + + /// + /// Validate that all packages have declared and concluded licenses. + /// + /// + /// + public void ValidateSbomPackagesForNTIA(List elementsList) + { + var packageElements = elementsList.Where(element => element is Package); + foreach (var packageElement in packageElements) + { + var packageSpdxId = packageElement.SpdxId; + + var packageHasSha256Hash = packageElement.VerifiedUsing. + Any(packageVerificationCode => packageVerificationCode.Algorithm == + HashAlgorithm.sha256); + + if (!packageHasSha256Hash) + { + throw new ParserException($"SBOM document is not NTIA compliant because package with SPDX ID {packageSpdxId} does not have a SHA256 hash."); + } + } + } + + /// + /// Sets metadata based on parsed SBOM elements. + /// + /// + public SpdxMetadata SetMetadata(ElementsResult result) + { + var metadata = new SpdxMetadata(); + var spdxDocumentElement = (SpdxDocument?)result.Elements.FirstOrDefault(element => element.Type == "SpdxDocument"); + + if (spdxDocumentElement == null) + { + return metadata; + } + + if (spdxDocumentElement.NamespaceMap != null && spdxDocumentElement.NamespaceMap.TryGetValue("sbom", out var namespaceUri)) + { + metadata.DocumentNamespace = new Uri(namespaceUri); + } + + metadata.Name = spdxDocumentElement.Name; + metadata.SpdxId = spdxDocumentElement.SpdxId; + metadata.DocumentDescribes = spdxDocumentElement.RootElement; + + var dataLicenseSpdxId = spdxDocumentElement.DataLicense; + var spdxDataLicenseElement = result.Elements. + FirstOrDefault(element => element.SpdxId == dataLicenseSpdxId) as AnyLicenseInfo; + metadata.DataLicense = spdxDataLicenseElement?.Name; + + var spdxCreationInfoElement = (CreationInfo?)result.Elements. + Where(element => element.Type == nameof(CreationInfo)). + FirstOrDefault(element => ((CreationInfo)element).Id == spdxDocumentElement.CreationInfoDetails); + + var creators = spdxCreationInfoElement?.CreatedBy + .Select(createdBy => result.Elements. + FirstOrDefault(element => element.SpdxId == createdBy) as Organization) + .Where(spdxOrganizationElement => spdxOrganizationElement != null) + .Select(spdxOrganizationElement => spdxOrganizationElement?.Name) + .ToList() ?? []; + + creators.AddRange(spdxCreationInfoElement?.CreatedUsing + .Select(createdBy => result.Elements. + FirstOrDefault(element => element.SpdxId == createdBy) as Tool) + .Where(spdxToolElement => spdxToolElement != null) + .Select(spdxToolElement => spdxToolElement?.Name) ?? []); + + var createdDate = DateTime.MinValue; + DateTime.TryParse(spdxCreationInfoElement?.Created, out createdDate); + + metadata.CreationInfo = new MetadataCreationInfo + { + Created = createdDate, + Creators = creators, + }; + + metadata.SpdxVersion = spdxCreationInfoElement?.SpecVersion; + + return metadata; + } + public SpdxMetadata GetMetadata() { if (!this.parsingComplete) diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs new file mode 100644 index 000000000..0d0f8e3ce --- /dev/null +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Linq; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; +using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; +using SbomChecksum = Microsoft.Sbom.Contracts.Checksum; + +namespace Microsoft.Sbom.Utils; + +/// +/// Provides extension methods to convert a SPDX object to +/// the equivalent internal object as defined in Sbom.Contracts. +/// +public static class SPDXToSbomFormatConverterExtensions +{ + /// + /// Converts a object to a object. + /// + /// + /// + public static SbomFile ToSbomFile(this File spdxFile) + { + var checksums = spdxFile.VerifiedUsing?.Select(c => c.ToSbomChecksum()); + + if (checksums == null || !checksums.Any() || checksums.All(c => c.Algorithm != AlgorithmName.SHA256)) + { + throw new ParserException("File hash is missing a SHA256 value"); + } + + // Not setting LicenseConcluded and LicenseInfoInFiles since the whole SBOM is required to set these values. + return new SbomFile + { + Checksum = checksums, + FileCopyrightText = spdxFile.CopyrightText, + Id = spdxFile.SpdxId, + Path = spdxFile.Name + }; + } + + /// + /// Convert a object to a object. + /// + /// + /// + internal static SbomChecksum ToSbomChecksum(this PackageVerificationCode spdxPackageVerificationCode) + { + return new SbomChecksum + { + Algorithm = AlgorithmName.FromString(spdxPackageVerificationCode.Algorithm.ToString()), + ChecksumValue = spdxPackageVerificationCode.HashValue, + }; + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs index 14604f39c..655125ba8 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text; using System.Threading.Tasks; using Microsoft.Sbom.Api.Convertors; using Microsoft.Sbom.Api.Entities.Output; @@ -51,6 +52,90 @@ public class SbomParserBasedValidationWorkflowTests : ValidationWorkflowTestsBas private readonly Mock signValidatorMock = new(); private readonly Mock signValidationProviderMock = new(); + [System.Diagnostics.CodeAnalysis.SuppressMessage("Style", "JSON002:Probable JSON string detected", Justification = "Need to use JSON string")] + private readonly string sbomMetadataJsonString = + @" + { + ""@context"": [ + ""https://spdx.org/rdf/3.0.1/spdx-context.jsonld"" + ], + ""@graph"": [ + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft"", + ""spdxId"": ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"", + ""type"": ""Organization"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""Microsoft.SBOMTool-3.0.2-preview.0.41"", + ""spdxId"": ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"", + ""type"": ""Tool"" + }, + { + ""@id"": ""_:creationinfo"", + ""created"": ""2023-05-11T00:24:54Z"", + ""createdBy"": [ + ""SPDXRef-Organization-4B8D792FFFFCD3AF92D53A739B6DF98DF2B1F367C2745DDC0085B30F51EBBC81"" + ], + ""createdUsing"": [ + ""SPDXRef-Tool-F3816A2B734CA08686741B17A8BC9020B8513FCE6A7BD33B1006102E2A1B55AA"" + ], + ""specVersion"": ""3.0"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-CreationInfo-0799B4D592549CF6159C30BA3E278BF063A6A241B8728C18E7AEC18BFC2CFF6F"", + ""type"": ""CreationInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""name"": ""CC0-1.0"", + ""spdxId"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""type"": ""AnyLicenseInfo"" + }, + { + ""creationInfo"": ""_:creationinfo"", + ""namespaceMap"": { + ""sbom"": ""https://sbom.microsoft/1:EUb7DmXV0UyaZH1sttfV8A:n5KOcNVrWkGNryWx2sCN2A/13824:29192253/5qzhWC8k2k2wzStO28rMVQ"" + }, + ""rootElement"": [ + ""root-element-example"" + ], + ""dataLicense"": ""SPDXRef-AnyLicenseInfo-6E237C55B0583CB7BBA05562316C54B0A105ABA04775017E2253237B9A64613C"", + ""profileConformance"": [ + ""software"", + ""core"" + ], + ""name"": ""spdx-doc-name"", + ""spdxId"": ""SPDXRef-SpdxDocument-B93EED20C16A89A887B753958D42B794DD3C6570D3C2725B56B43477B38E05A1"", + ""type"": ""SpdxDocument"" + }, + { + ""name"": ""./sample/path"", + ""software_copyrightText"": ""sampleCopyright"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-software_File-B4A9F99A3A03B9273AE34753D96564CB4F2B0FAD885BBD36B0DD619E9E8AC967"", + ""verifiedUsing"": [ + { + ""algorithm"": ""sha1"", + ""hashValue"": ""sha1value"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-PackageVerificationCode-B1565820A5CDAC40E0520D23F9D0B1497F240DDC51D72EAC6423D97D952D444F"", + ""type"": ""PackageVerificationCode"" + }, + { + ""algorithm"": ""sha256"", + ""hashValue"": ""sha256value"", + ""creationInfo"": ""_:creationinfo"", + ""spdxId"": ""SPDXRef-PackageVerificationCode-5D5B09F6DCB2D53A5FFFC60C4AC0D55FABDF556069D6631545F42AA6E3500F2E"", + ""type"": ""PackageVerificationCode"" + } + ], + ""type"": ""software_File"" + } + ] + } + "; + [TestInitialize] public void Init() { @@ -379,4 +464,144 @@ public async Task SbomParserBasedValidationWorkflowTests_ReturnsSuccessAndValida fileSystemMock.VerifyAll(); osUtilsMock.VerifyAll(); } + + [TestMethod] + public async Task SbomParserBasedValidationWorkflowTests_SetsComplianceStandard_Succeeds() + { + // Arrange + var bytes = Encoding.UTF8.GetBytes(sbomMetadataJsonString); + using var stream = new MemoryStream(bytes); + + var mockSpdx30Parser = new SPDX30Parser(stream); + + var manifestParserProvider = new Mock(); + var manifestInterface = new Mock(); + var configurationMock = new Mock(); + var sbomConfigs = new Mock(); + var fileSystemMock = new Mock(); + var outputWriterMock = new Mock(); + var recorder = new Mock(); + var hashCodeGeneratorMock = new Mock(); + + manifestInterface.Setup(m => m.CreateParser(It.IsAny())) + .Returns(mockSpdx30Parser); + manifestParserProvider.Setup(m => m.Get(It.IsAny())).Returns(manifestInterface.Object); + + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); + configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); + configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); + configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configurationMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "child1;child2;child3" }); + configurationMock.SetupGet(c => c.IgnoreMissing).Returns(new ConfigurationSetting { Value = true }); + configurationMock.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + configurationMock.SetupGet(c => c.FollowSymlinks).Returns(new ConfigurationSetting { Value = true }); + configurationMock.SetupGet(c => c.ValidateSignature).Returns(new ConfigurationSetting { Value = true }); + configurationMock.SetupGet(c => c.ManifestInfo).Returns(new ConfigurationSetting> + { + Value = new List() { Constants.SPDX30ManifestInfo } + }); + configurationMock.SetupGet(c => c.ComplianceStandard).Returns(new ConfigurationSetting { Value = "NTIA" }); + + ISbomConfig sbomConfig = new SbomConfig(fileSystemMock.Object) + { + ManifestInfo = Constants.SPDX30ManifestInfo, + ManifestJsonDirPath = "/root/_manifest", + ManifestJsonFilePath = "/root/_manifest/spdx_3.0/manifest.spdx.json", + MetadataBuilder = null, + Recorder = new SbomPackageDetailsRecorder() + }; + + sbomConfigs.Setup(c => c.Get(Constants.SPDX30ManifestInfo)).Returns(sbomConfig); + + fileSystemMock.Setup(f => f.OpenRead("/root/_manifest/spdx_3.0/manifest.spdx.json")).Returns(Stream.Null); + //fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) + // .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); + fileSystemMock.Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())) + .Returns((string root, string relativePath) => $"{root}/{relativePath}"); + fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())) + .Returns(true); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object); + + var directoryWalker = new DirectoryWalker(fileSystemMock.Object, mockLogger.Object, configurationMock.Object); + + hashCodeGeneratorMock.Setup(h => h.GenerateHashes( + It.IsAny(), + new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + .Returns((string fileName, AlgorithmName[] algos) => + new Checksum[] + { + new Checksum + { + ChecksumValue = $"{fileName}hash", + Algorithm = Constants.DefaultHashAlgorithmName + } + }); + + var fileHasher = new FileHasher( + hashCodeGeneratorMock.Object, + new SbomToolManifestPathConverter(configurationMock.Object, mockOSUtils.Object, fileSystemMock.Object, fileSystemUtilsExtensionMock.Object), + mockLogger.Object, + configurationMock.Object, + new Mock().Object, + new ManifestGeneratorProvider(null), + new FileTypeUtils()); + + var manifestFilterMock = new ManifestFolderFilter(configurationMock.Object, mockOSUtils.Object); + manifestFilterMock.Init(); + var fileFilterer = new ManifestFolderFilterer(manifestFilterMock, mockLogger.Object); + + var rootFileFilterMock = new DownloadedRootPathFilter(configurationMock.Object, fileSystemMock.Object, mockLogger.Object); + rootFileFilterMock.Init(); + + var osUtilsMock = new Mock(MockBehavior.Strict); + + var hashValidator = new ConcurrentSha256HashValidator(FileHashesDictionarySingleton.Instance); + var enumeratorChannel = new EnumeratorChannel(mockLogger.Object); + var fileConverter = new SbomFileToFileInfoConverter(new FileTypeUtils()); + var spdxFileFilterer = new FileFilterer(rootFileFilterMock, mockLogger.Object, configurationMock.Object, fileSystemMock.Object); + + var filesValidator = new FilesValidator( + directoryWalker, + configurationMock.Object, + mockLogger.Object, + fileHasher, + fileFilterer, + hashValidator, + enumeratorChannel, + fileConverter, + FileHashesDictionarySingleton.Instance, + spdxFileFilterer); + + var validator = new SbomParserBasedValidationWorkflow( + recorder.Object, + signValidationProviderMock.Object, + mockLogger.Object, + manifestParserProvider.Object, + configurationMock.Object, + sbomConfigs.Object, + filesValidator, + validationResultGenerator, + outputWriterMock.Object, + fileSystemMock.Object, + osUtilsMock.Object); + + var cc = new ConsoleCapture(); + + try + { + var result = await validator.RunAsync(); + Assert.IsTrue(result); + Assert.AreEqual("NTIA", mockSpdx30Parser.RequiredComplianceStandard); + } + finally + { + cc.Restore(); + } + + configurationMock.VerifyAll(); + signValidatorMock.VerifyAll(); + fileSystemMock.VerifyAll(); + osUtilsMock.VerifyAll(); + } } diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs index f0185208a..68d5aea9e 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs @@ -28,7 +28,11 @@ public void MissingPropertiesTest_NTIA_NoVerificationCode_Throws() { var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithFilesStrings.SbomFileWithMissingVerificationJsonString); using var stream = new MemoryStream(bytes); - var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA"); + var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA") + { + // Setting RequiredComplianceStandard to NTIA is done normally in the SBOMParserBasedValidationWorkflow but doing it manually here for testing purposes + RequiredComplianceStandard = "NTIA" + }; Assert.ThrowsException(() => this.Parse(parser)); } @@ -37,7 +41,11 @@ public void MissingPropertiesTest_NTIA_VerificationCodeWithNoSHA256_Throws() { var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithFilesStrings.SbomFileWithMissingSHA256JsonString); using var stream = new MemoryStream(bytes); - var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA"); + var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA") + { + // Setting RequiredComplianceStandard to NTIA is done normally in the SBOMParserBasedValidationWorkflow but doing it manually here for testing purposes + RequiredComplianceStandard = "NTIA" + }; Assert.ThrowsException(() => this.Parse(parser)); } diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs index 15f16bac9..d72412c59 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs @@ -65,7 +65,11 @@ public void DocCreation_NoName_NTIA_Throws() var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithMetadataJsonStrings.SbomWithSpdxDocumentMissingNameJsonString); using var stream = new MemoryStream(bytes); - var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA"); + var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA") + { + // Setting RequiredComplianceStandard to NTIA is done normally in the SBOMParserBasedValidationWorkflow but doing it manually here for testing purposes + RequiredComplianceStandard = "NTIA" + }; Assert.ThrowsException(() => this.Parse(parser)); } diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs index 0bb6f8710..07dda45d4 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs @@ -2,16 +2,10 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; -using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text.Json; -using System.Text.Json.Nodes; -using Microsoft.Sbom.Contracts; using Microsoft.Sbom.JsonAsynchronousNodeKit; -using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; -using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums; namespace Microsoft.Sbom.Parser; @@ -42,25 +36,19 @@ public ParserResults Parse(SPDX30Parser parser, Stream? stream = null, bool clos if (result is not null && result.Result is not null) { - var jsonElements = result.Result as IEnumerable; - var jsonList = jsonElements?.ToList(); results.FormatEnforcedSPDX3Result ??= new FormatEnforcedSPDX3(); switch (result.FieldName) { case SPDX30Parser.ContextProperty: - if (jsonList == null || !jsonList.Any() || jsonList.Count > 1) - { - throw new ParserException($"The context property is either empty or has more than one string."); - } - else - { - results.FormatEnforcedSPDX3Result.Context = (string)jsonList.First(); - } - + results.FormatEnforcedSPDX3Result.Context = (string?)((ContextsResult)result).Contexts.FirstOrDefault(); break; case SPDX30Parser.GraphProperty: - results.FormatEnforcedSPDX3Result.Graph = ConvertToElements(jsonList, ref results, parser.RequiredComplianceStandard, parser.EntitiesToEnforceComplianceStandardsFor); - parser.Metadata = this.SetMetadata(results); + var elementsResult = (ElementsResult)result; + results.FormatEnforcedSPDX3Result.Graph = elementsResult.Elements; + results.FilesCount = elementsResult.FilesCount; + results.PackagesCount = elementsResult.PackagesCount; + results.RelationshipsCount = elementsResult.RelationshipsCount; + results.ReferencesCount = elementsResult.ReferencesCount; break; default: Console.WriteLine($"Unrecognized FieldName: {result.FieldName}"); @@ -72,229 +60,4 @@ public ParserResults Parse(SPDX30Parser parser, Stream? stream = null, bool clos return results; } - - /// - /// Converts JSON objects to SPDX elements. - /// - /// - /// - public List ConvertToElements(List? jsonList, ref ParserResults results, ComplianceStandard? requiredComplianceStandard, IReadOnlyCollection? entitiesWithDifferentNTIARequirements) - { - var elementsList = new List(); - - if (jsonList is null) - { - return elementsList; - } - - foreach (JsonObject jsonObject in jsonList) - { - var entityType = GetEntityType(jsonObject, requiredComplianceStandard, entitiesWithDifferentNTIARequirements); - - object? deserializedElement = null; - try - { - deserializedElement = JsonSerializer.Deserialize(jsonObject.ToString(), entityType); - } - catch (Exception e) - { - throw new ParserException(e.Message); - } - - if (deserializedElement != null) - { - elementsList.Add((Element)deserializedElement); - - switch (entityType?.Name) - { - case string name when name.Contains("File"): - results.FilesCount += 1; - break; - case string name when name.Contains("Package"): - results.PackagesCount += 1; - break; - case string name when name.Contains("ExternalMap"): - results.ReferencesCount += 1; - break; - case string name when name.Contains("Relationship"): - results.RelationshipsCount += 1; - break; - default: - Console.WriteLine($"Unrecognized entity type: {entityType?.Name}"); - break; - } - } - } - - // Validate if elements meet required compliance standards - switch (requiredComplianceStandard) - { - case ComplianceStandard.NTIA: - ValidateNTIARequirements(elementsList); - break; - } - - return elementsList; - } - - public Type GetEntityType(JsonObject jsonObject, ComplianceStandard? requiredComplianceStandard, IReadOnlyCollection? entitiesWithDifferentNTIARequirements) - { - var assembly = typeof(Element).Assembly; - var entityType = jsonObject["type"]?.ToString(); - - // For these special cases, remove the prefix from the type. - switch (entityType) - { - case "software_File": - entityType = "File"; - break; - case "software_Package": - entityType = "Package"; - break; - } - - switch (requiredComplianceStandard) - { - case ComplianceStandard.NTIA: - entityType = "NTIA" + entityType; - break; - } - - var type = assembly.GetType($"Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.{entityType}") ?? throw new ParserException($"{entityType} on {jsonObject} is invalid."); - - return type; - } - - public void ValidateNTIARequirements(List elementsList) - { - ValidateSbomDocCreationForNTIA(elementsList); - ValidateSbomFilesForNTIA(elementsList); - ValidateSbomPackagesForNTIA(elementsList); - } - - /// - /// Validate that information about the SBOM document is present. - /// - /// - /// - public void ValidateSbomDocCreationForNTIA(List elementsList) - { - var spdxDocumentElements = elementsList.Where(element => element is SpdxDocument); - if (spdxDocumentElements.Count() != 1) - { - throw new ParserException("SBOM document is not NTIA compliant because it must only contain one SpdxDocument element."); - } - - var spdxDocumentElement = spdxDocumentElements.First(); - - var spdxCreationInfoElement = (CreationInfo?)elementsList. - Where(element => element.Type == nameof(CreationInfo)). - FirstOrDefault(element => ((CreationInfo)element).Id == spdxDocumentElement.CreationInfoDetails) - ?? throw new ParserException($"SBOM document is not NTIA compliant because it must have a creationInfo element with ID of {spdxDocumentElement.CreationInfoDetails}"); - } - - /// - /// Validate that all files have declared and concluded licenses. - /// - /// - /// - public void ValidateSbomFilesForNTIA(List elementsList) - { - var fileElements = elementsList.Where(element => element is NTIAFile); - foreach (var fileElement in fileElements) - { - var fileSpdxId = fileElement.SpdxId; - - var fileHasSha256Hash = fileElement.VerifiedUsing. - Any(packageVerificationCode => packageVerificationCode.Algorithm == - HashAlgorithm.sha256); - - if (!fileHasSha256Hash) - { - throw new ParserException($"SBOM document is not NTIA compliant because file with SPDX ID {fileSpdxId} does not have a SHA256 hash."); - } - } - } - - /// - /// Validate that all packages have declared and concluded licenses. - /// - /// - /// - public void ValidateSbomPackagesForNTIA(List elementsList) - { - var packageElements = elementsList.Where(element => element is Package); - foreach (var packageElement in packageElements) - { - var packageSpdxId = packageElement.SpdxId; - - var packageHasSha256Hash = packageElement.VerifiedUsing. - Any(packageVerificationCode => packageVerificationCode.Algorithm == - HashAlgorithm.sha256); - - if (!packageHasSha256Hash) - { - throw new ParserException($"SBOM document is not NTIA compliant because package with SPDX ID {packageSpdxId} does not have a SHA256 hash."); - } - } - } - - /// - /// Sets metadata based on parsed SBOM elements. - /// - /// - public SpdxMetadata SetMetadata(ParserResults result) - { - var metadata = new SpdxMetadata(); - var spdxDocumentElement = (SpdxDocument?)result.FormatEnforcedSPDX3Result.Graph.FirstOrDefault(element => element.Type == "SpdxDocument"); - - if (spdxDocumentElement == null) - { - return metadata; - } - - if (spdxDocumentElement.NamespaceMap != null && spdxDocumentElement.NamespaceMap.TryGetValue("sbom", out var namespaceUri)) - { - metadata.DocumentNamespace = new Uri(namespaceUri); - } - - metadata.Name = spdxDocumentElement.Name; - metadata.SpdxId = spdxDocumentElement.SpdxId; - metadata.DocumentDescribes = spdxDocumentElement.RootElement; - - var dataLicenseSpdxId = spdxDocumentElement.DataLicense; - var spdxDataLicenseElement = result.FormatEnforcedSPDX3Result.Graph. - FirstOrDefault(element => element.SpdxId == dataLicenseSpdxId) as AnyLicenseInfo; - metadata.DataLicense = spdxDataLicenseElement?.Name; - - var spdxCreationInfoElement = (CreationInfo?)result.FormatEnforcedSPDX3Result.Graph. - Where(element => element.Type == nameof(CreationInfo)). - FirstOrDefault(element => ((CreationInfo)element).Id == spdxDocumentElement.CreationInfoDetails); - - var creators = spdxCreationInfoElement?.CreatedBy - .Select(createdBy => result.FormatEnforcedSPDX3Result.Graph. - FirstOrDefault(element => element.SpdxId == createdBy) as Organization) - .Where(spdxOrganizationElement => spdxOrganizationElement != null) - .Select(spdxOrganizationElement => spdxOrganizationElement?.Name) - .ToList() ?? []; - - creators.AddRange(spdxCreationInfoElement?.CreatedUsing - .Select(createdBy => result.FormatEnforcedSPDX3Result.Graph. - FirstOrDefault(element => element.SpdxId == createdBy) as Tool) - .Where(spdxToolElement => spdxToolElement != null) - .Select(spdxToolElement => spdxToolElement?.Name) ?? []); - - var createdDate = DateTime.MinValue; - DateTime.TryParse(spdxCreationInfoElement?.Created, out createdDate); - - metadata.CreationInfo = new MetadataCreationInfo - { - Created = createdDate, - Creators = creators, - }; - - metadata.SpdxVersion = spdxCreationInfoElement?.SpecVersion; - - return metadata; - } } diff --git a/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs b/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs index 44cb95d1f..c622a7831 100644 --- a/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs +++ b/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs @@ -18,6 +18,7 @@ namespace Microsoft.Sbom.Targets.Tests.Utility; internal class GeneratedSbomValidator { private const string SPDX22Specification = "SPDX:2.2"; + private const string SPDX30Specification = "SPDX:3.0"; private readonly string sbomSpecification; public GeneratedSbomValidator(string sbomSpecification) @@ -91,9 +92,14 @@ internal void AssertSbomIsValid(string manifestPath, string buildDropPath, strin { Assert.IsTrue(namespaceValue.Contains($"{expectedNamespaceUriBase.Trim()}/{expectedPackageName}/{expectedPackageVersion}")); } - } else + } + else if (this.sbomSpecification.Equals(SPDX30Specification)) + { + Console.Write("SPDX 3.0 specified, validation is handled through parser and validation already."); + } + else { - Assert.Fail("An unexpected SBOM specification was used. Please specify SPDX 2.2."); + Assert.Fail("An unexpected SBOM specification was used. Please specify a valid SPDX version. Current supported versions are 2.2 or 3.0."); } } From 0c6c7a32b6861be383fe8304d39b067daf9f3a84 Mon Sep 17 00:00:00 2001 From: ppandrate Date: Thu, 6 Feb 2025 21:03:02 -0800 Subject: [PATCH 05/15] parsing deduplication --- .../Workflows/Helpers/FilesValidator.cs | 128 ++---------------- .../SBOMParserBasedValidationWorkflow.cs | 23 ++-- .../Parser/SPDX30Parser.cs | 72 +++++++--- .../SbomParserBasedValidationWorkflowTests.cs | 1 - .../Parser/SbomFileParserTests.cs | 4 +- .../Parser/SbomMetadataParserTests.cs | 2 +- 6 files changed, 78 insertions(+), 152 deletions(-) diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/FilesValidator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/FilesValidator.cs index 1d6f76b83..73dba24bc 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/FilesValidator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/FilesValidator.cs @@ -59,7 +59,8 @@ public FilesValidator( this.spdxFileFilterer = spdxFileFilterer ?? throw new ArgumentNullException(nameof(spdxFileFilterer)); } - public async Task<(int, List)> Validate(IEnumerable files) + public async Task<(int, List)> Validate(IEnumerable files) + where T : class { var errors = new List>(); var results = new List>(); @@ -128,81 +129,6 @@ public FilesValidator( return (successCount, failures.Values.ToList()); } - public async Task<(int, List)> Validate(IEnumerable files) - { - var errors = new List>(); - var results = new List>(); - var failures = new Dictionary(); - - var (onDiskFileResults, onDiskFileErrors) = GetOnDiskFiles(); - results.AddRange(onDiskFileResults); - errors.AddRange(onDiskFileErrors); - - var workflowErrors = channelUtils.Merge(errors.ToArray()); - await foreach (var error in workflowErrors.ReadAllAsync()) - { - failures.Add(error.Path, error); - } - - var (inSbomFileResults, inSbomFileErrors) = GetInsideSbomFiles(files); - results.AddRange(inSbomFileResults); - errors.AddRange(inSbomFileErrors); - - var successCount = 0; - var resultChannel = channelUtils.Merge(results.ToArray()); - await foreach (var validationResult in resultChannel.ReadAllAsync()) - { - successCount++; - } - - workflowErrors = channelUtils.Merge(errors.ToArray()); - - await foreach (var error in workflowErrors.ReadAllAsync()) - { - failures.Add(error.Path, error); - } - - foreach (var file in fileHashesDictionary.FileHashes) - { - if (failures.ContainsKey(file.Key)) - { - // If we have added a validation error for this file, we don't need to add another one. - continue; - } - - if (file.Value == null) - { - // This generally means that we have case variations in the file names. - failures.Add(file.Key, new FileValidationResult - { - ErrorType = ErrorType.AdditionalFile, - Path = file.Key, - }); - continue; - } - - switch (file.Value.FileLocation) - { - case FileLocation.OnDisk: - failures.Add(file.Key, new FileValidationResult - { - ErrorType = ErrorType.AdditionalFile, - Path = file.Key, - }); - break; - case FileLocation.InSbomFile: - failures.Add(file.Key, new FileValidationResult - { - ErrorType = ErrorType.MissingFile, - Path = file.Key, - }); - break; - } - } - - return (successCount, failures.Values.ToList()); - } - private (List>, List>) GetOnDiskFiles() { var errors = new List>(); @@ -235,53 +161,25 @@ public FilesValidator( return (filesWithHashes, errors); } - private (List>, List>) GetInsideSbomFiles(IEnumerable files) + private (List>, List>) GetInsideSbomFiles(IEnumerable files) + where T : class { var errors = new List>(); var filesWithHashes = new List>(); // Enumerate files from SBOM - var (sbomFiles, sbomFileErrors) = enumeratorChannel.Enumerate(() => files.Select(f => f.ToSbomFile())); - errors.Add(sbomFileErrors); - - log.Debug($"Splitting the workflow into {configuration.Parallelism.Value} threads."); - var splitFilesChannels = channelUtils.Split(sbomFiles, configuration.Parallelism.Value); - - log.Debug("Waiting for the workflow to finish..."); - foreach (var fileChannel in splitFilesChannels) + var sbomFiles = files.Select(f => f switch { - // Convert files to internal SBOM format. - var (internalSbomFiles, converterErrors) = fileConverter.Convert(fileChannel, FileLocation.InSbomFile); - errors.Add(converterErrors); - - // Filter files. - var (filteredSbomFiles, filterErrors) = spdxFileFilterer.Filter(internalSbomFiles); - errors.Add(filterErrors); - - var (validationResults, validationErrors) = hashValidator.Validate(filteredSbomFiles); - errors.Add(validationErrors); - - filesWithHashes.Add(validationResults); - } - - return (filesWithHashes, errors); - } + SPDXFile spdxFile => spdxFile.ToSbomFile(), + File file => file.ToSbomFile(), + _ => throw new InvalidOperationException("Unsupported file type") + }); - private (List>, List>) GetInsideSbomFiles(IEnumerable files) - { - var errors = new List>(); - var filesWithHashes = new List>(); - - // Enumerate files from SBOM - - var file = files.FirstOrDefault().ToSbomFile(); - Console.WriteLine(file.Path); - - var (sbomFiles, sbomFileErrors) = enumeratorChannel.Enumerate(() => files.Select(f => f.ToSbomFile())); + var (sbomFilesChannel, sbomFileErrors) = enumeratorChannel.Enumerate(() => sbomFiles); errors.Add(sbomFileErrors); log.Debug($"Splitting the workflow into {configuration.Parallelism.Value} threads."); - var splitFilesChannels = channelUtils.Split(sbomFiles, configuration.Parallelism.Value); + var splitFilesChannels = channelUtils.Split(sbomFilesChannel, configuration.Parallelism.Value); log.Debug("Waiting for the workflow to finish..."); foreach (var fileChannel in splitFilesChannels) @@ -292,10 +190,10 @@ public FilesValidator( // Filter files. var (filteredSbomFiles, filterErrors) = spdxFileFilterer.Filter(internalSbomFiles); - //errors.Add(filterErrors); + errors.Add(filterErrors); var (validationResults, validationErrors) = hashValidator.Validate(filteredSbomFiles); - //errors.Add(validationErrors); + errors.Add(validationErrors); filesWithHashes.Add(validationResults); } diff --git a/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs index 17698afad..64d026ff1 100644 --- a/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs +++ b/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs @@ -126,12 +126,7 @@ public async Task RunAsync() { case FilesResult filesResult: (successfullyValidatedFiles, fileValidationFailures) = await filesValidator.Validate(filesResult.Files); - var invalidInputFiles = fileValidationFailures.Where(f => f.ErrorType == ErrorType.InvalidInputFile).ToList(); - if (invalidInputFiles.Count != 0) - { - throw new InvalidDataException($"Your manifest file is malformed. {invalidInputFiles.First().Path}"); - } - + ValidateInvalidInputFiles(fileValidationFailures); break; case PackagesResult packagesResult: var packages = packagesResult.Packages.ToList(); @@ -151,12 +146,7 @@ public async Task RunAsync() totalNumberOfPackages = elementsResult.PackagesCount; (successfullyValidatedFiles, fileValidationFailures) = await filesValidator.Validate(elementsResult.Files); - invalidInputFiles = fileValidationFailures.Where(f => f.ErrorType == ErrorType.InvalidInputFile).ToList(); - if (invalidInputFiles.Count != 0) - { - throw new InvalidDataException($"Your manifest file is malformed. {invalidInputFiles.First().Path}"); - } - + ValidateInvalidInputFiles(fileValidationFailures); break; default: break; @@ -309,4 +299,13 @@ private void LogResultsSummary(ValidationResult validationResultOutput, IEnumera Console.WriteLine($"Unknown file failures . . . . . . . . . . . . . {validFailures.Count(v => v.ErrorType == ErrorType.Other)}"); } + + private void ValidateInvalidInputFiles(List fileValidationFailures) + { + var invalidInputFiles = fileValidationFailures.Where(f => f.ErrorType == ErrorType.InvalidInputFile).ToList(); + if (invalidInputFiles.Count != 0) + { + throw new InvalidDataException($"Your manifest file is malformed. {invalidInputFiles.First().Path}"); + } + } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs index 87868b877..9f0e1a0da 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs @@ -60,7 +60,6 @@ public class SPDX30Parser : ISbomParser public SPDX30Parser( Stream stream, - string? requiredComplianceStandard = null, JsonSerializerOptions? jsonSerializerOptions = null, int? bufferSize = null) { @@ -77,9 +76,9 @@ public SPDX30Parser( if (!string.IsNullOrEmpty(this.RequiredComplianceStandard)) { - if (!Enum.TryParse(requiredComplianceStandard, true, out var complianceStandardAsEnum)) + if (!Enum.TryParse(this.RequiredComplianceStandard, true, out var complianceStandardAsEnum)) { - throw new ParserException($"{requiredComplianceStandard} compliance standard is not supported."); + throw new ParserException($"{this.RequiredComplianceStandard} compliance standard is not supported."); } else { @@ -89,7 +88,7 @@ public SPDX30Parser( this.EntitiesToEnforceComplianceStandardsFor = this.entitiesWithDifferentNTIARequirements; break; default: - throw new ParserException($"{requiredComplianceStandard} compliance standard is not supported."); + throw new ParserException($"{this.RequiredComplianceStandard} compliance standard is not supported."); } } } @@ -181,6 +180,7 @@ public ElementsResult ConvertToElements(List? jsonList, ref ParserStateR { var elementsResult = new ElementsResult(result); var elementsList = new List(); + var elementsSpdxIdList = new HashSet(); var filesList = new List(); if (jsonList is null) @@ -193,11 +193,28 @@ public ElementsResult ConvertToElements(List? jsonList, ref ParserStateR return elementsResult; } - // Parse requiredComplianceStandard into ComplianceStandard enum - var complianceStandardEnum = ComplianceStandard.None; - if (!string.IsNullOrEmpty(requiredComplianceStandard) && !Enum.TryParse(requiredComplianceStandard, true, out complianceStandardEnum)) + var complianceStandardAsEnum = ComplianceStandard.None; + if (!string.IsNullOrEmpty(this.RequiredComplianceStandard)) { - throw new ParserException($"Unrecognized compliance standard: \"{requiredComplianceStandard}\""); + if (!Enum.TryParse(this.RequiredComplianceStandard, true, out complianceStandardAsEnum)) + { + throw new ParserException($"{this.RequiredComplianceStandard} compliance standard is not supported."); + } + else + { + switch (complianceStandardAsEnum) + { + case ComplianceStandard.NTIA: + this.EntitiesToEnforceComplianceStandardsFor = this.entitiesWithDifferentNTIARequirements; + break; + default: + throw new ParserException($"{this.RequiredComplianceStandard} compliance standard is not supported."); + } + } + } + else + { + Console.WriteLine("No required compliance standard."); } foreach (JsonObject jsonObject in jsonList) @@ -207,21 +224,33 @@ public ElementsResult ConvertToElements(List? jsonList, ref ParserStateR continue; } - var entityType = GetEntityType(jsonObject, complianceStandardEnum, entitiesWithDifferentNTIARequirements); + var entityType = GetEntityType(jsonObject, complianceStandardAsEnum); - object? deserializedElement = null; + object? deserializedObject = null; try { - deserializedElement = JsonSerializer.Deserialize(jsonObject.ToString(), entityType, jsonSerializerOptions); + deserializedObject = JsonSerializer.Deserialize(jsonObject.ToString(), entityType, jsonSerializerOptions); } catch (Exception e) { throw new ParserException(e.Message); } - if (deserializedElement != null) + if (deserializedObject != null) { - elementsList.Add((Element)deserializedElement); + var deserializedElement = (Element)deserializedObject; + + // Deduplication of elements by checking SPDX ID + var spdxId = deserializedElement.SpdxId; + if (!elementsSpdxIdList.TryGetValue(spdxId, out _)) + { + elementsList.Add(deserializedElement); + } + else + { + Console.WriteLine($"Duplicate element with SPDX ID {spdxId} found. Skipping."); + elementsSpdxIdList.Add(spdxId); + } switch (entityType?.Name) { @@ -246,7 +275,7 @@ public ElementsResult ConvertToElements(List? jsonList, ref ParserStateR } // Validate if elements meet required compliance standards - switch (complianceStandardEnum) + switch (complianceStandardAsEnum) { case ComplianceStandard.NTIA: ValidateNTIARequirements(elementsList); @@ -259,7 +288,7 @@ public ElementsResult ConvertToElements(List? jsonList, ref ParserStateR return elementsResult; } - public Type GetEntityType(JsonObject jsonObject, ComplianceStandard? requiredComplianceStandard, IReadOnlyCollection? entitiesWithDifferentNTIARequirements) + public Type GetEntityType(JsonObject jsonObject, ComplianceStandard? requiredComplianceStandard) { var assembly = typeof(Element).Assembly; var typeFromSbom = jsonObject["type"]?.ToString(); @@ -277,14 +306,15 @@ public Type GetEntityType(JsonObject jsonObject, ComplianceStandard? requiredCom } // If the entity type is in the list of entities that require different NTIA requirements, then add the NTIA prefix. - if (entitiesWithDifferentNTIARequirements?.Contains(entityType) == true) + switch (requiredComplianceStandard) { - switch (requiredComplianceStandard) - { - case ComplianceStandard.NTIA: + case ComplianceStandard.NTIA: + if (this.EntitiesToEnforceComplianceStandardsFor?.Contains(entityType) == true) + { entityType = "NTIA" + entityType; - break; - } + } + + break; } var type = assembly.GetType($"Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.{entityType}") ?? throw new ParserException($"Type \"{typeFromSbom} on {jsonObject} is invalid."); diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs index 655125ba8..0502becbf 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs @@ -468,7 +468,6 @@ public async Task SbomParserBasedValidationWorkflowTests_ReturnsSuccessAndValida [TestMethod] public async Task SbomParserBasedValidationWorkflowTests_SetsComplianceStandard_Succeeds() { - // Arrange var bytes = Encoding.UTF8.GetBytes(sbomMetadataJsonString); using var stream = new MemoryStream(bytes); diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs index 68d5aea9e..f64411daf 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomFileParserTests.cs @@ -28,7 +28,7 @@ public void MissingPropertiesTest_NTIA_NoVerificationCode_Throws() { var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithFilesStrings.SbomFileWithMissingVerificationJsonString); using var stream = new MemoryStream(bytes); - var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA") + var parser = new SPDX30Parser(stream) { // Setting RequiredComplianceStandard to NTIA is done normally in the SBOMParserBasedValidationWorkflow but doing it manually here for testing purposes RequiredComplianceStandard = "NTIA" @@ -41,7 +41,7 @@ public void MissingPropertiesTest_NTIA_VerificationCodeWithNoSHA256_Throws() { var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithFilesStrings.SbomFileWithMissingSHA256JsonString); using var stream = new MemoryStream(bytes); - var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA") + var parser = new SPDX30Parser(stream) { // Setting RequiredComplianceStandard to NTIA is done normally in the SBOMParserBasedValidationWorkflow but doing it manually here for testing purposes RequiredComplianceStandard = "NTIA" diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs index d72412c59..747fd5bdc 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs @@ -65,7 +65,7 @@ public void DocCreation_NoName_NTIA_Throws() var bytes = Encoding.UTF8.GetBytes(SbomFullDocWithMetadataJsonStrings.SbomWithSpdxDocumentMissingNameJsonString); using var stream = new MemoryStream(bytes); - var parser = new SPDX30Parser(stream, requiredComplianceStandard: "NTIA") + var parser = new SPDX30Parser(stream) { // Setting RequiredComplianceStandard to NTIA is done normally in the SBOMParserBasedValidationWorkflow but doing it manually here for testing purposes RequiredComplianceStandard = "NTIA" From b5a943e37335390871fad66ac5be13329bb1f189 Mon Sep 17 00:00:00 2001 From: ppandrate Date: Fri, 7 Feb 2025 10:51:44 -0800 Subject: [PATCH 06/15] generator deduplication + add context --- .../Executors/Spdx3SerializationStrategy.cs | 29 +++++++++++++++++-- src/Microsoft.Sbom.Api/Utils/Constants.cs | 2 ++ .../Parser/SPDX30Parser.cs | 2 +- .../ManifestGenerationWorkflowTests.cs | 29 ++++++++++++++----- 4 files changed, 51 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs index c1e2ee5cf..15861e808 100644 --- a/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs +++ b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs @@ -1,9 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System; using System.Collections.Generic; +using System.Text.Json; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Extensions; namespace Microsoft.Sbom.Api.Workflows.Helpers; @@ -76,10 +79,17 @@ public async Task> WriteJsonObjectsToSbomAsync( private void WriteElementsToSbom(GenerateResult generateResult) { // Write the JSON objects to the SBOM - // TODO: avoid this for loop - // TODO: can add deduplication here foreach (var serializer in generateResult.SerializerToJsonDocuments.Keys) { + // Write context + serializer.StartJsonArray("@context"); + var document = JsonDocument.Parse(Constants.Spdx3Context); + serializer.Write(document); + serializer.EndJsonArray(); + + // Deduplication of elements by checking SPDX ID + var elementsSpdxIdList = new HashSet(); + serializer.StartJsonArray("@graph"); var jsonDocuments = generateResult.SerializerToJsonDocuments[serializer]; @@ -87,7 +97,20 @@ private void WriteElementsToSbom(GenerateResult generateResult) { foreach (var element in jsonDocument.RootElement.EnumerateArray()) { - serializer.Write(element); + if (element.TryGetProperty("spdxId", out var spdxIdField)) + { + var spdxId = spdxIdField.GetString(); + + if (!elementsSpdxIdList.TryGetValue(spdxId, out _)) + { + serializer.Write(element); + elementsSpdxIdList.Add(spdxId); + } + else + { + Console.WriteLine($"Duplicate element with SPDX ID {spdxId} found. Skipping."); + } + } } } diff --git a/src/Microsoft.Sbom.Api/Utils/Constants.cs b/src/Microsoft.Sbom.Api/Utils/Constants.cs index 7b86f3102..982322691 100644 --- a/src/Microsoft.Sbom.Api/Utils/Constants.cs +++ b/src/Microsoft.Sbom.Api/Utils/Constants.cs @@ -66,4 +66,6 @@ public static class Constants public const string BsiFileName = "bsi.json"; public const string DeleteManifestDirBoolVariableName = "DeleteManifestDirIfPresent"; + + public const string Spdx3Context = "\"https://spdx.org/rdf/3.0.1/spdx-context.json\""; } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs index 9f0e1a0da..eb65367d4 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs @@ -245,11 +245,11 @@ public ElementsResult ConvertToElements(List? jsonList, ref ParserStateR if (!elementsSpdxIdList.TryGetValue(spdxId, out _)) { elementsList.Add(deserializedElement); + elementsSpdxIdList.Add(spdxId); } else { Console.WriteLine($"Duplicate element with SPDX ID {spdxId} found. Skipping."); - elementsSpdxIdList.Add(spdxId); } switch (entityType?.Name) diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs index 99e289158..a3f2a5903 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs @@ -78,13 +78,13 @@ public void Setup() } [TestMethod] - [DataRow("test", true, true)] - [DataRow("test", false, false)] - [DataRow("test", true, false)] - [DataRow("test", false, true)] - [DataRow("3.0", true, true)] - [DataRow("3.0", false, false)] - [DataRow("3.0", true, false)] + //[DataRow("test", true, true)] + //[DataRow("test", false, false)] + //[DataRow("test", true, false)] + //[DataRow("test", false, true)] + //[DataRow("3.0", true, true)] + //[DataRow("3.0", false, false)] + //[DataRow("3.0", true, false)] [DataRow("3.0", false, true)] public async Task ManifestGenerationWorkflowTests_Succeeds(string spdxVersionForGenerator, bool deleteExistingManifestDir, bool isDefaultSourceManifestDirPath) { @@ -383,12 +383,27 @@ await externalDocumentReferenceChannel.Writer.WriteAsync(new ExternalDocumentRef } else { + // Test context is correct + var context = resultJson["@context"].ToString(); + Assert.AreEqual("[\r\n \"https://spdx.org/rdf/3.0.1/spdx-context.json\"\r\n]", context); + + // Test elements were generated correctly var elements = resultJson["@graph"].ToArray(); var packages = elements.Where(element => element["type"].ToString() == "software_Package").ToList(); Assert.AreEqual(4, packages.Count); var creationInfo = elements.Where(element => element["type"].ToString() == "CreationInfo").ToList(); Assert.AreEqual(1, creationInfo.Count); + + // Test deduplication + var noAssertionElements = elements.Where(element => element["spdxId"].ToString() == "SPDXRef-Element-D6D57C0C9CC2CAC35C83DE0C8E4C8C37B87C0A58DA49BB31EBEBC6E200F54D4B").ToList(); + Assert.AreEqual(1, noAssertionElements.Count); + + var organizationElements = elements.Where(element => element["type"].ToString() == "Organization").ToList(); + Assert.AreEqual(3, organizationElements.Count); + + var organizationElement = elements.Where(element => element["spdxId"].ToString() == "SPDXRef-Organization-8560FC6692684D8DF52223FF78E30B9630A1CF5A6FA371AAE24FCA896AE20969").ToList(); + Assert.AreEqual(1, organizationElement.Count); } configurationMock.VerifyAll(); From 886040aa4ab6087413897b8586ed98e8000b4c3c Mon Sep 17 00:00:00 2001 From: ppandrate Date: Fri, 7 Feb 2025 10:59:41 -0800 Subject: [PATCH 07/15] remove from public docs until release --- docs/sbom-tool-arguments.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/sbom-tool-arguments.md b/docs/sbom-tool-arguments.md index 6aa8491f7..ea5d369f0 100644 --- a/docs/sbom-tool-arguments.md +++ b/docs/sbom-tool-arguments.md @@ -41,7 +41,6 @@ Actions TelemetryFilePath (-t) Specify a file where we should write detailed telemetry for the workflow. FollowSymlinks (-F) If set to false, we will not follow symlinks while traversing the build drop folder. Default is set to 'true'. ManifestInfo (-mi) A list of the name and version of the manifest format that we are using. - ComplianceStandard (-cs) The compliance standard to validate against. Generate -options - Generate a SBOM for all the files in the given build drop folder, and the packages in the components path. From da1a30ff2250b78a778b985a687b390bee7cac82 Mon Sep 17 00:00:00 2001 From: ppandrate Date: Fri, 7 Feb 2025 11:15:28 -0800 Subject: [PATCH 08/15] clean up changes + update interface pins --- .../Executors/GenerateResult.cs | 2 - .../Executors/SBOMFileToFileInfoConverter.cs | 1 - .../Filters/DownloadedRootPathFilter.cs | 43 ++++--------------- .../Parser/SPDX30Parser.cs | 4 +- .../SBOMGeneratorTest.cs | 19 ++++++++ .../Version_3_0/InterfaceConcretionTests.cs | 7 +++ .../ManifestGenerationWorkflowTests.cs | 12 +++--- 7 files changed, 43 insertions(+), 45 deletions(-) diff --git a/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs b/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs index 6a7dcd0d6..230295533 100644 --- a/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs +++ b/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs @@ -22,6 +22,4 @@ public GenerateResult(List errors, Dictionary - ///// Initializes the root path filters list. - ///// - //public void Init() - //{ - // logger.Verbose("Adding root path filter valid paths"); - // skipValidation = true; - - // if (configuration.RootPathFilter != null && !string.IsNullOrWhiteSpace(configuration.RootPathFilter.Value)) - // { - // skipValidation = false; - // validPaths = new HashSet(); - // var relativeRootPaths = configuration.RootPathFilter.Value.Split(';'); - - // var r = relativeRootPaths.Select(r => - // new FileInfo(fileSystemUtils.JoinPaths(configuration.BuildDropPath.Value, r)) - // .FullName); - - // validPaths.UnionWith(r); - - // foreach (var validPath in validPaths) - // { - // logger.Verbose($"Added valid path {validPath}"); - // } - // } - //} - + /// + /// Initializes the root path filters list. + /// public void Init() { logger.Verbose("Adding root path filter valid paths"); @@ -107,12 +83,11 @@ public void Init() validPaths = new HashSet(); var relativeRootPaths = configuration.RootPathFilter.Value.Split(';'); - foreach (var relativePath in relativeRootPaths) - { - var path = fileSystemUtils.JoinPaths(configuration.BuildDropPath.Value, relativePath); - var fullPath = new FileInfo(path).FullName; - validPaths.Add(fullPath); - } + var r = relativeRootPaths.Select(r => + new FileInfo(fileSystemUtils.JoinPaths(configuration.BuildDropPath.Value, r)) + .FullName); + + validPaths.UnionWith(r); foreach (var validPath in validPaths) { diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs index 42be03d9d..785e85d00 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs @@ -400,9 +400,9 @@ public void ValidateSbomPackagesForNTIA(List elementsList) /// Sets metadata based on parsed SBOM elements. /// /// - public SpdxMetadata SetMetadata(ElementsResult result) + public Spdx22Metadata SetMetadata(ElementsResult result) { - var metadata = new SpdxMetadata(); + var metadata = new Spdx22Metadata(); var spdxDocumentElement = (SpdxDocument?)result.Elements.FirstOrDefault(element => element.Type == "SpdxDocument"); if (spdxDocumentElement == null) diff --git a/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs b/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs index aca9781fc..3cf09b6c4 100644 --- a/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs +++ b/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs @@ -86,6 +86,25 @@ public async Task When_GenerateSbomAsync_WithRecordedErrors_Then_PopulateEntityE [TestMethod] public async Task When_GenerateSbomAsync_WithNoRecordedErrors_Then_EmptyEntityErrors() + { + mockRecorder.Setup(c => c.Errors).Returns(new List()).Verifiable(); + mockWorkflow.Setup(c => c.RunAsync()).Returns(Task.FromResult(true)).Verifiable(); + + var metadata = new SBOMMetadata() + { + PackageSupplier = "Contoso" + }; + + generator = new SbomGenerator(mockWorkflow.Object, mockGeneratorProvider.Object, mockRecorder.Object, new List(), mockSanitizer.Object); + var result = await generator.GenerateSbomAsync("rootPath", "compPath", metadata, runtimeConfiguration: runtimeConfiguration); + + Assert.AreEqual(0, result.Errors.Count); + mockRecorder.Verify(); + mockWorkflow.Verify(); + } + + [TestMethod] + public async Task When_GenerateSbomAsync_Spdx30Generator_WithNoRecordedErrors_Then_EmptyEntityErrors() { mockRecorder.Setup(c => c.Errors).Returns(new List()).Verifiable(); mockWorkflow.Setup(c => c.RunAsync()).Returns(Task.FromResult(true)).Verifiable(); diff --git a/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs b/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs index e13daa9bf..33a0d9216 100644 --- a/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs @@ -187,7 +187,11 @@ private class PinnedIFileTypeUtils : IFileTypeUtils private class PinnedIJsonArrayGenerator : IJsonArrayGenerator { + public ISbomConfig SbomConfig { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public string SpdxManifestVersion { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public Task> GenerateAsync() => throw new NotImplementedException(); + Task IJsonArrayGenerator.GenerateAsync() => throw new NotImplementedException(); } private class PinnedISbomRedactor : ISbomRedactor @@ -231,6 +235,7 @@ private class PinnedISbomConfig : ISbomConfig private class PinnedIMetadataBuilder : IMetadataBuilder { public string GetHeaderJsonString(IInternalMetadataProvider internalMetadataProvider) => throw new NotImplementedException(); + public bool TryGetCreationInfoJson(IInternalMetadataProvider internalMetadataProvider, out GenerationResult generationResult) => throw new NotImplementedException(); public bool TryGetExternalRefArrayHeaderName(out string headerName) => throw new NotImplementedException(); public bool TryGetFilesArrayHeaderName(out string headerName) => throw new NotImplementedException(); public bool TryGetPackageArrayHeaderName(out string headerName) => throw new NotImplementedException(); @@ -247,6 +252,7 @@ private class PinnedIManifestToolJsonSerializer : IManifestToolJsonSerializer public void StartJsonArray(string arrayHeader) => throw new NotImplementedException(); public void StartJsonObject() => throw new NotImplementedException(); public void Write(JsonDocument jsonDocument) => throw new NotImplementedException(); + public void Write(JsonElement jsonElement) => throw new NotImplementedException(); public void WriteJsonString(string jsonString) => throw new NotImplementedException(); } @@ -322,6 +328,7 @@ private class PinnedIConfiguration : IConfiguration public ConfigurationSetting EnablePackageMetadataParsing { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } public ConfigurationSetting SbomPath { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } public ConfigurationSetting SbomDir { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } + public ConfigurationSetting ComplianceStandard { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } } private class PinnedISettingSourceable : ISettingSourceable diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs index a3f2a5903..d07fd64e2 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs @@ -78,12 +78,12 @@ public void Setup() } [TestMethod] - //[DataRow("test", true, true)] - //[DataRow("test", false, false)] - //[DataRow("test", true, false)] - //[DataRow("test", false, true)] - //[DataRow("3.0", true, true)] - //[DataRow("3.0", false, false)] + [DataRow("test", true, true)] + [DataRow("test", false, false)] + [DataRow("test", true, false)] + [DataRow("test", false, true)] + [DataRow("3.0", true, true)] + [DataRow("3.0", false, false)] //[DataRow("3.0", true, false)] [DataRow("3.0", false, true)] public async Task ManifestGenerationWorkflowTests_Succeeds(string spdxVersionForGenerator, bool deleteExistingManifestDir, bool isDefaultSourceManifestDirPath) From 06ef61544087388e80c88fbc998f65c1b7b5a855 Mon Sep 17 00:00:00 2001 From: ppandrate Date: Fri, 7 Feb 2025 12:00:58 -0800 Subject: [PATCH 09/15] update spdxmetadata - including this breaking change since this PR has other breaking changes --- src/Microsoft.Sbom.Contracts/Contracts/Spdx22Metadata.cs | 6 ------ src/Microsoft.Sbom.Extensions/ISbomParser.cs | 2 +- .../Parser/SPDXParser.cs | 4 ++-- .../Parser/SPDX30Parser.cs | 8 ++++---- .../Version_3_0/InterfaceConcretionTests.cs | 4 ++-- .../Parser/SbomFileParserTests.cs | 2 +- .../Parser/SbomMetadataParserTests.cs | 4 ++-- 7 files changed, 12 insertions(+), 18 deletions(-) delete mode 100644 src/Microsoft.Sbom.Contracts/Contracts/Spdx22Metadata.cs diff --git a/src/Microsoft.Sbom.Contracts/Contracts/Spdx22Metadata.cs b/src/Microsoft.Sbom.Contracts/Contracts/Spdx22Metadata.cs deleted file mode 100644 index 776884819..000000000 --- a/src/Microsoft.Sbom.Contracts/Contracts/Spdx22Metadata.cs +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -namespace Microsoft.Sbom.Contracts; - -public class Spdx22Metadata : SpdxMetadata { } diff --git a/src/Microsoft.Sbom.Extensions/ISbomParser.cs b/src/Microsoft.Sbom.Extensions/ISbomParser.cs index 243daa2e2..8db849683 100644 --- a/src/Microsoft.Sbom.Extensions/ISbomParser.cs +++ b/src/Microsoft.Sbom.Extensions/ISbomParser.cs @@ -27,7 +27,7 @@ public interface ISbomParser /// /// /// - Spdx22Metadata GetMetadata(); + SpdxMetadata GetMetadata(); /// /// This function is called by the sbom tool upon initialization to get all the diff --git a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs b/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs index 57c717718..53a4b248b 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx22SbomParser/Parser/SPDXParser.cs @@ -172,14 +172,14 @@ internal SPDXParser( return null; } - public Spdx22Metadata GetMetadata() + public SpdxMetadata GetMetadata() { if (!this.parsingComplete) { throw new ParserException($"{nameof(this.GetMetadata)} can only be called after Parsing is complete to ensure that a whole object is returned."); } - var spdxMetadata = new Spdx22Metadata(); + var spdxMetadata = new SpdxMetadata(); foreach (var kvp in this.metadata) { switch (kvp.Key) diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs index 785e85d00..7690ba780 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs @@ -39,7 +39,7 @@ public class SPDX30Parser : ISbomParser public string? RequiredComplianceStandard; public IReadOnlyCollection? EntitiesToEnforceComplianceStandardsFor; - public Spdx22Metadata Metadata = new Spdx22Metadata(); + public SpdxMetadata Metadata = new SpdxMetadata(); private readonly LargeJsonParser parser; private readonly IList observedFieldNames = new List(); private readonly bool requiredFieldsCheck = true; @@ -400,9 +400,9 @@ public void ValidateSbomPackagesForNTIA(List elementsList) /// Sets metadata based on parsed SBOM elements. /// /// - public Spdx22Metadata SetMetadata(ElementsResult result) + public SpdxMetadata SetMetadata(ElementsResult result) { - var metadata = new Spdx22Metadata(); + var metadata = new SpdxMetadata(); var spdxDocumentElement = (SpdxDocument?)result.Elements.FirstOrDefault(element => element.Type == "SpdxDocument"); if (spdxDocumentElement == null) @@ -455,7 +455,7 @@ public Spdx22Metadata SetMetadata(ElementsResult result) return metadata; } - public Spdx22Metadata GetMetadata() + public SpdxMetadata GetMetadata() { if (!this.parsingComplete) { diff --git a/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs b/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs index 33a0d9216..c85a48cf5 100644 --- a/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs @@ -286,7 +286,7 @@ private class PinnedISignValidator : ISignValidator private class PinnedISbomParser : ISbomParser { - public Spdx22Metadata GetMetadata() => throw new NotImplementedException(); + public SpdxMetadata GetMetadata() => throw new NotImplementedException(); public ParserStateResult Next() => throw new NotImplementedException(); public ManifestInfo[] RegisterManifest() => throw new NotImplementedException(); } @@ -376,7 +376,7 @@ private class PinnedISettingSourceable : ISettingSourceable // ScanSettings (CD) // SettingSource // Snippet - // Spdx22Metadata + // SpdxMetadata // SPDXFile // SPDXPackage // SPDXRelationship diff --git a/test/Microsoft.Sbom.Parsers.Spdx22SbomParser.Tests/Parser/SbomFileParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx22SbomParser.Tests/Parser/SbomFileParserTests.cs index 2ed2c5cc0..a595e3964 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx22SbomParser.Tests/Parser/SbomFileParserTests.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx22SbomParser.Tests/Parser/SbomFileParserTests.cs @@ -42,7 +42,7 @@ public void MetadataPopulates() var metadata = parser.GetMetadata(); Assert.IsNotNull(metadata); - Assert.IsInstanceOfType(metadata, typeof(Spdx22Metadata)); + Assert.IsInstanceOfType(metadata, typeof(SpdxMetadata)); Assert.IsNotNull(metadata.CreationInfo); var expectedTime = DateTime.Parse("2023-05-11T00:24:54Z").ToUniversalTime(); Assert.AreEqual(expectedTime, metadata.CreationInfo.Created); diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs index f1c667cd7..5aee4fb7b 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs @@ -28,7 +28,7 @@ public void MetadataPopulates() Assert.AreEqual(5, results.FormatEnforcedSPDX3Result.Graph.Count()); var metadata = parser.GetMetadata(); - Assert.IsInstanceOfType(metadata, typeof(Spdx22Metadata)); + Assert.IsInstanceOfType(metadata, typeof(SpdxMetadata)); Assert.IsNotNull(metadata); Assert.IsNotNull(metadata.DocumentNamespace); Assert.AreEqual("spdx-doc-name", metadata.Name); @@ -51,7 +51,7 @@ public void MetadataEmpty() var results = this.Parse(parser); var metadata = parser.GetMetadata(); - Assert.IsInstanceOfType(metadata, typeof(Spdx22Metadata)); + Assert.IsInstanceOfType(metadata, typeof(SpdxMetadata)); Assert.IsNotNull(metadata); Assert.IsNull(metadata.DocumentNamespace); Assert.IsNull(metadata.Name); From 0116139bd4d36e79b5d7473bb3362fee0355a6fb Mon Sep 17 00:00:00 2001 From: ppandrate Date: Mon, 10 Feb 2025 15:10:34 -0800 Subject: [PATCH 10/15] fix unit tests --- src/Microsoft.Sbom.Api/ConfigurationProfile.cs | 6 +++--- .../Workflows/SBOMGenerationWorkflow.cs | 11 ++++++++++- .../Helpers/RelationshipsArrayGeneratorTest.cs | 5 +++++ .../Workflows/ManifestGenerationWorkflowTests.cs | 2 +- 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.Sbom.Api/ConfigurationProfile.cs b/src/Microsoft.Sbom.Api/ConfigurationProfile.cs index a5897949b..e5bbe88de 100644 --- a/src/Microsoft.Sbom.Api/ConfigurationProfile.cs +++ b/src/Microsoft.Sbom.Api/ConfigurationProfile.cs @@ -40,7 +40,8 @@ public ConfigurationProfile() .ForMember(c => c.NamespaceUriUniquePart, o => o.Ignore()) .ForMember(c => c.NamespaceUriBase, o => o.Ignore()) .ForMember(c => c.DeleteManifestDirIfPresent, o => o.Ignore()) - .ForMember(c => c.PackageSupplier, o => o.Ignore()); + .ForMember(c => c.PackageSupplier, o => o.Ignore()) + .ForMember(c => c.ComplianceStandard, o => o.Ignore()); CreateMap() #pragma warning disable CS0618 // 'Configuration.ManifestPath' is obsolete: 'This field is not provided by the user or configFile, set by system' @@ -67,8 +68,7 @@ public ConfigurationProfile() .ForMember(c => c.NamespaceUriUniquePart, o => o.Ignore()) .ForMember(c => c.NamespaceUriBase, o => o.Ignore()) .ForMember(c => c.DeleteManifestDirIfPresent, o => o.Ignore()) - .ForMember(c => c.PackageSupplier, o => o.Ignore()) - .ForMember(c => c.ComplianceStandard, o => o.Ignore()); + .ForMember(c => c.PackageSupplier, o => o.Ignore()); // Create config for the generation args, ignoring other action members CreateMap() diff --git a/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs index f3e2a4932..ae0644193 100644 --- a/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs +++ b/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs @@ -15,6 +15,7 @@ using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; using PowerArgs; using Serilog; using Constants = Microsoft.Sbom.Api.Utils.Constants; @@ -98,8 +99,16 @@ public virtual async Task RunAsync() { sbomConfigs.ApplyToEachConfig(config => config.JsonSerializer.StartJsonObject()); + var manifestInfos = sbomConfigs.GetManifestInfos(); + + // If manifestInfos is empty (for example, this is the case for unit tests where GetManifestInfos() is not implemented), use the default SPDX 2.2 manifest info + if (!manifestInfos.Any()) + { + manifestInfos = new List { Constants.SPDX22ManifestInfo }; + } + // Use the WriteJsonObjectsToSbomAsync method based on the SPDX version in manifest info - foreach (var manifestInfo in sbomConfigs.GetManifestInfos()) + foreach (var manifestInfo in manifestInfos) { var config = sbomConfigs.Get(manifestInfo); diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs index 8defd003b..866932b36 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs @@ -79,6 +79,11 @@ public void Setup() MetadataBuilder = metadataBuilder, Recorder = recorder, }; + + // Set up the relationshipsArrayGenerator sbomConfig, this is normally done in SBOM generation workflow. + // Since we are testing the RelationshipsArrayGenerator, we need to set it up manually. + relationshipsArrayGenerator.SbomConfig = sbomConfig; + fileSystemUtilsMock.Setup(f => f.CreateDirectory(ManifestJsonDirPath)); fileSystemUtilsMock.Setup(f => f.OpenWrite(JsonFilePath)).Returns(new MemoryStream()); diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs index d07fd64e2..2cc196fe4 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs @@ -84,7 +84,7 @@ public void Setup() [DataRow("test", false, true)] [DataRow("3.0", true, true)] [DataRow("3.0", false, false)] - //[DataRow("3.0", true, false)] + [DataRow("3.0", true, false)] [DataRow("3.0", false, true)] public async Task ManifestGenerationWorkflowTests_Succeeds(string spdxVersionForGenerator, bool deleteExistingManifestDir, bool isDefaultSourceManifestDirPath) { From a12527f55826570e7b18608ff64ebe0848f8f6a1 Mon Sep 17 00:00:00 2001 From: ppandrate Date: Tue, 11 Feb 2025 12:53:16 -0800 Subject: [PATCH 11/15] Refactor constants --- Microsoft.Sbom.sln | 6 ++ .../Config/ApiConfigurationBuilder.cs | 16 ++--- .../Config/ConfigSanitizer.cs | 25 ++++---- src/Microsoft.Sbom.Api/Config/Validator.cs | 4 +- ...NameConfigurationSettingAddingConverter.cs | 4 +- ...evelConfigurationSettingAddingConverter.cs | 4 +- .../SbomToolManifestPathConverter.cs | 4 +- .../output/ValidationResultGenerator.cs | 6 +- .../Executors/ComponentDetectionBaseWalker.cs | 8 ++- ...{GenerateResult.cs => GenerationResult.cs} | 18 +++--- .../JsonSerializationStrategyFactory.cs | 4 +- ...XSBOMReaderForExternalDocumentReference.cs | 22 +++---- .../Executors/Spdx2SerializationStrategy.cs | 45 ++++---------- .../Executors/Spdx3SerializationStrategy.cs | 10 ++-- .../SPDX22ManifestConfigHandler.cs | 18 +++--- .../Microsoft.Sbom.Api.csproj | 1 + .../Output/Telemetry/TelemetryRecorder.cs | 4 +- src/Microsoft.Sbom.Api/SBOMGenerator.cs | 3 +- src/Microsoft.Sbom.Api/Utils/Constants.cs | 55 ----------------- src/Microsoft.Sbom.Api/Utils/FileTypeUtils.cs | 3 +- .../Utils/SBOMFormatExtensions.cs | 16 ----- .../ExternalDocumentReferenceGenerator.cs | 6 +- .../Workflows/Helpers/FileArrayGenerator.cs | 4 +- .../Workflows/Helpers/IJsonArrayGenerator.cs | 2 +- .../Helpers/PackageArrayGenerator.cs | 4 +- .../Helpers/RelationshipsArrayGenerator.cs | 6 +- .../Workflows/SBOMGenerationWorkflow.cs | 12 ++-- .../SBOMParserBasedValidationWorkflow.cs | 4 +- .../Microsoft.Sbom.Constants.csproj | 19 ++++++ src/Microsoft.Sbom.Constants/SpdxConstants.cs | 60 +++++++++++++++++++ .../ValidationService.cs | 3 +- .../ServiceCollectionExtensions.cs | 6 +- .../Entities/ManifestInfo.cs | 17 ++++++ .../Constants.cs | 10 ---- .../Generator.cs | 13 ++-- ...osoft.Sbom.Parsers.Spdx30SbomParser.csproj | 1 + .../Parser/SPDX30Parser.cs | 13 ++-- src/Microsoft.Sbom.Tool/ValidationService.cs | 3 +- .../Config/ConfigSanitizerTests.cs | 8 +-- .../Config/ConfigurationBuilderTestsBase.cs | 4 +- .../ConfigurationBuilderTestsForGeneration.cs | 12 ++-- .../ConfigurationBuilderTestsForValidation.cs | 4 +- .../output/ValidationResultGeneratorTests.cs | 4 +- .../ComponentToPackageInfoConverterTests.cs | 4 +- .../ExternalDocumentReferenceWriterTest.cs | 6 +- .../Executors/FileHasherTests.cs | 26 ++++---- .../Executors/HashValidatorTests.cs | 16 ++--- ...ReaderForExternalDocumentReferenceTests.cs | 4 +- .../Microsoft.Sbom.Api.Tests.csproj | 1 + .../Version_3_0/InterfaceConcretionTests.cs | 6 +- .../RelationshipsArrayGeneratorTest.cs | 6 +- .../ManifestGenerationWorkflowTests.cs | 14 ++--- .../SbomParserBasedValidationWorkflowTests.cs | 40 ++++++------- .../Parser/GeneratorTests.cs | 6 +- .../Parser/SbomMetadataParserTests.cs | 6 +- 55 files changed, 316 insertions(+), 310 deletions(-) rename src/Microsoft.Sbom.Api/Executors/{GenerateResult.cs => GenerationResult.cs} (55%) create mode 100644 src/Microsoft.Sbom.Constants/Microsoft.Sbom.Constants.csproj create mode 100644 src/Microsoft.Sbom.Constants/SpdxConstants.cs diff --git a/Microsoft.Sbom.sln b/Microsoft.Sbom.sln index cd1421e7a..cabc83db9 100644 --- a/Microsoft.Sbom.sln +++ b/Microsoft.Sbom.sln @@ -61,6 +61,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Sbom.Parsers.Spdx EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests", "test\Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests\Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests.csproj", "{E3FE33BB-FAB2-4F60-B930-BEB736AACE25}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Sbom.Constants", "src\Microsoft.Sbom.Constants\Microsoft.Sbom.Constants.csproj", "{B898DBCA-E6AF-496A-A0E0-A3D159018CE4}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -143,6 +145,10 @@ Global {E3FE33BB-FAB2-4F60-B930-BEB736AACE25}.Debug|Any CPU.Build.0 = Debug|Any CPU {E3FE33BB-FAB2-4F60-B930-BEB736AACE25}.Release|Any CPU.ActiveCfg = Release|Any CPU {E3FE33BB-FAB2-4F60-B930-BEB736AACE25}.Release|Any CPU.Build.0 = Release|Any CPU + {B898DBCA-E6AF-496A-A0E0-A3D159018CE4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B898DBCA-E6AF-496A-A0E0-A3D159018CE4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B898DBCA-E6AF-496A-A0E0-A3D159018CE4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B898DBCA-E6AF-496A-A0E0-A3D159018CE4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs b/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs index 95dfd2a2b..81042c5f0 100644 --- a/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs +++ b/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs @@ -10,8 +10,8 @@ using Microsoft.Sbom.Contracts.Enums; using Microsoft.Sbom.Extensions.Entities; using Serilog.Events; -using ApiConstants = Microsoft.Sbom.Api.Utils.Constants; -using Constants = Microsoft.Sbom.Common.Constants; +using SbomConstants = Microsoft.Sbom.Common.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Config; @@ -126,7 +126,7 @@ public static InputConfiguration GetConfiguration( // TODO: update to SPDX 3.0 for default. if (specifications is null || specifications.Count == 0) { - specifications = ApiConstants.SupportedSbomSpecifications; + specifications = SpdxConstants.SupportedSbomSpecifications; } var sanitizedRuntimeConfiguration = SanitiseRuntimeConfiguration(runtimeConfiguration); @@ -187,7 +187,7 @@ private static void SetVerbosity(RuntimeConfiguration sanitizedRuntimeConfigurat System.Diagnostics.Tracing.EventLevel.LogAlways => GetConfigurationSetting(LogEventLevel.Verbose), System.Diagnostics.Tracing.EventLevel.Warning => GetConfigurationSetting(LogEventLevel.Warning), System.Diagnostics.Tracing.EventLevel.Verbose => GetConfigurationSetting(LogEventLevel.Verbose), - _ => GetConfigurationSetting(Constants.DefaultLogLevel), + _ => GetConfigurationSetting(SbomConstants.DefaultLogLevel), }; } @@ -206,17 +206,17 @@ private static RuntimeConfiguration SanitiseRuntimeConfiguration(RuntimeConfigur { runtimeConfiguration = new RuntimeConfiguration { - WorkflowParallelism = Constants.DefaultParallelism, + WorkflowParallelism = SbomConstants.DefaultParallelism, Verbosity = System.Diagnostics.Tracing.EventLevel.Warning, DeleteManifestDirectoryIfPresent = false, FollowSymlinks = true }; } - if (runtimeConfiguration.WorkflowParallelism < Constants.MinParallelism - || runtimeConfiguration.WorkflowParallelism > Constants.MaxParallelism) + if (runtimeConfiguration.WorkflowParallelism < SbomConstants.MinParallelism + || runtimeConfiguration.WorkflowParallelism > SbomConstants.MaxParallelism) { - runtimeConfiguration.WorkflowParallelism = Constants.DefaultParallelism; + runtimeConfiguration.WorkflowParallelism = SbomConstants.DefaultParallelism; } return runtimeConfiguration; diff --git a/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs b/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs index 51374e83c..3a17c6911 100644 --- a/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs +++ b/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs @@ -14,7 +14,8 @@ using PowerArgs; using Serilog; using Serilog.Core; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SbomConstants = Microsoft.Sbom.Common.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Config; @@ -48,7 +49,7 @@ public IConfiguration SanitizeConfig(IConfiguration configuration) // Create temporary logger to show logs during config sanitizing var logger = new LoggerConfiguration() .MinimumLevel.ControlledBy(new LoggingLevelSwitch { MinimumLevel = configuration.Verbosity.Value }) - .WriteTo.Console(outputTemplate: Constants.LoggerTemplate) + .WriteTo.Console(outputTemplate: SpdxConstants.LoggerTemplate) .CreateLogger(); // If BuildDropPath is null then run the logic to check whether it is required or not based on the current configuration. @@ -82,21 +83,21 @@ public IConfiguration SanitizeConfig(IConfiguration configuration) if (configuration2 is not null) { // Prevent null value for LicenseInformationTimeoutInSeconds. - // Values of (0, Constants.MaxLicenseFetchTimeoutInSeconds] are allowed. Negative values are replaced with the default, and - // the higher values are truncated to the maximum of Common.Constants.MaxLicenseFetchTimeoutInSeconds + // Values of (0, SpdxConstants.MaxLicenseFetchTimeoutInSeconds] are allowed. Negative values are replaced with the default, and + // the higher values are truncated to the maximum of Common.SpdxConstants.MaxLicenseFetchTimeoutInSeconds if (configuration2.LicenseInformationTimeoutInSeconds is null) { - configuration2.LicenseInformationTimeoutInSeconds = new(Common.Constants.DefaultLicenseFetchTimeoutInSeconds, SettingSource.Default); + configuration2.LicenseInformationTimeoutInSeconds = new(SbomConstants.DefaultLicenseFetchTimeoutInSeconds, SettingSource.Default); } else if (configuration2.LicenseInformationTimeoutInSeconds.Value <= 0) { - logger.Warning($"Negative and Zero Values not allowed for timeout. Using the default {Common.Constants.DefaultLicenseFetchTimeoutInSeconds} seconds instead."); - configuration2.LicenseInformationTimeoutInSeconds.Value = Common.Constants.DefaultLicenseFetchTimeoutInSeconds; + logger.Warning($"Negative and Zero Values not allowed for timeout. Using the default {SbomConstants.DefaultLicenseFetchTimeoutInSeconds} seconds instead."); + configuration2.LicenseInformationTimeoutInSeconds.Value = SbomConstants.DefaultLicenseFetchTimeoutInSeconds; } - else if (configuration2.LicenseInformationTimeoutInSeconds.Value > Common.Constants.MaxLicenseFetchTimeoutInSeconds) + else if (configuration2.LicenseInformationTimeoutInSeconds.Value > SbomConstants.MaxLicenseFetchTimeoutInSeconds) { - logger.Warning($"Specified timeout exceeds maximum allowed. Truncating the timeout to {Common.Constants.MaxLicenseFetchTimeoutInSeconds} seconds."); - configuration2.LicenseInformationTimeoutInSeconds.Value = Common.Constants.MaxLicenseFetchTimeoutInSeconds; + logger.Warning($"Specified timeout exceeds maximum allowed. Truncating the timeout to {SbomConstants.MaxLicenseFetchTimeoutInSeconds} seconds."); + configuration2.LicenseInformationTimeoutInSeconds.Value = SbomConstants.MaxLicenseFetchTimeoutInSeconds; } // Check if arg -lto is specified but -li is not @@ -280,7 +281,7 @@ private ConfigurationSetting GetManifestDirPath(ConfigurationSetting { - Value = fileSystemUtils.JoinPaths(buildDropPath, Constants.ManifestFolder), + Value = fileSystemUtils.JoinPaths(buildDropPath, SpdxConstants.ManifestFolder), Source = SettingSource.Default }; } @@ -297,7 +298,7 @@ private string EnsurePathEndsWithManifestFolderForGenerate(string value, Manifes if (manifestToolAction == ManifestToolActions.Generate) { // For generate action, add the _manifest folder at the end of the path - return fileSystemUtils.JoinPaths(value, Constants.ManifestFolder); + return fileSystemUtils.JoinPaths(value, SpdxConstants.ManifestFolder); } return value; diff --git a/src/Microsoft.Sbom.Api/Config/Validator.cs b/src/Microsoft.Sbom.Api/Config/Validator.cs index 0230a4ea3..f544c103c 100644 --- a/src/Microsoft.Sbom.Api/Config/Validator.cs +++ b/src/Microsoft.Sbom.Api/Config/Validator.cs @@ -7,9 +7,9 @@ using Microsoft.Sbom.Api.Config.Args; using Microsoft.Sbom.Api.Exceptions; using Microsoft.Sbom.Api.Output.Telemetry; -using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Api.Workflows; using Microsoft.Sbom.Common.Config; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Config; @@ -36,7 +36,7 @@ public async Task Validate() bool result; try { - if (configuration.ManifestInfo.Value.Any(Constants.SupportedSpdxManifests.Contains)) + if (configuration.ManifestInfo.Value.Any(SpdxConstants.SupportedSpdxManifests.Contains)) { result = await parserValidationWorkflow.RunAsync(); } diff --git a/src/Microsoft.Sbom.Api/Config/ValueConverters/HashAlgorithmNameConfigurationSettingAddingConverter.cs b/src/Microsoft.Sbom.Api/Config/ValueConverters/HashAlgorithmNameConfigurationSettingAddingConverter.cs index 33a2bda29..37762cedc 100644 --- a/src/Microsoft.Sbom.Api/Config/ValueConverters/HashAlgorithmNameConfigurationSettingAddingConverter.cs +++ b/src/Microsoft.Sbom.Api/Config/ValueConverters/HashAlgorithmNameConfigurationSettingAddingConverter.cs @@ -2,9 +2,9 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using AutoMapper; -using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Contracts.Enums; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Config.ValueConverters; @@ -30,7 +30,7 @@ public ConfigurationSetting Convert(AlgorithmName sourceMember, R return new ConfigurationSetting { Source = settingSource, - Value = sourceMember ?? Constants.DefaultHashAlgorithmName + Value = sourceMember ?? SpdxConstants.DefaultHashAlgorithmName }; } } diff --git a/src/Microsoft.Sbom.Api/Config/ValueConverters/LogEventLevelConfigurationSettingAddingConverter.cs b/src/Microsoft.Sbom.Api/Config/ValueConverters/LogEventLevelConfigurationSettingAddingConverter.cs index 4e1a1552c..da7748caf 100644 --- a/src/Microsoft.Sbom.Api/Config/ValueConverters/LogEventLevelConfigurationSettingAddingConverter.cs +++ b/src/Microsoft.Sbom.Api/Config/ValueConverters/LogEventLevelConfigurationSettingAddingConverter.cs @@ -4,7 +4,7 @@ using AutoMapper; using Microsoft.Sbom.Common.Config; using Serilog.Events; -using Constants = Microsoft.Sbom.Common.Constants; +using SbomConstants = Microsoft.Sbom.Common.Constants; namespace Microsoft.Sbom.Api.Config.ValueConverters; @@ -30,7 +30,7 @@ public ConfigurationSetting Convert(LogEventLevel? sourceMember, return new ConfigurationSetting { Source = settingSource, - Value = sourceMember ?? Constants.DefaultLogLevel + Value = sourceMember ?? SbomConstants.DefaultLogLevel }; } } diff --git a/src/Microsoft.Sbom.Api/Converters/SbomToolManifestPathConverter.cs b/src/Microsoft.Sbom.Api/Converters/SbomToolManifestPathConverter.cs index c8e90adf4..12f7b603d 100644 --- a/src/Microsoft.Sbom.Api/Converters/SbomToolManifestPathConverter.cs +++ b/src/Microsoft.Sbom.Api/Converters/SbomToolManifestPathConverter.cs @@ -5,7 +5,7 @@ using Microsoft.Sbom.Api.Exceptions; using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Convertors; @@ -53,7 +53,7 @@ public SbomToolManifestPathConverter(IConfiguration configuration, IOSUtils osUt // Allow spdx files to be outside the root path, all externalDocumentReference must be in the file array regardless of where they are located. // More details are in this spec: https://github.com/spdx/spdx-spec/issues/571 - if (!path.EndsWith(Constants.SPDXFileExtension, osUtils.GetFileSystemStringComparisonType())) + if (!path.EndsWith(SpdxConstants.SPDXFileExtension, osUtils.GetFileSystemStringComparisonType())) { throw new InvalidPathException($"The file at {path} is outside the root path {buildDropPath}."); } diff --git a/src/Microsoft.Sbom.Api/Entities/output/ValidationResultGenerator.cs b/src/Microsoft.Sbom.Api/Entities/output/ValidationResultGenerator.cs index adc63f398..a0d225fe7 100644 --- a/src/Microsoft.Sbom.Api/Entities/output/ValidationResultGenerator.cs +++ b/src/Microsoft.Sbom.Api/Entities/output/ValidationResultGenerator.cs @@ -4,8 +4,8 @@ using System; using System.Collections.Generic; using System.Linq; -using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common.Config; +using ApiConstants = Microsoft.Sbom.Api.Utils.Constants; namespace Microsoft.Sbom.Api.Entities.Output; @@ -72,8 +72,8 @@ public ValidationResult Build() List validationErrors; List skippedErrors; - validationErrors = NodeValidationResults.Where(r => !Constants.SkipFailureReportingForErrors.Contains(r.ErrorType)).ToList(); - skippedErrors = NodeValidationResults.Where(r => Constants.SkipFailureReportingForErrors.Contains(r.ErrorType)).ToList(); + validationErrors = NodeValidationResults.Where(r => !ApiConstants.SkipFailureReportingForErrors.Contains(r.ErrorType)).ToList(); + skippedErrors = NodeValidationResults.Where(r => ApiConstants.SkipFailureReportingForErrors.Contains(r.ErrorType)).ToList(); if (configuration.IgnoreMissing.Value) { diff --git a/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs b/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs index 8a88acf89..924628949 100644 --- a/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs +++ b/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs @@ -18,8 +18,8 @@ using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Extensions; -using Constants = Microsoft.Sbom.Api.Utils.Constants; using ILogger = Serilog.ILogger; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Executors; @@ -79,8 +79,8 @@ public ComponentDetectionBaseWalker( cliArgumentBuilder.AddDetectorArg("SPDX22SBOM", "EnableIfDefaultOff"); cliArgumentBuilder.AddDetectorArg("ConanLock", "EnableIfDefaultOff"); - // Iterate over all supported SPDX manifests and apply the necessary logic - foreach (var supportedSpdxManifest in Constants.SupportedSpdxManifests) + // Iterate over all supported SPDX manifests and apply the necessary logic. Break after we find one match, since we can only match the sbomConfig once. + foreach (var supportedSpdxManifest in SpdxConstants.SupportedSpdxManifests) { if (sbomConfigs.TryGet(supportedSpdxManifest, out var spdxSbomConfig)) { @@ -90,6 +90,8 @@ public ComponentDetectionBaseWalker( { cliArgumentBuilder.AddArg("DirectoryExclusionList", directory); } + + break; } } diff --git a/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs b/src/Microsoft.Sbom.Api/Executors/GenerationResult.cs similarity index 55% rename from src/Microsoft.Sbom.Api/Executors/GenerateResult.cs rename to src/Microsoft.Sbom.Api/Executors/GenerationResult.cs index 230295533..ab1355f36 100644 --- a/src/Microsoft.Sbom.Api/Executors/GenerateResult.cs +++ b/src/Microsoft.Sbom.Api/Executors/GenerationResult.cs @@ -11,15 +11,15 @@ namespace Microsoft.Sbom.Api.Workflows.Helpers; /// /// Result from GenerateAsync /// -public class GenerateResult - { - public List Errors { get; set; } +public class GenerationResult +{ + public List Errors { get; set; } - public Dictionary> SerializerToJsonDocuments { get; set; } + public Dictionary> SerializerToJsonDocuments { get; set; } - public GenerateResult(List errors, Dictionary> serializerToJsonDocuments) - { - Errors = errors; - SerializerToJsonDocuments = serializerToJsonDocuments; - } + public GenerationResult(List errors, Dictionary> serializerToJsonDocuments) + { + Errors = errors; + SerializerToJsonDocuments = serializerToJsonDocuments; + } } diff --git a/src/Microsoft.Sbom.Api/Executors/JsonSerializationStrategyFactory.cs b/src/Microsoft.Sbom.Api/Executors/JsonSerializationStrategyFactory.cs index 6b359137b..a9be8854a 100644 --- a/src/Microsoft.Sbom.Api/Executors/JsonSerializationStrategyFactory.cs +++ b/src/Microsoft.Sbom.Api/Executors/JsonSerializationStrategyFactory.cs @@ -1,13 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; + namespace Microsoft.Sbom.Api.Workflows.Helpers; public static class JsonSerializationStrategyFactory { public static IJsonSerializationStrategy GetStrategy(string manifestInfoSpdxVersion) { - if (manifestInfoSpdxVersion == "3.0") + if (manifestInfoSpdxVersion == SpdxConstants.SPDX30ManifestInfo.Version) { return new Spdx3SerializationStrategy(); } diff --git a/src/Microsoft.Sbom.Api/Executors/SPDXSBOMReaderForExternalDocumentReference.cs b/src/Microsoft.Sbom.Api/Executors/SPDXSBOMReaderForExternalDocumentReference.cs index 3e3a2a91c..dab9096b8 100644 --- a/src/Microsoft.Sbom.Api/Executors/SPDXSBOMReaderForExternalDocumentReference.cs +++ b/src/Microsoft.Sbom.Api/Executors/SPDXSBOMReaderForExternalDocumentReference.cs @@ -17,8 +17,8 @@ using Microsoft.Sbom.Extensions; using Microsoft.Sbom.Extensions.Entities; using Serilog; -using Constants = Microsoft.Sbom.Api.Utils.Constants; using ErrorType = Microsoft.Sbom.Api.Entities.ErrorType; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Executors; @@ -81,7 +81,7 @@ public virtual (ChannelReader results, ChannelRea IList externalDocumentReferenceInfos = new List(); await foreach (var file in sbomFileLocation.ReadAllAsync()) { - if (!file.EndsWith(Constants.SPDXFileExtension, StringComparison.OrdinalIgnoreCase)) + if (!file.EndsWith(SpdxConstants.SPDXFileExtension, StringComparison.OrdinalIgnoreCase)) { log.Warning($"The file {file} is not an spdx document."); } @@ -151,13 +151,13 @@ private ExternalDocumentReferenceInfo ReadJson(string file) string versionValue; string rootElementValue; - if (root.TryGetProperty(Constants.SpdxVersionString, out var version)) + if (root.TryGetProperty(SpdxConstants.SpdxVersionString, out var version)) { versionValue = version.GetString(); } else { - throw new Exception($"{Constants.SpdxVersionString} property could not be parsed from referenced SPDX Document '{file}', this is not a valid SPDX-2.2 Document."); + throw new Exception($"{SpdxConstants.SpdxVersionString} property could not be parsed from referenced SPDX Document '{file}', this is not a valid SPDX-2.2 Document."); } if (!IsSPDXVersionSupported(versionValue)) @@ -165,31 +165,31 @@ private ExternalDocumentReferenceInfo ReadJson(string file) throw new Exception($"The SPDX version ${versionValue} is not valid format in the referenced SBOM, we currently only support SPDX-2.2 SBOM format."); } - if (root.TryGetProperty(Constants.NameString, out var name)) + if (root.TryGetProperty(SpdxConstants.NameString, out var name)) { nameValue = name.GetString(); } else { - throw new Exception($"{Constants.NameString} property could not be parsed from referenced SPDX Document '{file}'."); + throw new Exception($"{SpdxConstants.NameString} property could not be parsed from referenced SPDX Document '{file}'."); } - if (root.TryGetProperty(Constants.DocumentNamespaceString, out var documentNamespace)) + if (root.TryGetProperty(SpdxConstants.DocumentNamespaceString, out var documentNamespace)) { documentNamespaceValue = documentNamespace.GetString(); } else { - throw new Exception($"{Constants.DocumentNamespaceString} property could not be parsed from referenced SPDX Document '{file}'."); + throw new Exception($"{SpdxConstants.DocumentNamespaceString} property could not be parsed from referenced SPDX Document '{file}'."); } - if (root.TryGetProperty(Constants.DocumentDescribesString, out var rootElements)) + if (root.TryGetProperty(SpdxConstants.DocumentDescribesString, out var rootElements)) { - rootElementValue = rootElements.EnumerateArray().FirstOrDefault().ToString() ?? Constants.DefaultRootElement; + rootElementValue = rootElements.EnumerateArray().FirstOrDefault().ToString() ?? SpdxConstants.DefaultRootElement; } else { - throw new Exception($"{Constants.DocumentDescribesString} property could not be parsed from referenced SPDX Document '{file}'."); + throw new Exception($"{SpdxConstants.DocumentDescribesString} property could not be parsed from referenced SPDX Document '{file}'."); } return new ExternalDocumentReferenceInfo diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs index d337c9fd3..7e4ff6408 100644 --- a/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs +++ b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs @@ -81,59 +81,36 @@ public async Task> WriteJsonObjectsToSbomAsync( // Files section var filesGenerateResult = await fileArrayGenerator.GenerateAsync(); filesGenerateResult.Errors.AddRange(filesGenerateResult.Errors); - - foreach (var serializer in filesGenerateResult.SerializerToJsonDocuments.Keys) - { - foreach (var jsonDocument in filesGenerateResult.SerializerToJsonDocuments[serializer]) - { - serializer.Write(jsonDocument); - } - - serializer.EndJsonArray(); - } + WriteJsonObjectsFromGenerationResult(filesGenerateResult); // Packages section var packagesGenerateResult = await packageArrayGenerator.GenerateAsync(); packagesGenerateResult.Errors.AddRange(packagesGenerateResult.Errors); - - foreach (var serializer in packagesGenerateResult.SerializerToJsonDocuments.Keys) - { - foreach (var jsonDocument in packagesGenerateResult.SerializerToJsonDocuments[serializer]) - { - serializer.Write(jsonDocument); - } - - serializer.EndJsonArray(); - } + WriteJsonObjectsFromGenerationResult(packagesGenerateResult); // External Document Reference section var externalDocumentReferenceGenerateResult = await externalDocumentReferenceGenerator.GenerateAsync(); externalDocumentReferenceGenerateResult.Errors.AddRange(externalDocumentReferenceGenerateResult.Errors); - - foreach (var serializer in externalDocumentReferenceGenerateResult.SerializerToJsonDocuments.Keys) - { - foreach (var jsonDocument in externalDocumentReferenceGenerateResult.SerializerToJsonDocuments[serializer]) - { - serializer.Write(jsonDocument); - } - - serializer.EndJsonArray(); - } + WriteJsonObjectsFromGenerationResult(externalDocumentReferenceGenerateResult); // Relationships section var relationshipGenerateResult = await relationshipsArrayGenerator.GenerateAsync(); relationshipGenerateResult.Errors.AddRange(relationshipGenerateResult.Errors); + WriteJsonObjectsFromGenerationResult(relationshipGenerateResult); + + return errors; + } - foreach (var serializer in relationshipGenerateResult.SerializerToJsonDocuments.Keys) + private void WriteJsonObjectsFromGenerationResult(GenerationResult generationResult) + { + foreach (var serializer in generationResult.SerializerToJsonDocuments.Keys) { - foreach (var jsonDocument in relationshipGenerateResult.SerializerToJsonDocuments[serializer]) + foreach (var jsonDocument in generationResult.SerializerToJsonDocuments[serializer]) { serializer.Write(jsonDocument); } serializer.EndJsonArray(); } - - return errors; } } diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs index 15861e808..72eebc98a 100644 --- a/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs +++ b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs @@ -6,8 +6,8 @@ using System.Text.Json; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; -using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Extensions; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Workflows.Helpers; @@ -76,21 +76,21 @@ public async Task> WriteJsonObjectsToSbomAsync( return generateResult.Errors; } - private void WriteElementsToSbom(GenerateResult generateResult) + private void WriteElementsToSbom(GenerationResult generateResult) { // Write the JSON objects to the SBOM foreach (var serializer in generateResult.SerializerToJsonDocuments.Keys) { // Write context - serializer.StartJsonArray("@context"); - var document = JsonDocument.Parse(Constants.Spdx3Context); + serializer.StartJsonArray(SpdxConstants.SPDXContextHeaderName); + var document = JsonDocument.Parse(SpdxConstants.SPDX3ContextValue); serializer.Write(document); serializer.EndJsonArray(); // Deduplication of elements by checking SPDX ID var elementsSpdxIdList = new HashSet(); - serializer.StartJsonArray("@graph"); + serializer.StartJsonArray(SpdxConstants.SPDXGraphHeaderName); var jsonDocuments = generateResult.SerializerToJsonDocuments[serializer]; foreach (var jsonDocument in jsonDocuments) diff --git a/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX22ManifestConfigHandler.cs b/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX22ManifestConfigHandler.cs index 65271935e..3a53a6018 100644 --- a/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX22ManifestConfigHandler.cs +++ b/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX22ManifestConfigHandler.cs @@ -7,7 +7,7 @@ using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Extensions; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Manifest.ManifestConfigHandlers; @@ -24,21 +24,21 @@ public class SPDX22ManifestConfigHandler : IManifestConfigHandler // directory path for SPDX 2.2 is // root/_manifest/spdx_2.2/ - private string SbomDirPath => fileSystemUtils.JoinPaths(ManifestDirPath, $"{Constants.SPDX22ManifestInfo.Name.ToLower()}_{Constants.SPDX22ManifestInfo.Version.ToLower()}"); + private string SbomDirPath => fileSystemUtils.JoinPaths(ManifestDirPath, $"{SpdxConstants.SPDX22ManifestInfo.Name.ToLower()}_{SpdxConstants.SPDX22ManifestInfo.Version.ToLower()}"); // sbom file path is manifest.spdx.json in the sbom directory. - private string SbomFilePath => fileSystemUtils.JoinPaths(SbomDirPath, $"manifest.{Constants.SPDX22ManifestInfo.Name.ToLower()}.json"); + private string SbomFilePath => fileSystemUtils.JoinPaths(SbomDirPath, $"manifest.{SpdxConstants.SPDX22ManifestInfo.Name.ToLower()}.json"); // sha file is sbom file + .sha256 private string ManifestJsonSha256FilePath => $"{SbomFilePath}.sha256"; // catalog file is always manifest.cat - private string CatalogFilePath => fileSystemUtils.JoinPaths(SbomDirPath, Constants.CatalogFileName); + private string CatalogFilePath => fileSystemUtils.JoinPaths(SbomDirPath, SpdxConstants.CatalogFileName); // bsi.json file contains build session metadata and is always bsi.json - private string BsiJsonFilePath => fileSystemUtils.JoinPaths(SbomDirPath, Constants.BsiFileName); + private string BsiJsonFilePath => fileSystemUtils.JoinPaths(SbomDirPath, SpdxConstants.BsiFileName); - private IMetadataBuilder MetadataBuilder => metadataBuilderFactory.Get(Constants.SPDX22ManifestInfo); + private IMetadataBuilder MetadataBuilder => metadataBuilderFactory.Get(SpdxConstants.SPDX22ManifestInfo); public SPDX22ManifestConfigHandler( IConfiguration configuration, @@ -54,7 +54,7 @@ public bool TryGetManifestConfig(out ISbomConfig sbomConfig) { sbomConfig = new SbomConfig(fileSystemUtils) { - ManifestInfo = Constants.SPDX22ManifestInfo, + ManifestInfo = SpdxConstants.SPDX22ManifestInfo, ManifestJsonDirPath = SbomDirPath, ManifestJsonFilePath = SbomFilePath, CatalogFilePath = CatalogFilePath, @@ -70,7 +70,7 @@ public bool TryGetManifestConfig(out ISbomConfig sbomConfig) if (configuration.ManifestToolAction == ManifestToolActions.Generate) { if (configuration.ManifestInfo?.Value != null - && !configuration.ManifestInfo.Value.Contains(Constants.SPDX22ManifestInfo)) + && !configuration.ManifestInfo.Value.Contains(SpdxConstants.SPDX22ManifestInfo)) { return false; } @@ -83,7 +83,7 @@ public bool TryGetManifestConfig(out ISbomConfig sbomConfig) // We can only validate one format at a time, so check if its this one and return true/false. if (configuration.ManifestInfo?.Value != null && configuration.ManifestInfo.Value.Count == 1 - && configuration.ManifestInfo.Value.Contains(Constants.SPDX22ManifestInfo)) + && configuration.ManifestInfo.Value.Contains(SpdxConstants.SPDX22ManifestInfo)) { return true; } diff --git a/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj b/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj index 82acc4db0..f68a55dd1 100644 --- a/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj +++ b/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj @@ -54,6 +54,7 @@ + diff --git a/src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs b/src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs index 0a8572668..ec0742ef7 100644 --- a/src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs +++ b/src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs @@ -20,7 +20,7 @@ using Serilog; using Serilog.Core; using Spectre.Console; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Output.Telemetry; @@ -82,7 +82,7 @@ public async Task LogException(Exception exception) { logger = new LoggerConfiguration() .MinimumLevel.ControlledBy(new LoggingLevelSwitch { MinimumLevel = Configuration.Verbosity.Value }) - .WriteTo.Console(outputTemplate: Constants.LoggerTemplate) + .WriteTo.Console(outputTemplate: SpdxConstants.LoggerTemplate) .CreateLogger(); } diff --git a/src/Microsoft.Sbom.Api/SBOMGenerator.cs b/src/Microsoft.Sbom.Api/SBOMGenerator.cs index 06c9d259a..d4f1b680a 100644 --- a/src/Microsoft.Sbom.Api/SBOMGenerator.cs +++ b/src/Microsoft.Sbom.Api/SBOMGenerator.cs @@ -16,6 +16,7 @@ using Microsoft.Sbom.Common.Config.Validators; using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Contracts.Enums; +using Microsoft.Sbom.Extensions.Entities; using PowerArgs; namespace Microsoft.Sbom.Api; @@ -138,7 +139,7 @@ public IEnumerable GetRequiredAlgorithms(SbomSpecification specif public IEnumerable GetSupportedSBOMSpecifications() => generatorProvider .GetSupportedManifestInfos() - .Select(g => g.ToSBOMSpecification()) + .Select(ManifestInfo.ToSBOMSpecification) .ToList(); private InputConfiguration ValidateConfig(InputConfiguration config) diff --git a/src/Microsoft.Sbom.Api/Utils/Constants.cs b/src/Microsoft.Sbom.Api/Utils/Constants.cs index 982322691..0a51b6f61 100644 --- a/src/Microsoft.Sbom.Api/Utils/Constants.cs +++ b/src/Microsoft.Sbom.Api/Utils/Constants.cs @@ -2,70 +2,15 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; -using System.Collections.ObjectModel; -using Microsoft.Sbom.Contracts; -using Microsoft.Sbom.Contracts.Enums; -using Microsoft.Sbom.Extensions.Entities; namespace Microsoft.Sbom.Api.Utils; public static class Constants { - public const string ManifestFolder = "_manifest"; - public const string LoggerTemplate = "##[{Level:w}]{Message}{NewLine}{Exception}"; - - public static ManifestInfo SPDX22ManifestInfo = new ManifestInfo - { - Name = "SPDX", - Version = "2.2" - }; - - public static ManifestInfo SPDX30ManifestInfo = new ManifestInfo - { - Name = "SPDX", - Version = "3.0" - }; - - public static Collection SupportedSpdxManifests = new() - { - SPDX22ManifestInfo, - SPDX30ManifestInfo, - }; - - //public static SbomSpecification SPDX22Specification = SPDX22ManifestInfo.ToSBOMSpecification(); - - public static Collection SupportedSbomSpecifications = new() - { - SPDX22ManifestInfo.ToSBOMSpecification(), - SPDX30ManifestInfo.ToSBOMSpecification(), - }; - - // TODO: move to test csproj - public static ManifestInfo TestManifestInfo = new ManifestInfo - { - Name = "TestManifest", - Version = "1.0.0" - }; - public static List SkipFailureReportingForErrors = new() { Entities.ErrorType.ManifestFolder, Entities.ErrorType.FilteredRootPath, Entities.ErrorType.ReferencedSbomFile, }; - - public static AlgorithmName DefaultHashAlgorithmName = AlgorithmName.SHA256; - - public const string SPDXFileExtension = ".spdx.json"; - public const string DocumentNamespaceString = "documentNamespace"; - public const string NameString = "name"; - public const string DocumentDescribesString = "documentDescribes"; - public const string SpdxVersionString = "spdxVersion"; - public const string DefaultRootElement = "SPDXRef-Document"; - public const string CatalogFileName = "manifest.cat"; - public const string BsiFileName = "bsi.json"; - - public const string DeleteManifestDirBoolVariableName = "DeleteManifestDirIfPresent"; - - public const string Spdx3Context = "\"https://spdx.org/rdf/3.0.1/spdx-context.json\""; } diff --git a/src/Microsoft.Sbom.Api/Utils/FileTypeUtils.cs b/src/Microsoft.Sbom.Api/Utils/FileTypeUtils.cs index 8689ce997..2b08e78c6 100644 --- a/src/Microsoft.Sbom.Api/Utils/FileTypeUtils.cs +++ b/src/Microsoft.Sbom.Api/Utils/FileTypeUtils.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using Microsoft.Sbom.Contracts.Enums; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Utils; @@ -14,7 +15,7 @@ public class FileTypeUtils : IFileTypeUtils { public List GetFileTypesBy(string fileName) { - if (!string.IsNullOrWhiteSpace(fileName) && fileName.EndsWith(Constants.SPDXFileExtension, StringComparison.OrdinalIgnoreCase)) + if (!string.IsNullOrWhiteSpace(fileName) && fileName.EndsWith(SpdxConstants.SPDXFileExtension, StringComparison.OrdinalIgnoreCase)) { return new List { FileType.SPDX }; } diff --git a/src/Microsoft.Sbom.Api/Utils/SBOMFormatExtensions.cs b/src/Microsoft.Sbom.Api/Utils/SBOMFormatExtensions.cs index 9cbe58cc5..cfa8f545e 100644 --- a/src/Microsoft.Sbom.Api/Utils/SBOMFormatExtensions.cs +++ b/src/Microsoft.Sbom.Api/Utils/SBOMFormatExtensions.cs @@ -31,20 +31,4 @@ public static ManifestInfo ToManifestInfo(this SbomSpecification specification) Version = specification.Version }; } - - /// - /// Converts a to a object. - /// - /// - /// - /// - public static SbomSpecification ToSBOMSpecification(this ManifestInfo manifestInfo) - { - if (manifestInfo is null) - { - throw new ArgumentNullException(nameof(manifestInfo)); - } - - return new SbomSpecification(manifestInfo.Name, manifestInfo.Version); - } } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs index 6e1581938..6ded20cf0 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs @@ -40,7 +40,7 @@ public ExternalDocumentReferenceGenerator( this.recorder = recorder ?? throw new ArgumentNullException(nameof(recorder)); } - public async Task GenerateAsync() + public async Task GenerateAsync() { using (recorder.TraceEvent(Events.ExternalDocumentReferenceGeneration)) { @@ -52,7 +52,7 @@ public async Task GenerateAsync() if (!sourcesProviders.Any()) { log.Debug($"No source providers found for {ProviderType.ExternalDocumentReference}"); - return new GenerateResult(totalErrors, serializersToJsonDocs); + return new GenerationResult(totalErrors, serializersToJsonDocs); } // Write the start of the array, if supported. @@ -86,7 +86,7 @@ public async Task GenerateAsync() } } - return new GenerateResult(totalErrors, serializersToJsonDocs); + return new GenerationResult(totalErrors, serializersToJsonDocs); } } } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs index 67e0d34cf..00e2aae89 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs @@ -48,7 +48,7 @@ public FileArrayGenerator( /// The serializer used to write the SBOM. /// The header key for the file array object. /// - public async Task GenerateAsync() + public async Task GenerateAsync() { using (recorder.TraceEvent(Events.FilesGeneration)) { @@ -89,7 +89,7 @@ public async Task GenerateAsync() } } - return new GenerateResult(totalErrors, serializersToJsonDocs); + return new GenerationResult(totalErrors, serializersToJsonDocs); } } } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs index cb6229059..2ce05e555 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs @@ -21,5 +21,5 @@ public interface IJsonArrayGenerator /// specific type into the array. /// /// The list of failures. - Task GenerateAsync(); + Task GenerateAsync(); } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs index 72f69373b..31f3b5e9e 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs @@ -44,7 +44,7 @@ public PackageArrayGenerator( this.sbomConfigs = sbomConfigs ?? throw new ArgumentNullException(nameof(sbomConfigs)); } - public async Task GenerateAsync() + public async Task GenerateAsync() { using (recorder.TraceEvent(Events.PackagesGeneration)) { @@ -117,7 +117,7 @@ public async Task GenerateAsync() } } - return new GenerateResult(totalErrors, serializersToJsonDocs); + return new GenerationResult(totalErrors, serializersToJsonDocs); } } } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs index c6e77a72f..934c5223d 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs @@ -44,7 +44,7 @@ public RelationshipsArrayGenerator( this.recorder = recorder; } - public async Task GenerateAsync() + public async Task GenerateAsync() { using (recorder.TraceEvent(Events.RelationshipsGeneration)) { @@ -111,7 +111,7 @@ public async Task GenerateAsync() log.Debug($"Wrote {count} relationship elements in the SBOM."); } - return new GenerateResult(totalErrors, serializersToJsonDocs); + return new GenerationResult(totalErrors, serializersToJsonDocs); } } @@ -180,5 +180,5 @@ private IEnumerator GetRelationships(RelationshipType relationship } } - Task IJsonArrayGenerator.GenerateAsync() => throw new System.NotImplementedException(); + Task IJsonArrayGenerator.GenerateAsync() => throw new System.NotImplementedException(); } diff --git a/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs index ae0644193..b963c5a38 100644 --- a/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs +++ b/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs @@ -18,7 +18,7 @@ using Microsoft.Sbom.Extensions.Entities; using PowerArgs; using Serilog; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Workflows; @@ -104,7 +104,7 @@ public virtual async Task RunAsync() // If manifestInfos is empty (for example, this is the case for unit tests where GetManifestInfos() is not implemented), use the default SPDX 2.2 manifest info if (!manifestInfos.Any()) { - manifestInfos = new List { Constants.SPDX22ManifestInfo }; + manifestInfos = new List { SpdxConstants.SPDX22ManifestInfo }; } // Use the WriteJsonObjectsToSbomAsync method based on the SPDX version in manifest info @@ -235,22 +235,22 @@ private void RemoveExistingManifestDirectory() if (fileSystemUtils.DirectoryExists(rootManifestFolderPath)) { bool.TryParse( - osUtils.GetEnvironmentVariable(Constants.DeleteManifestDirBoolVariableName), + osUtils.GetEnvironmentVariable(SpdxConstants.DeleteManifestDirBoolVariableName), out var deleteSbomDirSwitch); - recorder.RecordSwitch(Constants.DeleteManifestDirBoolVariableName, deleteSbomDirSwitch); + recorder.RecordSwitch(SpdxConstants.DeleteManifestDirBoolVariableName, deleteSbomDirSwitch); if (!deleteSbomDirSwitch && !(configuration.DeleteManifestDirIfPresent?.Value ?? false)) { throw new ManifestFolderExistsException( $"The BuildDropRoot folder already contains a _manifest folder. Please" + $" delete this folder before running the generation or set the " + - $"{Constants.DeleteManifestDirBoolVariableName} environment variable to 'true' to " + + $"{SpdxConstants.DeleteManifestDirBoolVariableName} environment variable to 'true' to " + $"overwrite this folder."); } log.Warning( - $"Deleting pre-existing folder {rootManifestFolderPath} as {Constants.DeleteManifestDirBoolVariableName}" + + $"Deleting pre-existing folder {rootManifestFolderPath} as {SpdxConstants.DeleteManifestDirBoolVariableName}" + $" is 'true'."); fileSystemUtils.DeleteDir(rootManifestFolderPath, true); } diff --git a/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs index 64d026ff1..5f5d5886c 100644 --- a/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs +++ b/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs @@ -24,7 +24,7 @@ using Microsoft.Sbom.Parser; using PowerArgs; using Serilog; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using ApiConstants = Microsoft.Sbom.Api.Utils.Constants; namespace Microsoft.Sbom.Api.Workflows; @@ -187,7 +187,7 @@ public async Task RunAsync() await outputWriter.WriteAsync(JsonSerializer.Serialize(validationResultOutput, options)); - validFailures = fileValidationFailures.Where(f => !Constants.SkipFailureReportingForErrors.Contains(f.ErrorType)); + validFailures = fileValidationFailures.Where(f => !ApiConstants.SkipFailureReportingForErrors.Contains(f.ErrorType)); if (configuration.IgnoreMissing.Value) { diff --git a/src/Microsoft.Sbom.Constants/Microsoft.Sbom.Constants.csproj b/src/Microsoft.Sbom.Constants/Microsoft.Sbom.Constants.csproj new file mode 100644 index 000000000..ec54f1af0 --- /dev/null +++ b/src/Microsoft.Sbom.Constants/Microsoft.Sbom.Constants.csproj @@ -0,0 +1,19 @@ + + + + True + Constants used in the generated SBOM file according to SPDX specifications. + Microsoft.Sbom.Constants + True + $(StrongNameSigningKeyFilePath) + + + + False + + + + + + + diff --git a/src/Microsoft.Sbom.Constants/SpdxConstants.cs b/src/Microsoft.Sbom.Constants/SpdxConstants.cs new file mode 100644 index 000000000..21dc789bf --- /dev/null +++ b/src/Microsoft.Sbom.Constants/SpdxConstants.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.ObjectModel; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.Sbom.Extensions.Entities; + +namespace Microsoft.Sbom.Constants; + +public static class SpdxConstants +{ + public const string ManifestFolder = "_manifest"; + public const string LoggerTemplate = "##[{Level:w}]{Message}{NewLine}{Exception}"; + + public static ManifestInfo SPDX22ManifestInfo = new ManifestInfo + { + Name = "SPDX", + Version = "2.2" + }; + + public static ManifestInfo SPDX30ManifestInfo = new ManifestInfo + { + Name = "SPDX", + Version = "3.0" + }; + + public static Collection SupportedSbomSpecifications = new() + { + ManifestInfo.ToSBOMSpecification(SPDX22ManifestInfo), + ManifestInfo.ToSBOMSpecification(SPDX30ManifestInfo), + }; + + public static Collection SupportedSpdxManifests = new() + { + SPDX22ManifestInfo, + SPDX30ManifestInfo, + }; + + // TODO: move to test csproj + public static ManifestInfo TestManifestInfo = new ManifestInfo + { + Name = "TestManifest", + Version = "1.0.0" + }; + + public static AlgorithmName DefaultHashAlgorithmName = AlgorithmName.SHA256; + public const string SPDXFileExtension = ".spdx.json"; + public const string DocumentNamespaceString = "documentNamespace"; + public const string NameString = "name"; + public const string DocumentDescribesString = "documentDescribes"; + public const string SpdxVersionString = "spdxVersion"; + public const string DefaultRootElement = "SPDXRef-Document"; + public const string CatalogFileName = "manifest.cat"; + public const string BsiFileName = "bsi.json"; + public const string DeleteManifestDirBoolVariableName = "DeleteManifestDirIfPresent"; + public const string SPDX3ContextValue = "\"https://spdx.org/rdf/3.0.1/spdx-context.json\""; + public const string SPDXContextHeaderName = "@context"; + public const string SPDXGraphHeaderName = "@graph"; +} diff --git a/src/Microsoft.Sbom.DotNetTool/ValidationService.cs b/src/Microsoft.Sbom.DotNetTool/ValidationService.cs index cde2f9f9f..43fcd9f51 100644 --- a/src/Microsoft.Sbom.DotNetTool/ValidationService.cs +++ b/src/Microsoft.Sbom.DotNetTool/ValidationService.cs @@ -10,6 +10,7 @@ using Microsoft.Sbom.Api.Output.Telemetry; using Microsoft.Sbom.Api.Workflows; using IConfiguration = Microsoft.Sbom.Common.Config.IConfiguration; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Tool; @@ -40,7 +41,7 @@ public async Task StartAsync(CancellationToken cancellationToken) bool result; try { - if (configuration.ManifestInfo.Value.Contains(Api.Utils.Constants.SPDX22ManifestInfo)) + if (configuration.ManifestInfo.Value.Contains(SpdxConstants.SPDX22ManifestInfo)) { result = await parserValidationWorkflow.RunAsync(); } diff --git a/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs b/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs index 12d7988e7..b65390f1c 100644 --- a/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs +++ b/src/Microsoft.Sbom.Extensions.DependencyInjection/ServiceCollectionExtensions.cs @@ -39,8 +39,8 @@ using Serilog.Events; using Serilog.Extensions.Logging; using Serilog.Filters; -using Constants = Microsoft.Sbom.Api.Utils.Constants; using ILogger = Serilog.ILogger; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Extensions.DependencyInjection; @@ -162,7 +162,7 @@ public static IServiceCollection AddSbomTool(this IServiceCollection services, L var manifestData = new ManifestData(); - if (!configuration.ManifestInfo.Value.Any(manifestInfo => Constants.SupportedSpdxManifests.Contains(manifestInfo))) + if (!configuration.ManifestInfo.Value.Any(manifestInfo => SpdxConstants.SupportedSpdxManifests.Contains(manifestInfo))) { var sbomConfig = sbomConfigs.Get(configuration.ManifestInfo?.Value?.FirstOrDefault()); var parserProvider = x.GetRequiredService(); @@ -215,7 +215,7 @@ private static ILogger CreateLogger(LogEventLevel logLevel) .WriteTo.Map( LoggingEnricher.PrintStderrPropertyName, (printLogsToStderr, wt) => wt.Logger(lc => lc - .WriteTo.Console(outputTemplate: Constants.LoggerTemplate, standardErrorFromLevel: printLogsToStderr ? LogEventLevel.Debug : null) + .WriteTo.Console(outputTemplate: SpdxConstants.LoggerTemplate, standardErrorFromLevel: printLogsToStderr ? LogEventLevel.Debug : null) // Don't write the detection times table from DetectorProcessingService to the console, only the log file .Filter.ByExcluding(Matching.WithProperty("DetectionTimeLine", x => !string.IsNullOrEmpty(x)))), diff --git a/src/Microsoft.Sbom.Extensions/Entities/ManifestInfo.cs b/src/Microsoft.Sbom.Extensions/Entities/ManifestInfo.cs index c47888be9..30787d031 100644 --- a/src/Microsoft.Sbom.Extensions/Entities/ManifestInfo.cs +++ b/src/Microsoft.Sbom.Extensions/Entities/ManifestInfo.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using Microsoft.Sbom.Contracts; namespace Microsoft.Sbom.Extensions.Entities; @@ -92,4 +93,20 @@ public string ToLowerString() { return $"{Name.ToLowerInvariant()}:{Version.ToUpperInvariant()}"; } + + /// + /// Converts a to a object. + /// + /// + /// + /// + public static SbomSpecification ToSBOMSpecification(ManifestInfo manifestInfo) + { + if (manifestInfo is null) + { + throw new ArgumentNullException(nameof(manifestInfo)); + } + + return new SbomSpecification(manifestInfo.Name, manifestInfo.Version); + } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs index 1558ed0aa..e03f5dafb 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Constants.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using Microsoft.Sbom.Contracts.Enums; -using Microsoft.Sbom.Extensions.Entities; using HashAlgorithm = Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums.HashAlgorithm; namespace Microsoft.Sbom.Parsers.Spdx30SbomParser; @@ -18,9 +17,6 @@ internal static class Constants internal const string SPDXDocumentNameFormatString = "{0} {1}"; internal const string PackageSupplierFormatString = "Organization: {0}"; - internal const string SPDXContextHeaderName = "@context"; - internal const string SPDXGraphHeaderName = "@graph"; - internal const string SPDXVersionHeaderName = "spdxVersion"; internal const string DataLicenseHeaderName = "dataLicense"; internal const string SPDXIDHeaderName = "SPDXID"; @@ -44,12 +40,6 @@ internal static class Constants /// internal static IEnumerable NoAssertionListValue = new List { NoAssertionValue }; - internal static ManifestInfo Spdx30ManifestInfo = new ManifestInfo - { - Name = SPDXName, - Version = SPDXVersion - }; - public static readonly Dictionary AlgorithmMap = new() { { AlgorithmName.SHA1, HashAlgorithm.sha1 }, diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs index c9d6caa6b..131857539 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs @@ -19,6 +19,7 @@ using RelationshipType = Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums.RelationshipType; using SbomEntities = Microsoft.Sbom.Extensions.Entities; using SHA1 = System.Security.Cryptography.SHA1; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; using SpdxEntities = Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; namespace Microsoft.Sbom.Parsers.Spdx30SbomParser; @@ -333,7 +334,7 @@ public GenerationResult GenerateJsonDocument(IInternalMetadataProvider internalM throw new ArgumentNullException(nameof(internalMetadataProvider)); } - var generationData = internalMetadataProvider.GetGenerationData(Constants.Spdx30ManifestInfo); + var generationData = internalMetadataProvider.GetGenerationData(SpdxConstants.SPDX30ManifestInfo); var sbomToolName = internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolName); var sbomToolVersion = internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolVersion); @@ -446,7 +447,7 @@ private List ConvertSbomFileToSpdxFileAndRelationships(InternalSbomFile } // Generate SPDX file element - var spdxFileElement = new SpdxEntities.File + var spdxFileElement = new File { VerifiedUsing = packageVerificationCodes, Name = GeneratorUtils.EnsureRelativePathStartsWithDot(fileInfo.Path), @@ -531,7 +532,7 @@ private List GetSpdxRelationshipsAndLicensesFromSbomPackage(SbomPackage var licenseDeclaredElement = GenerateLicenseElement(packageInfo.LicenseInfo?.Declared); spdxRelationshipAndLicenseElementsToAddToSBOM.Add(licenseDeclaredElement); - var spdxRelationshipLicenseDeclaredElement = new Entities.Relationship + var spdxRelationshipLicenseDeclaredElement = new SpdxEntities.Relationship { From = spdxPackage.SpdxId, RelationshipType = RelationshipType.HAS_DECLARED_LICENSE, @@ -593,7 +594,7 @@ private PackageVerificationCode GetPackageVerificationCode(IInternalMetadataProv { // Get a list of SHA1 checksums IList sha1Checksums = new List(); - foreach (var checksumArray in internalMetadataProvider.GetGenerationData(Constants.Spdx30ManifestInfo).Checksums) + foreach (var checksumArray in internalMetadataProvider.GetGenerationData(SpdxConstants.SPDX30ManifestInfo).Checksums) { sha1Checksums.Add(checksumArray .Where(c => c.Algorithm == AlgorithmName.SHA1) @@ -616,7 +617,7 @@ private PackageVerificationCode GetPackageVerificationCode(IInternalMetadataProv return packageVerificationCode; } - public ManifestInfo RegisterManifest() => Constants.Spdx30ManifestInfo; + public ManifestInfo RegisterManifest() => SpdxConstants.SPDX30ManifestInfo; public IDictionary GetMetadataDictionary(IInternalMetadataProvider internalMetadataProvider) { @@ -625,7 +626,7 @@ public IDictionary GetMetadataDictionary(IInternalMetadataProvid throw new ArgumentNullException(nameof(internalMetadataProvider)); } - var generationData = internalMetadataProvider.GetGenerationData(Constants.Spdx30ManifestInfo); + var generationData = internalMetadataProvider.GetGenerationData(SpdxConstants.SPDX30ManifestInfo); var sbomToolName = internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolName); var sbomToolVersion = internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolVersion); diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Microsoft.Sbom.Parsers.Spdx30SbomParser.csproj b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Microsoft.Sbom.Parsers.Spdx30SbomParser.csproj index 95b226258..5cf0b3308 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Microsoft.Sbom.Parsers.Spdx30SbomParser.csproj +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Microsoft.Sbom.Parsers.Spdx30SbomParser.csproj @@ -10,6 +10,7 @@ + diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs index 7690ba780..c0ce167a8 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs @@ -15,7 +15,7 @@ using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities.Enums; -using SPDXConstants = Microsoft.Sbom.Parsers.Spdx30SbomParser.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Parser; @@ -29,8 +29,8 @@ namespace Microsoft.Sbom.Parser; /// public class SPDX30Parser : ISbomParser { - public const string ContextProperty = SPDXConstants.SPDXContextHeaderName; - public const string GraphProperty = SPDXConstants.SPDXGraphHeaderName; + public const string ContextProperty = SpdxConstants.SPDXContextHeaderName; + public const string GraphProperty = SpdxConstants.SPDXGraphHeaderName; public static readonly IReadOnlyCollection RequiredFields = new List { ContextProperty, @@ -45,11 +45,6 @@ public class SPDX30Parser : ISbomParser private readonly bool requiredFieldsCheck = true; private readonly JsonSerializerOptions jsonSerializerOptions; private bool parsingComplete = false; - private readonly ManifestInfo spdxManifestInfo = new() - { - Name = SPDXConstants.SPDXName, - Version = SPDXConstants.SPDXVersion, - }; private readonly IReadOnlyCollection entitiesWithDifferentNTIARequirements = new List { @@ -466,5 +461,5 @@ public SpdxMetadata GetMetadata() return this.Metadata; } - public ManifestInfo[] RegisterManifest() => new ManifestInfo[] { this.spdxManifestInfo }; + public ManifestInfo[] RegisterManifest() => new ManifestInfo[] { SpdxConstants.SPDX30ManifestInfo }; } diff --git a/src/Microsoft.Sbom.Tool/ValidationService.cs b/src/Microsoft.Sbom.Tool/ValidationService.cs index 55dbe5a62..73eb4f51b 100644 --- a/src/Microsoft.Sbom.Tool/ValidationService.cs +++ b/src/Microsoft.Sbom.Tool/ValidationService.cs @@ -10,6 +10,7 @@ using Microsoft.Sbom.Api.Output.Telemetry; using Microsoft.Sbom.Api.Workflows; using IConfiguration = Microsoft.Sbom.Common.Config.IConfiguration; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Tool; @@ -40,7 +41,7 @@ public async Task StartAsync(CancellationToken cancellationToken) bool result; try { - if (configuration.ManifestInfo.Value.Contains(Api.Utils.Constants.SPDX22ManifestInfo)) + if (configuration.ManifestInfo.Value.Contains(SpdxConstants.SPDX22ManifestInfo)) { result = await parserValidationWorkflow.RunAsync(); } diff --git a/test/Microsoft.Sbom.Api.Tests/Config/ConfigSanitizerTests.cs b/test/Microsoft.Sbom.Api.Tests/Config/ConfigSanitizerTests.cs index 98e2249b4..d8dfd30d8 100644 --- a/test/Microsoft.Sbom.Api.Tests/Config/ConfigSanitizerTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Config/ConfigSanitizerTests.cs @@ -18,7 +18,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using PowerArgs; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Tests.Config; @@ -81,7 +81,7 @@ private Configuration GetConfigurationBaseObject() { Source = SettingSource.Default, Value = new List - { Constants.TestManifestInfo } + { SpdxConstants.TestManifestInfo } }, Verbosity = new ConfigurationSetting { @@ -152,13 +152,13 @@ public void NoValueForManifestInfoForValidation_SetsDefaultValue() var config = GetConfigurationBaseObject(); config.ManifestToolAction = ManifestToolActions.Validate; config.ManifestInfo.Value.Clear(); - mockAssemblyConfig.SetupGet(a => a.DefaultManifestInfoForValidationAction).Returns(Constants.TestManifestInfo); + mockAssemblyConfig.SetupGet(a => a.DefaultManifestInfoForValidationAction).Returns(SpdxConstants.TestManifestInfo); var sanitizedConfig = configSanitizer.SanitizeConfig(config); Assert.IsNotNull(sanitizedConfig.ManifestInfo.Value); Assert.AreEqual(1, sanitizedConfig.ManifestInfo.Value.Count); - Assert.AreEqual(Constants.TestManifestInfo, sanitizedConfig.ManifestInfo.Value.First()); + Assert.AreEqual(SpdxConstants.TestManifestInfo, sanitizedConfig.ManifestInfo.Value.First()); mockAssemblyConfig.VerifyGet(a => a.DefaultManifestInfoForValidationAction); } diff --git a/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsBase.cs b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsBase.cs index f6b72decd..460bad37f 100644 --- a/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsBase.cs +++ b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsBase.cs @@ -11,7 +11,7 @@ using Microsoft.Sbom.Contracts.Entities; using Microsoft.Sbom.Contracts.Interfaces; using Moq; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Config.Tests; @@ -26,7 +26,7 @@ protected void Init() { fileSystemUtilsMock = new Mock(); mockAssemblyConfig = new Mock(); - mockAssemblyConfig.SetupGet(a => a.DefaultManifestInfoForValidationAction).Returns(Constants.TestManifestInfo); + mockAssemblyConfig.SetupGet(a => a.DefaultManifestInfoForValidationAction).Returns(SpdxConstants.TestManifestInfo); configValidators = new ConfigValidator[] { diff --git a/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForGeneration.cs b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForGeneration.cs index faf1dacca..a2ad05abe 100644 --- a/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForGeneration.cs +++ b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForGeneration.cs @@ -6,11 +6,11 @@ using Microsoft.Sbom.Api.Config.Args; using Microsoft.Sbom.Api.Exceptions; using Microsoft.Sbom.Api.Tests; -using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common.Config; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using PowerArgs; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Config.Tests; @@ -142,7 +142,7 @@ public async Task ConfigurationBuilderTest_Generation_DefaultManifestDirPath_Add Assert.IsNotNull(config); Assert.IsNotNull(config.ManifestDirPath); - var expectedPath = Path.Join(args.BuildDropPath, Constants.ManifestFolder); + var expectedPath = Path.Join(args.BuildDropPath, SpdxConstants.ManifestFolder); Assert.AreEqual(Path.GetFullPath(expectedPath), Path.GetFullPath(config.ManifestDirPath.Value)); fileSystemUtilsMock.VerifyAll(); @@ -172,7 +172,7 @@ public async Task ConfigurationBuilderTest_Generation_UserManifestDirPath_AddsMa Assert.IsNotNull(config); Assert.IsNotNull(config.ManifestDirPath); - var expectedPath = Path.Join("ManifestDirPath", Constants.ManifestFolder); + var expectedPath = Path.Join("ManifestDirPath", SpdxConstants.ManifestFolder); Assert.AreEqual(Path.GetFullPath(config.ManifestDirPath.Value), Path.GetFullPath(expectedPath)); fileSystemUtilsMock.VerifyAll(); @@ -202,7 +202,7 @@ public async Task ConfigurationBuilderTest_Generation_NSBaseUri_Validated() Assert.IsNotNull(config); Assert.IsNotNull(config.ManifestDirPath); - var expectedPath = Path.Join("ManifestDirPath", Constants.ManifestFolder); + var expectedPath = Path.Join("ManifestDirPath", SpdxConstants.ManifestFolder); Assert.AreEqual(Path.GetFullPath(config.ManifestDirPath.Value), Path.GetFullPath(expectedPath)); fileSystemUtilsMock.VerifyAll(); @@ -234,7 +234,7 @@ public async Task ConfigurationBuilderTest_Generation_BadNSBaseUriWithDefaultVal Assert.IsNotNull(config); Assert.IsNotNull(config.ManifestDirPath); - var expectedPath = Path.Join("ManifestDirPath", Constants.ManifestFolder); + var expectedPath = Path.Join("ManifestDirPath", SpdxConstants.ManifestFolder); Assert.AreEqual(Path.GetFullPath(config.ManifestDirPath.Value), Path.GetFullPath(expectedPath)); fileSystemUtilsMock.VerifyAll(); @@ -265,7 +265,7 @@ public async Task ConfigurationBuilderTest_Generation_NullNSBaseUriChangesToDefa Assert.IsNotNull(config); Assert.IsNotNull(args.ManifestDirPath); Assert.IsNotNull(config.NamespaceUriBase); - Assert.AreEqual(config.ManifestDirPath.Value, Path.Join("ManifestDirPath", Constants.ManifestFolder)); + Assert.AreEqual(config.ManifestDirPath.Value, Path.Join("ManifestDirPath", SpdxConstants.ManifestFolder)); fileSystemUtilsMock.VerifyAll(); } diff --git a/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForValidation.cs b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForValidation.cs index b9df82982..296173961 100644 --- a/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForValidation.cs +++ b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForValidation.cs @@ -11,7 +11,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using PowerArgs; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Config.Tests; @@ -149,7 +149,7 @@ public async Task ConfigurationBuilderTest_Validation_DefaultManifestDirPath_Add Assert.IsNotNull(config); Assert.IsNotNull(config.ManifestDirPath); - Assert.AreEqual(Path.Join("BuildDropPath", Constants.ManifestFolder), config.ManifestDirPath.Value); + Assert.AreEqual(Path.Join("BuildDropPath", SpdxConstants.ManifestFolder), config.ManifestDirPath.Value); fileSystemUtilsMock.VerifyAll(); } diff --git a/test/Microsoft.Sbom.Api.Tests/Entities/output/ValidationResultGeneratorTests.cs b/test/Microsoft.Sbom.Api.Tests/Entities/output/ValidationResultGeneratorTests.cs index e6121b424..c2db1a66c 100644 --- a/test/Microsoft.Sbom.Api.Tests/Entities/output/ValidationResultGeneratorTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Entities/output/ValidationResultGeneratorTests.cs @@ -5,13 +5,13 @@ using System.Collections.Concurrent; using System.Collections.Generic; using Microsoft.Sbom.Api.Tests; -using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Contracts.Enums; using Microsoft.Sbom.Extensions.Entities; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Entities.Output.Tests; @@ -270,7 +270,7 @@ private static Mock GetDefaultConfigurationMock(bool ignoreMissi configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); - configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); configurationMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "child1;child2;child3" }); configurationMock.SetupGet(c => c.ValidateSignature).Returns(new ConfigurationSetting { Value = true }); configurationMock.SetupGet(c => c.IgnoreMissing).Returns(new ConfigurationSetting { Value = ignoreMissing }); diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/ComponentToPackageInfoConverterTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/ComponentToPackageInfoConverterTests.cs index 93f9cf0b9..34e8dbd16 100644 --- a/test/Microsoft.Sbom.Api.Tests/Executors/ComponentToPackageInfoConverterTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Executors/ComponentToPackageInfoConverterTests.cs @@ -12,7 +12,6 @@ using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Manifest; using Microsoft.Sbom.Api.Tests; -using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Extensions; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -20,6 +19,7 @@ using HashAlgorithmName = Microsoft.Sbom.Contracts.Enums.AlgorithmName; using ILogger = Serilog.ILogger; using PackageInfo = Microsoft.Sbom.Contracts.SbomPackage; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Executors.Tests; @@ -40,7 +40,7 @@ public void Setup() public ComponentToPackageInfoConverterTests() { mockConfiguration.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); - mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); mockConfiguration.SetupGet(c => c.BuildComponentPath).Returns(new ConfigurationSetting { Value = "root" }); manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new TestManifestGenerator() }); diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/ExternalDocumentReferenceWriterTest.cs b/test/Microsoft.Sbom.Api.Tests/Executors/ExternalDocumentReferenceWriterTest.cs index 486bdf5c4..3c8b0e5f0 100644 --- a/test/Microsoft.Sbom.Api.Tests/Executors/ExternalDocumentReferenceWriterTest.cs +++ b/test/Microsoft.Sbom.Api.Tests/Executors/ExternalDocumentReferenceWriterTest.cs @@ -18,7 +18,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Serilog; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Tests.Executors; @@ -37,12 +37,12 @@ public async Task PassExternalDocumentReferenceInfosChannel_ReturnsJsonDocWithSe var metadataBuilder = new MetadataBuilder( mockLogger.Object, manifestGeneratorProvider, - Constants.TestManifestInfo, + SpdxConstants.TestManifestInfo, recorderMock.Object); var jsonFilePath = "/root/_manifest/manifest.json"; var sbomConfig = new SbomConfig(fileSystemUtilsMock.Object) { - ManifestInfo = Constants.TestManifestInfo, + ManifestInfo = SpdxConstants.TestManifestInfo, ManifestJsonDirPath = "/root/_manifest", ManifestJsonFilePath = jsonFilePath, MetadataBuilder = metadataBuilder, diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/FileHasherTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/FileHasherTests.cs index 032d5e09d..e3ee5ba79 100644 --- a/test/Microsoft.Sbom.Api.Tests/Executors/FileHasherTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Executors/FileHasherTests.cs @@ -21,7 +21,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Serilog; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Executors.Tests; @@ -61,14 +61,14 @@ public async Task FileHasherTest_Validate_MultipleFiles_SucceedsAsync() var manifestPathConverter = new Mock(); mockConfiguration.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); - mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); hashCodeGeneratorMock.Setup(m => m.GenerateHashes( It.IsAny(), - new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + new AlgorithmName[] { SpdxConstants.DefaultHashAlgorithmName })) .Returns(new Checksum[] { - new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = "hash" } + new Checksum { Algorithm = SpdxConstants.DefaultHashAlgorithmName, ChecksumValue = "hash" } }); manifestPathConverter.Setup(m => m.Convert(It.IsAny(), false)).Returns((string r, bool v) => (r, true)); @@ -107,17 +107,17 @@ public async Task FileHasherTest_Validate_MultipleFiles_SucceedsAsync() public async Task FileHasherTest_Validate_ManifestPathConverterThrows_ReturnsValidationFailureAsync() { mockConfiguration.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); - mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); var hashCodeGeneratorMock = new Mock(); var manifestPathConverter = new Mock(); hashCodeGeneratorMock.Setup(m => m.GenerateHashes( It.IsAny(), - new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + new AlgorithmName[] { SpdxConstants.DefaultHashAlgorithmName })) .Returns(new Checksum[] { - new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = "hash" } + new Checksum { Algorithm = SpdxConstants.DefaultHashAlgorithmName, ChecksumValue = "hash" } }); manifestPathConverter.Setup(m => m.Convert(It.IsAny(), false)).Returns((string r, bool v) => (r, true)); @@ -174,21 +174,21 @@ public async Task FileHasherTest_Validate_ManifestPathConverterThrows_ReturnsVal public async Task FileHasherTest_Validate_HashError_ReturnsValidationFailureAsync() { mockConfiguration.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); - mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); var hashCodeGeneratorMock = new Mock(); var manifestPathConverter = new Mock(); hashCodeGeneratorMock.SetupSequence(m => m.GenerateHashes( It.IsAny(), - new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + new AlgorithmName[] { SpdxConstants.DefaultHashAlgorithmName })) .Returns(new Checksum[] { - new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = "hash" } + new Checksum { Algorithm = SpdxConstants.DefaultHashAlgorithmName, ChecksumValue = "hash" } }) .Returns(new Checksum[] { - new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = string.Empty } + new Checksum { Algorithm = SpdxConstants.DefaultHashAlgorithmName, ChecksumValue = string.Empty } }) .Throws(new UnauthorizedAccessException("Can't access file")); manifestPathConverter.Setup(m => m.Convert(It.IsAny(), false)).Returns((string r, bool v) => (r, true)); @@ -249,10 +249,10 @@ public async Task FileHasherTest_Generate_MultipleFiles_SucceedsAsync() hashCodeGeneratorMock.Setup(m => m.GenerateHashes( It.IsAny(), - new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + new AlgorithmName[] { SpdxConstants.DefaultHashAlgorithmName })) .Returns(new Checksum[] { - new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = "hash" } + new Checksum { Algorithm = SpdxConstants.DefaultHashAlgorithmName, ChecksumValue = "hash" } }); var manifestInfoList = new List diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/HashValidatorTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/HashValidatorTests.cs index 2703c0d6c..c2ec7e51a 100644 --- a/test/Microsoft.Sbom.Api.Tests/Executors/HashValidatorTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Executors/HashValidatorTests.cs @@ -7,7 +7,6 @@ using System.Threading.Channels; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; -using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Contracts.Enums; @@ -15,6 +14,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using ErrorType = Microsoft.Sbom.Api.Entities.ErrorType; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Executors.Tests; @@ -37,12 +37,12 @@ public async Task HashValidatorTest_ValidHash_SucceedsAsync() } var configuration = new Mock(); - configuration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configuration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); var files = Channel.CreateUnbounded(); foreach (var file in fileList) { - await files.Writer.WriteAsync(new InternalSbomFileInfo { Path = file.ToUpper(), Checksum = new Checksum[] { new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = $"{file}_hash" } } }); + await files.Writer.WriteAsync(new InternalSbomFileInfo { Path = file.ToUpper(), Checksum = new Checksum[] { new Checksum { Algorithm = SpdxConstants.DefaultHashAlgorithmName, ChecksumValue = $"{file}_hash" } } }); } files.Writer.Complete(); @@ -76,12 +76,12 @@ public async Task HashValidatorTest_InValidHash_ReturnsValidationErrorAsync() } var configuration = new Mock(); - configuration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configuration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); var files = Channel.CreateUnbounded(); foreach (var file in fileList) { - await files.Writer.WriteAsync(new InternalSbomFileInfo { Path = file.ToUpper(), Checksum = new Checksum[] { new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = $"{file}_hash" } } }); + await files.Writer.WriteAsync(new InternalSbomFileInfo { Path = file.ToUpper(), Checksum = new Checksum[] { new Checksum { Algorithm = SpdxConstants.DefaultHashAlgorithmName, ChecksumValue = $"{file}_hash" } } }); } files.Writer.Complete(); @@ -120,18 +120,18 @@ public async Task HashValidatorTest_AdditionalFile_ReturnsAdditionalFileFailureA } var configuration = new Mock(); - configuration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configuration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); var files = Channel.CreateUnbounded(); var errors = Channel.CreateUnbounded(); foreach (var file in fileList) { - await files.Writer.WriteAsync(new InternalSbomFileInfo { Path = file.ToUpper(), Checksum = new Checksum[] { new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = $"{file}_hash" } } }); + await files.Writer.WriteAsync(new InternalSbomFileInfo { Path = file.ToUpper(), Checksum = new Checksum[] { new Checksum { Algorithm = SpdxConstants.DefaultHashAlgorithmName, ChecksumValue = $"{file}_hash" } } }); } // Additional file. - await files.Writer.WriteAsync(new InternalSbomFileInfo { Path = "TEST4", Checksum = new Checksum[] { new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = $"TEST4_hash" } } }); + await files.Writer.WriteAsync(new InternalSbomFileInfo { Path = "TEST4", Checksum = new Checksum[] { new Checksum { Algorithm = SpdxConstants.DefaultHashAlgorithmName, ChecksumValue = $"TEST4_hash" } } }); files.Writer.Complete(); errors.Writer.Complete(); diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/SPDXSBOMReaderForExternalDocumentReferenceTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/SPDXSBOMReaderForExternalDocumentReferenceTests.cs index 7635412a7..8782d2296 100644 --- a/test/Microsoft.Sbom.Api.Tests/Executors/SPDXSBOMReaderForExternalDocumentReferenceTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Executors/SPDXSBOMReaderForExternalDocumentReferenceTests.cs @@ -17,7 +17,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Serilog; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Tests.Executors; @@ -40,7 +40,7 @@ public class SPDXSBOMReaderForExternalDocumentReferenceTests public SPDXSBOMReaderForExternalDocumentReferenceTests() { mockConfiguration.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); - mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); mockConfiguration.SetupGet(c => c.BuildComponentPath).Returns(new ConfigurationSetting { Value = "root" }); manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new TestManifestGenerator() }); diff --git a/test/Microsoft.Sbom.Api.Tests/Microsoft.Sbom.Api.Tests.csproj b/test/Microsoft.Sbom.Api.Tests/Microsoft.Sbom.Api.Tests.csproj index c2c124050..07d1ddb78 100644 --- a/test/Microsoft.Sbom.Api.Tests/Microsoft.Sbom.Api.Tests.csproj +++ b/test/Microsoft.Sbom.Api.Tests/Microsoft.Sbom.Api.Tests.csproj @@ -20,6 +20,7 @@ + diff --git a/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs b/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs index c85a48cf5..81970349b 100644 --- a/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/VersionSpecificPins/Version_3_0/InterfaceConcretionTests.cs @@ -191,7 +191,7 @@ private class PinnedIJsonArrayGenerator : IJsonArrayGenerator throw new NotImplementedException(); set => throw new NotImplementedException(); } public Task> GenerateAsync() => throw new NotImplementedException(); - Task IJsonArrayGenerator.GenerateAsync() => throw new NotImplementedException(); + Task IJsonArrayGenerator.GenerateAsync() => throw new NotImplementedException(); } private class PinnedISbomRedactor : ISbomRedactor @@ -235,12 +235,12 @@ private class PinnedISbomConfig : ISbomConfig private class PinnedIMetadataBuilder : IMetadataBuilder { public string GetHeaderJsonString(IInternalMetadataProvider internalMetadataProvider) => throw new NotImplementedException(); - public bool TryGetCreationInfoJson(IInternalMetadataProvider internalMetadataProvider, out GenerationResult generationResult) => throw new NotImplementedException(); + public bool TryGetCreationInfoJson(IInternalMetadataProvider internalMetadataProvider, out Extensions.Entities.GenerationResult generationResult) => throw new NotImplementedException(); public bool TryGetExternalRefArrayHeaderName(out string headerName) => throw new NotImplementedException(); public bool TryGetFilesArrayHeaderName(out string headerName) => throw new NotImplementedException(); public bool TryGetPackageArrayHeaderName(out string headerName) => throw new NotImplementedException(); public bool TryGetRelationshipsHeaderName(out string headerName) => throw new NotImplementedException(); - public bool TryGetRootPackageJson(IInternalMetadataProvider internalMetadataProvider, out GenerationResult generationResult) => throw new NotImplementedException(); + public bool TryGetRootPackageJson(IInternalMetadataProvider internalMetadataProvider, out Extensions.Entities.GenerationResult generationResult) => throw new NotImplementedException(); } private class PinnedIManifestToolJsonSerializer : IManifestToolJsonSerializer diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs index 866932b36..a2475c009 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs @@ -18,7 +18,7 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Serilog; -using Constants = Microsoft.Sbom.Api.Utils.Constants; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Tests.Workflows.Helpers; @@ -69,11 +69,11 @@ public void Setup() metadataBuilder = new MetadataBuilder( mockLogger.Object, manifestGeneratorProvider, - Constants.TestManifestInfo, + SpdxConstants.TestManifestInfo, recorderMock.Object); sbomConfig = new SbomConfig(fileSystemUtilsMock.Object) { - ManifestInfo = Constants.TestManifestInfo, + ManifestInfo = SpdxConstants.TestManifestInfo, ManifestJsonDirPath = ManifestJsonDirPath, ManifestJsonFilePath = JsonFilePath, MetadataBuilder = metadataBuilder, diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs index 2cc196fe4..9f49a3e3b 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs @@ -42,10 +42,10 @@ using Newtonsoft.Json.Linq; using Spectre.Console; using Checksum = Microsoft.Sbom.Contracts.Checksum; -using Constants = Microsoft.Sbom.Api.Utils.Constants; using Generator30 = Microsoft.Sbom.Parsers.Spdx30SbomParser.Generator; using IComponentDetector = Microsoft.Sbom.Api.Utils.IComponentDetector; using ILogger = Serilog.ILogger; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Workflows.Tests; @@ -90,8 +90,8 @@ public async Task ManifestGenerationWorkflowTests_Succeeds(string spdxVersionFor { var manifestInfoPerSpdxVersion = new Dictionary { - { "test", Constants.TestManifestInfo }, - { "3.0", Constants.SPDX30ManifestInfo } + { "test", SpdxConstants.TestManifestInfo }, + { "3.0", SpdxConstants.SPDX30ManifestInfo } }; ManifestGeneratorProvider manifestGeneratorProvider = null; @@ -344,7 +344,7 @@ await externalDocumentReferenceChannel.Writer.WriteAsync(new ExternalDocumentRef var externalDocumentReferenceGenerator = new ExternalDocumentReferenceGenerator(mockLogger.Object, sourcesProvider, recorderMock.Object); - var generateResult = new GenerateResult(new List(), new Dictionary>()); + var generateResult = new Helpers.GenerationResult(new List(), new Dictionary>()); relationshipArrayGenerator .Setup(r => r.GenerateAsync()) .ReturnsAsync(generateResult); @@ -422,7 +422,7 @@ public async Task ManifestGenerationWorkflowTests_SBOMDirExists_Throws() mockOSUtils.Setup(o => o.GetEnvironmentVariable(It.IsAny())).Returns("false"); var sbomConfig = new SbomConfig(fileSystemMock.Object) { - ManifestInfo = Constants.TestManifestInfo, + ManifestInfo = SpdxConstants.TestManifestInfo, ManifestJsonDirPath = "/root/_manifest", ManifestJsonFilePath = "/root/_manifest/manifest.json" }; @@ -448,7 +448,7 @@ public async Task ManifestGenerationWorkflowTests_SBOMDir_NotDefault_NotDeleted( fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); var sbomConfig = new SbomConfig(fileSystemMock.Object) { - ManifestInfo = Constants.TestManifestInfo, + ManifestInfo = SpdxConstants.TestManifestInfo, ManifestJsonDirPath = "/root/_manifest", ManifestJsonFilePath = "/root/_manifest/manifest.json" }; @@ -456,7 +456,7 @@ public async Task ManifestGenerationWorkflowTests_SBOMDir_NotDefault_NotDeleted( fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); fileSystemMock.Setup(f => f.DeleteDir(It.IsAny(), true)).Verifiable(); var fileArrayGeneratorMock = new Mock>(); - var generateResult = new GenerateResult(new List(), new Dictionary>()); + var generateResult = new Helpers.GenerationResult(new List(), new Dictionary>()); fileArrayGeneratorMock.Setup(f => f.GenerateAsync()).ReturnsAsync(generateResult); var workflow = new SbomGenerationWorkflow( diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs index 0502becbf..57d225fc4 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs @@ -33,9 +33,9 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Serilog; -using Constants = Microsoft.Sbom.Api.Utils.Constants; using ErrorType = Microsoft.Sbom.Api.Entities.ErrorType; using SpdxChecksum = Microsoft.Sbom.Parsers.Spdx22SbomParser.Entities.Checksum; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Workflows; @@ -178,7 +178,7 @@ public async Task SbomParserBasedValidationWorkflowTests_ReturnsSuccessAndValida configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); - configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); configurationMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "child1;child2;child3" }); configurationMock.SetupGet(c => c.IgnoreMissing).Returns(new ConfigurationSetting { Value = true }); configurationMock.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); @@ -186,18 +186,18 @@ public async Task SbomParserBasedValidationWorkflowTests_ReturnsSuccessAndValida configurationMock.SetupGet(c => c.ValidateSignature).Returns(new ConfigurationSetting { Value = true }); configurationMock.SetupGet(c => c.ManifestInfo).Returns(new ConfigurationSetting> { - Value = new List() { Constants.SPDX22ManifestInfo } + Value = new List() { SpdxConstants.SPDX22ManifestInfo } }); ISbomConfig sbomConfig = new SbomConfig(fileSystemMock.Object) { - ManifestInfo = Constants.SPDX22ManifestInfo, + ManifestInfo = SpdxConstants.SPDX22ManifestInfo, ManifestJsonDirPath = "/root/_manifest", ManifestJsonFilePath = "/root/_manifest/spdx_2.2/manifest.spdx.json", MetadataBuilder = null, Recorder = new SbomPackageDetailsRecorder() }; - sbomConfigs.Setup(c => c.Get(Constants.SPDX22ManifestInfo)).Returns(sbomConfig); + sbomConfigs.Setup(c => c.Get(SpdxConstants.SPDX22ManifestInfo)).Returns(sbomConfig); fileSystemMock.Setup(f => f.OpenRead("/root/_manifest/spdx_2.2/manifest.spdx.json")).Returns(Stream.Null); fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) @@ -211,14 +211,14 @@ public async Task SbomParserBasedValidationWorkflowTests_ReturnsSuccessAndValida hashCodeGeneratorMock.Setup(h => h.GenerateHashes( It.IsAny(), - new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + new AlgorithmName[] { SpdxConstants.DefaultHashAlgorithmName })) .Returns((string fileName, AlgorithmName[] algos) => new Checksum[] { new Checksum { ChecksumValue = $"{fileName}hash", - Algorithm = Constants.DefaultHashAlgorithmName + Algorithm = SpdxConstants.DefaultHashAlgorithmName } }); @@ -329,7 +329,7 @@ public async Task SbomParserBasedValidationWorkflowTests_ReturnsSuccessAndValida configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); - configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); configurationMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "child1;child2;child3" }); configurationMock.SetupGet(c => c.IgnoreMissing).Returns(new ConfigurationSetting { Value = false }); configurationMock.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); @@ -337,18 +337,18 @@ public async Task SbomParserBasedValidationWorkflowTests_ReturnsSuccessAndValida configurationMock.SetupGet(c => c.ValidateSignature).Returns(new ConfigurationSetting { Value = true }); configurationMock.SetupGet(c => c.ManifestInfo).Returns(new ConfigurationSetting> { - Value = new List() { Constants.SPDX22ManifestInfo } + Value = new List() { SpdxConstants.SPDX22ManifestInfo } }); ISbomConfig sbomConfig = new SbomConfig(fileSystemMock.Object) { - ManifestInfo = Constants.SPDX22ManifestInfo, + ManifestInfo = SpdxConstants.SPDX22ManifestInfo, ManifestJsonDirPath = "/root/_manifest", ManifestJsonFilePath = "/root/_manifest/spdx_2.2/manifest.spdx.json", MetadataBuilder = null, Recorder = new SbomPackageDetailsRecorder() }; - sbomConfigs.Setup(c => c.Get(Constants.SPDX22ManifestInfo)).Returns(sbomConfig); + sbomConfigs.Setup(c => c.Get(SpdxConstants.SPDX22ManifestInfo)).Returns(sbomConfig); fileSystemMock.Setup(f => f.OpenRead("/root/_manifest/spdx_2.2/manifest.spdx.json")).Returns(Stream.Null); fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) @@ -362,20 +362,20 @@ public async Task SbomParserBasedValidationWorkflowTests_ReturnsSuccessAndValida hashCodeGeneratorMock.Setup(h => h.GenerateHashes( It.IsAny(), - new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + new AlgorithmName[] { SpdxConstants.DefaultHashAlgorithmName })) .Returns((string fileName, AlgorithmName[] algos) => new Checksum[] { new Checksum { ChecksumValue = $"{fileName}hash", - Algorithm = Constants.DefaultHashAlgorithmName + Algorithm = SpdxConstants.DefaultHashAlgorithmName } }); hashCodeGeneratorMock.Setup(h => h.GenerateHashes( It.Is(a => a == "/root/child2/grandchild1/file10"), - new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + new AlgorithmName[] { SpdxConstants.DefaultHashAlgorithmName })) .Throws(new FileNotFoundException()); var fileHasher = new FileHasher( @@ -489,7 +489,7 @@ public async Task SbomParserBasedValidationWorkflowTests_SetsComplianceStandard_ configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); - configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = SpdxConstants.DefaultHashAlgorithmName }); configurationMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "child1;child2;child3" }); configurationMock.SetupGet(c => c.IgnoreMissing).Returns(new ConfigurationSetting { Value = true }); configurationMock.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); @@ -497,20 +497,20 @@ public async Task SbomParserBasedValidationWorkflowTests_SetsComplianceStandard_ configurationMock.SetupGet(c => c.ValidateSignature).Returns(new ConfigurationSetting { Value = true }); configurationMock.SetupGet(c => c.ManifestInfo).Returns(new ConfigurationSetting> { - Value = new List() { Constants.SPDX30ManifestInfo } + Value = new List() { SpdxConstants.SPDX30ManifestInfo } }); configurationMock.SetupGet(c => c.ComplianceStandard).Returns(new ConfigurationSetting { Value = "NTIA" }); ISbomConfig sbomConfig = new SbomConfig(fileSystemMock.Object) { - ManifestInfo = Constants.SPDX30ManifestInfo, + ManifestInfo = SpdxConstants.SPDX30ManifestInfo, ManifestJsonDirPath = "/root/_manifest", ManifestJsonFilePath = "/root/_manifest/spdx_3.0/manifest.spdx.json", MetadataBuilder = null, Recorder = new SbomPackageDetailsRecorder() }; - sbomConfigs.Setup(c => c.Get(Constants.SPDX30ManifestInfo)).Returns(sbomConfig); + sbomConfigs.Setup(c => c.Get(SpdxConstants.SPDX30ManifestInfo)).Returns(sbomConfig); fileSystemMock.Setup(f => f.OpenRead("/root/_manifest/spdx_3.0/manifest.spdx.json")).Returns(Stream.Null); //fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) @@ -526,14 +526,14 @@ public async Task SbomParserBasedValidationWorkflowTests_SetsComplianceStandard_ hashCodeGeneratorMock.Setup(h => h.GenerateHashes( It.IsAny(), - new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + new AlgorithmName[] { SpdxConstants.DefaultHashAlgorithmName })) .Returns((string fileName, AlgorithmName[] algos) => new Checksum[] { new Checksum { ChecksumValue = $"{fileName}hash", - Algorithm = Constants.DefaultHashAlgorithmName + Algorithm = SpdxConstants.DefaultHashAlgorithmName } }); diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/GeneratorTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/GeneratorTests.cs index 3055ada9c..b722f9f10 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/GeneratorTests.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/GeneratorTests.cs @@ -17,8 +17,8 @@ using Microsoft.Sbom.Parsers.Spdx30SbomParser; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; -using Constants = Microsoft.Sbom.Parsers.Spdx30SbomParser.Constants; using ILogger = Serilog.ILogger; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Parser; @@ -195,12 +195,12 @@ private IInternalMetadataProvider CreateInternalMetadataProvider() { ISbomConfig sbomConfig = new SbomConfig(fileSystemMock.Object) { - ManifestInfo = Constants.Spdx30ManifestInfo, + ManifestInfo = SpdxConstants.SPDX30ManifestInfo, Recorder = new SbomPackageDetailsRecorder() }; mockConfigHandler.Setup(c => c.TryGetManifestConfig(out sbomConfig)).Returns(true); - recorderMock.Setup(r => r.RecordSBOMFormat(Constants.Spdx30ManifestInfo, It.IsAny())); + recorderMock.Setup(r => r.RecordSBOMFormat(SpdxConstants.SPDX30ManifestInfo, It.IsAny())); mockLogger.Setup(l => l.Debug(It.IsAny())); var config = new Configuration diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs index 5aee4fb7b..4956e531f 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomMetadataParserTests.cs @@ -8,8 +8,8 @@ using Microsoft.Sbom.Contracts; using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; using Microsoft.Sbom.Parser.JsonStrings; -using Microsoft.Sbom.Parsers.Spdx30SbomParser; using Microsoft.VisualStudio.TestTools.UnitTesting; +using Spdx30 = Microsoft.Sbom.Parsers.Spdx30SbomParser; namespace Microsoft.Sbom.Parser; @@ -121,8 +121,8 @@ public void StreamClosedTestReturnsNull() public void StreamEmptyTestReturnsNull() { using var stream = new MemoryStream(); - stream.Read(new byte[Constants.ReadBufferSize]); - var buffer = new byte[Constants.ReadBufferSize]; + stream.Read(new byte[Spdx30.Constants.ReadBufferSize]); + var buffer = new byte[Spdx30.Constants.ReadBufferSize]; Assert.ThrowsException(() => new SPDXParser(stream)); } From 2c3d5943d66d50f1db85c2a4930843da27f0add6 Mon Sep 17 00:00:00 2001 From: ppandrate Date: Wed, 12 Feb 2025 16:57:54 -0800 Subject: [PATCH 12/15] Clean up and refactor code, address PR comments --- .../Validators/ManifestInfoValidator.cs | 36 +++ src/Microsoft.Sbom.Api/Entities/ErrorType.cs | 5 +- .../Executors/IJsonSerializationStrategy.cs | 8 +- .../Executors/Spdx2SerializationStrategy.cs | 8 +- .../Executors/Spdx3SerializationStrategy.cs | 16 +- .../Output/MetadataBuilder.cs | 11 +- .../ExternalDocumentReferenceGenerator.cs | 16 +- .../Workflows/Helpers/FileArrayGenerator.cs | 14 +- .../Helpers/JsonDocumentCollection.cs | 24 ++ .../Helpers/PackageArrayGenerator.cs | 31 +-- .../Helpers/RelationshipsArrayGenerator.cs | 21 +- .../SBOMParserBasedValidationWorkflow.cs | 45 ++-- .../Config/Configuration.cs | 6 +- .../IManifestGenerator.cs | 24 -- .../Generator.cs | 67 +++-- .../Parser/ContextsResult.cs | 6 +- .../Parser/SPDX30Parser.cs | 246 ++++++++++-------- .../SPDXToSbomFormatConverterExtensions.cs | 2 +- .../Validators/ManifestInfoValidatorTests.cs | 43 +++ .../ManifestGenerationWorkflowTests.cs | 1 - .../SbomParserBasedValidationWorkflowTests.cs | 2 - .../Parser/SbomParserTestsBase.cs | 2 +- 22 files changed, 360 insertions(+), 274 deletions(-) create mode 100644 src/Microsoft.Sbom.Api/Config/Validators/ManifestInfoValidator.cs create mode 100644 src/Microsoft.Sbom.Api/Workflows/Helpers/JsonDocumentCollection.cs create mode 100644 test/Microsoft.Sbom.Api.Tests/Config/Validators/ManifestInfoValidatorTests.cs diff --git a/src/Microsoft.Sbom.Api/Config/Validators/ManifestInfoValidator.cs b/src/Microsoft.Sbom.Api/Config/Validators/ManifestInfoValidator.cs new file mode 100644 index 000000000..69ced2804 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Validators/ManifestInfoValidator.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config.Attributes; +using Microsoft.Sbom.Common.Config.Validators; +using Microsoft.Sbom.Constants; +using Microsoft.Sbom.Extensions.Entities; +using PowerArgs; + +namespace Microsoft.Sbom.Api.Config.Validators; + +/// +/// Validates if manifest info is valid. +/// +public class ManifestInfoValidator : ConfigValidator +{ + public ManifestInfoValidator(IAssemblyConfig assemblyConfig) + : base(typeof(ValidUriAttribute), assemblyConfig) + { + } + + public ManifestInfoValidator(Type supportedAttribute, IAssemblyConfig assemblyConfig) + : base(supportedAttribute, assemblyConfig) + { + } + + public override void ValidateInternal(string paramName, object paramValue, Attribute attribute) + { + if (paramValue is not null && !SpdxConstants.SupportedSpdxManifests.Contains(paramValue as ManifestInfo)) + { + throw new ValidationArgException($"The value of {paramName} must be a valid ManifestInfo. Supported SPDX versions include 2.2 and 3.0."); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Entities/ErrorType.cs b/src/Microsoft.Sbom.Api/Entities/ErrorType.cs index 7d05e88b7..6957f3ce7 100644 --- a/src/Microsoft.Sbom.Api/Entities/ErrorType.cs +++ b/src/Microsoft.Sbom.Api/Entities/ErrorType.cs @@ -50,5 +50,8 @@ public enum ErrorType ManifestFileSigningError = 12, [EnumMember(Value = "Invalid input file")] - InvalidInputFile = 13 + InvalidInputFile = 13, + + [EnumMember(Value = "Invalid manifest SPDX version")] + InvalidManifestVersion = 14 } diff --git a/src/Microsoft.Sbom.Api/Executors/IJsonSerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/IJsonSerializationStrategy.cs index 4751dfbec..7c9c71985 100644 --- a/src/Microsoft.Sbom.Api/Executors/IJsonSerializationStrategy.cs +++ b/src/Microsoft.Sbom.Api/Executors/IJsonSerializationStrategy.cs @@ -10,13 +10,13 @@ namespace Microsoft.Sbom.Api.Workflows.Helpers; public interface IJsonSerializationStrategy { - public void AddToFilesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config); + public void AddToFilesSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config); - public void AddToPackagesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config); + public void AddToPackagesSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config); - public bool AddToRelationshipsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config); + public bool AddToRelationshipsSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config); - public void AddToExternalDocRefsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config); + public void AddToExternalDocRefsSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config); public void AddHeadersToSbom(ISbomConfigProvider sbomConfigs) { diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs index 7e4ff6408..6d44377ab 100644 --- a/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs +++ b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs @@ -13,7 +13,7 @@ namespace Microsoft.Sbom.Api.Workflows.Helpers; /// public class Spdx2SerializationStrategy : IJsonSerializationStrategy { - public void AddToFilesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + public void AddToFilesSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config) { if (config.MetadataBuilder.TryGetFilesArrayHeaderName(out var headerName)) { @@ -22,7 +22,7 @@ public void AddToFilesSupportingConfig(ref IList elementsSupporting } } - public void AddToPackagesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + public void AddToPackagesSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config) { if (config.MetadataBuilder.TryGetPackageArrayHeaderName(out var headerName)) { @@ -31,7 +31,7 @@ public void AddToPackagesSupportingConfig(ref IList elementsSupport } } - public bool AddToRelationshipsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + public bool AddToRelationshipsSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config) { if (config.MetadataBuilder.TryGetRelationshipsHeaderName(out var headerName)) { @@ -42,7 +42,7 @@ public bool AddToRelationshipsSupportingConfig(ref IList elementsSu return false; } - public void AddToExternalDocRefsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + public void AddToExternalDocRefsSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config) { if (config.MetadataBuilder.TryGetExternalRefArrayHeaderName(out var headerName)) { diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs index 72eebc98a..41d541084 100644 --- a/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs +++ b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs @@ -16,22 +16,22 @@ namespace Microsoft.Sbom.Api.Workflows.Helpers; /// public class Spdx3SerializationStrategy : IJsonSerializationStrategy { - public void AddToFilesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + public void AddToFilesSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config) { elementsSupportingConfigs.Add(config); } - public void AddToPackagesSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + public void AddToPackagesSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config) { elementsSupportingConfigs.Add(config); } - public bool AddToRelationshipsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + public bool AddToRelationshipsSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config) { return true; } - public void AddToExternalDocRefsSupportingConfig(ref IList elementsSupportingConfigs, ISbomConfig config) + public void AddToExternalDocRefsSupportingConfig(IList elementsSupportingConfigs, ISbomConfig config) { elementsSupportingConfigs.Add(config); } @@ -101,14 +101,14 @@ private void WriteElementsToSbom(GenerationResult generateResult) { var spdxId = spdxIdField.GetString(); - if (!elementsSpdxIdList.TryGetValue(spdxId, out _)) + if (elementsSpdxIdList.TryGetValue(spdxId, out _)) { - serializer.Write(element); - elementsSpdxIdList.Add(spdxId); + Console.WriteLine($"Duplicate element with SPDX ID {spdxId} found. Skipping."); } else { - Console.WriteLine($"Duplicate element with SPDX ID {spdxId} found. Skipping."); + serializer.Write(element); + elementsSpdxIdList.Add(spdxId); } } } diff --git a/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs b/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs index 1d39a7313..38c90acf6 100644 --- a/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs +++ b/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs @@ -8,6 +8,7 @@ using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Extensions; using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Parsers.Spdx22SbomParser; using Serilog; namespace Microsoft.Sbom.Api.Output; @@ -54,7 +55,7 @@ public bool TryGetFilesArrayHeaderName(out string headerName) { try { - headerName = manifestGenerator.FilesArrayHeaderName; + headerName = ((Generator)manifestGenerator).FilesArrayHeaderName; return true; } catch (NotSupportedException) @@ -69,7 +70,7 @@ public bool TryGetPackageArrayHeaderName(out string headerName) { try { - headerName = manifestGenerator.PackagesArrayHeaderName; + headerName = ((Generator)manifestGenerator).PackagesArrayHeaderName; return true; } catch (NotSupportedException) @@ -84,7 +85,7 @@ public bool TryGetExternalRefArrayHeaderName(out string headerName) { try { - headerName = manifestGenerator.ExternalDocumentRefArrayHeaderName; + headerName = ((Generator)manifestGenerator).ExternalDocumentRefArrayHeaderName; return true; } catch (NotSupportedException) @@ -134,7 +135,7 @@ public bool TryGetCreationInfoJson(IInternalMetadataProvider internalMetadataPro catch (NotSupportedException) { generationResult = null; - logger.Warning("Root package serialization not supported on this SBOM format."); + logger.Warning("Root package serialization is not supported on this SBOM format."); return false; } } @@ -143,7 +144,7 @@ public bool TryGetRelationshipsHeaderName(out string headerName) { try { - headerName = manifestGenerator.RelationshipsArrayHeaderName; + headerName = ((Generator)manifestGenerator).RelationshipsArrayHeaderName; return headerName != null; } catch (NotSupportedException) diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs index 6ded20cf0..99d8a11d2 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text.Json; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Output.Telemetry; @@ -45,20 +44,20 @@ public async Task GenerateAsync() using (recorder.TraceEvent(Events.ExternalDocumentReferenceGeneration)) { var totalErrors = new List(); - var serializersToJsonDocs = new Dictionary>(); + var jsonDocumentCollection = new JsonDocumentCollection(); var sourcesProviders = this.sourcesProviders .Where(s => s.IsSupported(ProviderType.ExternalDocumentReference)); if (!sourcesProviders.Any()) { log.Debug($"No source providers found for {ProviderType.ExternalDocumentReference}"); - return new GenerationResult(totalErrors, serializersToJsonDocs); + return new GenerationResult(totalErrors, jsonDocumentCollection.SerializersToJson); } // Write the start of the array, if supported. IList externalRefArraySupportingConfigs = new List(); var serializationStrategy = JsonSerializationStrategyFactory.GetStrategy(SpdxManifestVersion); - serializationStrategy.AddToExternalDocRefsSupportingConfig(ref externalRefArraySupportingConfigs, this.SbomConfig); + serializationStrategy.AddToExternalDocRefsSupportingConfig(externalRefArraySupportingConfigs, this.SbomConfig); foreach (var sourcesProvider in sourcesProviders) { @@ -69,12 +68,7 @@ public async Task GenerateAsync() await foreach (var jsonResults in jsonDocResults.ReadAllAsync()) { - if (!serializersToJsonDocs.ContainsKey(jsonResults.Serializer)) - { - serializersToJsonDocs[jsonResults.Serializer] = new List(); - } - - serializersToJsonDocs[jsonResults.Serializer].Add(jsonResults.Document); + jsonDocumentCollection.AddJsonDocument(jsonResults.Serializer, jsonResults.Document); totalJsonDocumentsWritten++; } @@ -86,7 +80,7 @@ public async Task GenerateAsync() } } - return new GenerationResult(totalErrors, serializersToJsonDocs); + return new GenerationResult(totalErrors, jsonDocumentCollection.SerializersToJson); } } } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs index 00e2aae89..67aca5860 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text.Json; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Output.Telemetry; @@ -60,23 +59,18 @@ public async Task GenerateAsync() // Write the start of the array, if supported. IList filesArraySupportingSBOMs = new List(); var serializationStrategy = JsonSerializationStrategyFactory.GetStrategy(SpdxManifestVersion); - serializationStrategy.AddToFilesSupportingConfig(ref filesArraySupportingSBOMs, this.SbomConfig); + serializationStrategy.AddToFilesSupportingConfig(filesArraySupportingSBOMs, this.SbomConfig); this.logger.Verbose("Started writing files array for {configFile}.", this.SbomConfig.ManifestJsonFilePath); - var serializersToJsonDocs = new Dictionary>(); + var jsonDocumentCollection = new JsonDocumentCollection(); foreach (var sourcesProvider in sourcesProviders) { var (jsondDocResults, errors) = sourcesProvider.Get(filesArraySupportingSBOMs); await foreach (var jsonResults in jsondDocResults.ReadAllAsync()) { - if (!serializersToJsonDocs.ContainsKey(jsonResults.Serializer)) - { - serializersToJsonDocs[jsonResults.Serializer] = new List(); - } - - serializersToJsonDocs[jsonResults.Serializer].Add(jsonResults.Document); + jsonDocumentCollection.AddJsonDocument(jsonResults.Serializer, jsonResults.Document); } await foreach (var error in errors.ReadAllAsync()) @@ -89,7 +83,7 @@ public async Task GenerateAsync() } } - return new GenerationResult(totalErrors, serializersToJsonDocs); + return new GenerationResult(totalErrors, jsonDocumentCollection.SerializersToJson); } } } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/JsonDocumentCollection.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/JsonDocumentCollection.cs new file mode 100644 index 000000000..c18f6b275 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/JsonDocumentCollection.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Text.Json; + +namespace Microsoft.Sbom.Api.Workflows.Helpers; + +public class JsonDocumentCollection +{ + public Dictionary> SerializersToJson { get; } + + public void AddJsonDocument(T key, JsonDocument document) + { + if (SerializersToJson.TryGetValue(key, out var jsonDocuments)) + { + jsonDocuments.Add(document); + } + else + { + SerializersToJson.Add(key, new List { document }); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs index 31f3b5e9e..5a2d752c9 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text.Json; using System.Threading.Tasks; using Microsoft.Sbom.Api.Entities; using Microsoft.Sbom.Api.Output.Telemetry; @@ -56,22 +55,17 @@ public async Task GenerateAsync() // Write the start of the array, if supported. IList packagesArraySupportingConfigs = new List(); var serializationStrategy = JsonSerializationStrategyFactory.GetStrategy(SpdxManifestVersion); - serializationStrategy.AddToPackagesSupportingConfig(ref packagesArraySupportingConfigs, this.SbomConfig); + serializationStrategy.AddToPackagesSupportingConfig(packagesArraySupportingConfigs, this.SbomConfig); var (jsonDocResults, errors) = sourcesProvider.Get(packagesArraySupportingConfigs); - // 6. Collect all the json elements to be written to the serializer. + // Collect all the json elements to be written to the serializer. var totalJsonDocumentsWritten = 0; - var serializersToJsonDocs = new Dictionary>(); + var jsonDocumentCollection = new JsonDocumentCollection(); await foreach (var jsonDocResult in jsonDocResults.ReadAllAsync()) { - if (!serializersToJsonDocs.ContainsKey(jsonDocResult.Serializer)) - { - serializersToJsonDocs[jsonDocResult.Serializer] = new List(); - } - - serializersToJsonDocs[jsonDocResult.Serializer].Add(jsonDocResult.Document); + jsonDocumentCollection.AddJsonDocument(jsonDocResult.Serializer, jsonDocResult.Document); totalJsonDocumentsWritten++; } @@ -94,13 +88,7 @@ public async Task GenerateAsync() // Write the root package information to SBOM. if (sbomConfig.MetadataBuilder.TryGetRootPackageJson(sbomConfigs, out var generationResult)) { - if (!serializersToJsonDocs.ContainsKey(sbomConfig.JsonSerializer)) - { - serializersToJsonDocs[sbomConfig.JsonSerializer] = new List(); - } - - serializersToJsonDocs[sbomConfig.JsonSerializer].Add(generationResult?.Document); - + jsonDocumentCollection.AddJsonDocument(sbomConfig.JsonSerializer, generationResult?.Document); sbomConfig.Recorder.RecordRootPackageId(generationResult?.ResultMetadata?.EntityId); sbomConfig.Recorder.RecordDocumentId(generationResult?.ResultMetadata?.DocumentId); } @@ -108,16 +96,11 @@ public async Task GenerateAsync() // Write creation info to SBOM. Creation info element is only applicable for SPDX 3.0 and above. if (sbomConfig.MetadataBuilder.TryGetCreationInfoJson(sbomConfigs, out generationResult)) { - if (!serializersToJsonDocs.ContainsKey(sbomConfig.JsonSerializer)) - { - serializersToJsonDocs[sbomConfig.JsonSerializer] = new List(); - } - - serializersToJsonDocs[sbomConfig.JsonSerializer].Add(generationResult?.Document); + jsonDocumentCollection.AddJsonDocument(sbomConfig.JsonSerializer, generationResult?.Document); } } - return new GenerationResult(totalErrors, serializersToJsonDocs); + return new GenerationResult(totalErrors, jsonDocumentCollection.SerializersToJson); } } } diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs index 934c5223d..be2b579ca 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs @@ -28,6 +28,8 @@ public class RelationshipsArrayGenerator : IJsonArrayGenerator GenerateAsync() @@ -49,17 +52,14 @@ public async Task GenerateAsync() using (recorder.TraceEvent(Events.RelationshipsGeneration)) { var totalErrors = new List(); - var serializersToJsonDocs = new Dictionary>(); + var jsonDocumentCollection = new JsonDocumentCollection(); IList relationshipsArraySupportingConfigs = new List(); var serializationStrategy = JsonSerializationStrategyFactory.GetStrategy(SpdxManifestVersion); // Write the relationship array only if supported - if (serializationStrategy.AddToRelationshipsSupportingConfig(ref relationshipsArraySupportingConfigs, this.SbomConfig)) + if (serializationStrategy.AddToRelationshipsSupportingConfig(relationshipsArraySupportingConfigs, this.SbomConfig)) { - // Get generation data - var generationData = this.SbomConfig.Recorder.GetGenerationData(); - var jsonChannelsArray = new ChannelReader[] { // Packages relationships @@ -100,18 +100,13 @@ public async Task GenerateAsync() await foreach (var jsonDoc in channelUtils.Merge(jsonChannelsArray).ReadAllAsync()) { count++; - if (!serializersToJsonDocs.ContainsKey(this.SbomConfig.JsonSerializer)) - { - serializersToJsonDocs[this.SbomConfig.JsonSerializer] = new List(); - } - - serializersToJsonDocs[this.SbomConfig.JsonSerializer].Add(jsonDoc); + jsonDocumentCollection.AddJsonDocument(this.SbomConfig.JsonSerializer, jsonDoc); } log.Debug($"Wrote {count} relationship elements in the SBOM."); } - return new GenerationResult(totalErrors, serializersToJsonDocs); + return new GenerationResult(totalErrors, jsonDocumentCollection.SerializersToJson); } } @@ -179,6 +174,4 @@ private IEnumerator GetRelationships(RelationshipType relationship } } } - - Task IJsonArrayGenerator.GenerateAsync() => throw new System.NotImplementedException(); } diff --git a/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs index 5f5d5886c..63db87bdd 100644 --- a/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs +++ b/src/Microsoft.Sbom.Api/Workflows/SBOMParserBasedValidationWorkflow.cs @@ -97,20 +97,7 @@ public async Task RunAsync() } } - // Validate compliance standard for SPDX 3.0 parsers and above - if (!string.IsNullOrEmpty(configuration.ComplianceStandard?.Value) && Convert.ToDouble(sbomConfig.ManifestInfo.Version) >= 3.0) - { - var complianceStandard = configuration.ComplianceStandard.Value; - try - { - ((SPDX30Parser)sbomParser).RequiredComplianceStandard = complianceStandard; - } - catch (Exception e) - { - recorder.RecordException(e); - log.Error($"Unable to use the given compliance standard {complianceStandard} to parse the SBOM."); - } - } + SetComplianceStandard(sbomConfig.ManifestInfo.Version, sbomParser); var successfullyValidatedFiles = 0; List fileValidationFailures = null; @@ -126,7 +113,7 @@ public async Task RunAsync() { case FilesResult filesResult: (successfullyValidatedFiles, fileValidationFailures) = await filesValidator.Validate(filesResult.Files); - ValidateInvalidInputFiles(fileValidationFailures); + ThrowOnInvalidInputFiles(fileValidationFailures); break; case PackagesResult packagesResult: var packages = packagesResult.Packages.ToList(); @@ -146,7 +133,7 @@ public async Task RunAsync() totalNumberOfPackages = elementsResult.PackagesCount; (successfullyValidatedFiles, fileValidationFailures) = await filesValidator.Validate(elementsResult.Files); - ValidateInvalidInputFiles(fileValidationFailures); + ThrowOnInvalidInputFiles(fileValidationFailures); break; default: break; @@ -300,7 +287,7 @@ private void LogResultsSummary(ValidationResult validationResultOutput, IEnumera Console.WriteLine($"Unknown file failures . . . . . . . . . . . . . {validFailures.Count(v => v.ErrorType == ErrorType.Other)}"); } - private void ValidateInvalidInputFiles(List fileValidationFailures) + private void ThrowOnInvalidInputFiles(List fileValidationFailures) { var invalidInputFiles = fileValidationFailures.Where(f => f.ErrorType == ErrorType.InvalidInputFile).ToList(); if (invalidInputFiles.Count != 0) @@ -308,4 +295,28 @@ private void ValidateInvalidInputFiles(List fileValidation throw new InvalidDataException($"Your manifest file is malformed. {invalidInputFiles.First().Path}"); } } + + /// + /// Set compliance standard for SPDX 3.0 parsers and above. + /// + /// + private void SetComplianceStandard(string spdxVersion, ISbomParser sbomParser) + { + // Note that spdxVersion is already validated to be a valid double in ConfigValidator. + var spdxVersionAsDouble = Convert.ToDouble(spdxVersion); + + if (!string.IsNullOrEmpty(configuration.ComplianceStandard?.Value) && spdxVersionAsDouble >= 3.0) + { + var complianceStandard = configuration.ComplianceStandard.Value; + try + { + (sbomParser as SPDX30Parser).RequiredComplianceStandard = complianceStandard; + } + catch (Exception e) + { + recorder.RecordException(e); + log.Error($"Unable to use the given compliance standard {complianceStandard} to parse the SBOM."); + } + } + } } diff --git a/src/Microsoft.Sbom.Common/Config/Configuration.cs b/src/Microsoft.Sbom.Common/Config/Configuration.cs index f5d279184..ed8c55eb4 100644 --- a/src/Microsoft.Sbom.Common/Config/Configuration.cs +++ b/src/Microsoft.Sbom.Common/Config/Configuration.cs @@ -54,7 +54,7 @@ public class Configuration : IConfiguration2 private static readonly AsyncLocal> verbosity = new(); private static readonly AsyncLocal> sbomPath = new(); private static readonly AsyncLocal> sbomDir = new(); - private static readonly AsyncLocal> complianceStandard = new(); + private static readonly AsyncLocal> complianceStandard = new(); /// [DirectoryExists] @@ -342,11 +342,9 @@ public ConfigurationSetting SbomDir } /// - public ConfigurationSetting ComplianceStandard + public ConfigurationSetting ComplianceStandard { get => complianceStandard.Value; set => complianceStandard.Value = value; } - - ConfigurationSetting IConfiguration.ComplianceStandard { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } } diff --git a/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs b/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs index 00ea35cd0..82573d0e2 100644 --- a/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs +++ b/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs @@ -104,30 +104,6 @@ public interface IManifestGenerator /// string Version { get; } - /// - /// Gets the value of the header to use where the files section of the SBOM will be placed. - /// If this is not supported, this method should throw a . - /// - string FilesArrayHeaderName { get; } - - /// - /// Gets the value of the header to use where the packages section of the SBOM will be placed. - /// If this is not supported, this method should throw a . - /// - string PackagesArrayHeaderName { get; } - - /// - /// Gets the value of the header where the relationship data about this SBOM will be placed. - /// If this is not supported, this method should throw a . - /// - string RelationshipsArrayHeaderName { get; } - - /// - /// Gets the value of the header where the external document reference data about this SBOM will be placed. - /// If this is not supported, this method should throw a . - /// - string ExternalDocumentRefArrayHeaderName { get; } - /// /// Return a dictionary of items that need to be added to the header of /// the generated SBOM. diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs index 131857539..0f96fd19e 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs @@ -33,14 +33,6 @@ public class Generator : IManifestGenerator public string Version { get; set; } = string.Join("-", Constants.SPDXName, Constants.SPDXVersion); - string IManifestGenerator.FilesArrayHeaderName => throw new NotSupportedException(); - - string IManifestGenerator.PackagesArrayHeaderName => throw new NotSupportedException(); - - string IManifestGenerator.RelationshipsArrayHeaderName => throw new NotSupportedException(); - - string IManifestGenerator.ExternalDocumentRefArrayHeaderName => throw new NotSupportedException(); - private JsonSerializerOptions serializerOptions = new JsonSerializerOptions { Converters = { new ElementSerializer() }, @@ -321,6 +313,29 @@ public GenerationResult GenerateJsonDocument(SbomEntities.Relationship relations }; } + public IDictionary GetMetadataDictionary(IInternalMetadataProvider internalMetadataProvider) + { + if (internalMetadataProvider is null) + { + throw new ArgumentNullException(nameof(internalMetadataProvider)); + } + + var generationData = internalMetadataProvider.GetGenerationData(SpdxConstants.SPDX30ManifestInfo); + + var (sbomToolName, sbomToolVersion, packageName, packageVersion, documentName, creationInfo) = GetCommonMetadata(internalMetadataProvider); + + return new Dictionary + { + { Constants.SPDXVersionHeaderName, Version }, + { Constants.DataLicenseHeaderName, Constants.DataLicenceValue }, + { Constants.SPDXIDHeaderName, Constants.SPDXDocumentIdValue }, + { Constants.DocumentNameHeaderName, documentName }, + { Constants.DocumentNamespaceHeaderName, internalMetadataProvider.GetDocumentNamespace() }, + { Constants.CreationInfoHeaderName, creationInfo }, + { Constants.DocumentDescribesHeaderName, new string[] { generationData.RootPackageId } } + }; + } + /// /// Generate all SPDX elements related to document creation. /// @@ -336,14 +351,10 @@ public GenerationResult GenerateJsonDocument(IInternalMetadataProvider internalM var generationData = internalMetadataProvider.GetGenerationData(SpdxConstants.SPDX30ManifestInfo); - var sbomToolName = internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolName); - var sbomToolVersion = internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolVersion); - var packageName = internalMetadataProvider.GetPackageName(); - var packageVersion = internalMetadataProvider.GetPackageVersion(); + var (sbomToolName, sbomToolVersion, packageName, packageVersion, documentName, creationInfo) = GetCommonMetadata(internalMetadataProvider); var orgName = internalMetadataProvider.GetPackageSupplier(); var toolName = sbomToolName + "-" + sbomToolVersion; - var documentName = string.Format(Constants.SPDXDocumentNameFormatString, packageName, packageVersion); var spdxOrganization = new Organization { @@ -408,6 +419,8 @@ public GenerationResult GenerateJsonDocument(IInternalMetadataProvider internalM }; } + public ManifestInfo RegisterManifest() => SpdxConstants.SPDX30ManifestInfo; + /// /// Use file info to generate file and relationship spdx elements. /// @@ -617,19 +630,10 @@ private PackageVerificationCode GetPackageVerificationCode(IInternalMetadataProv return packageVerificationCode; } - public ManifestInfo RegisterManifest() => SpdxConstants.SPDX30ManifestInfo; - - public IDictionary GetMetadataDictionary(IInternalMetadataProvider internalMetadataProvider) + private (string sbomToolName, string sbomToolVersion, string packageName, string packageVersion, string documentName, CreationInfo creationInfo) GetCommonMetadata(IInternalMetadataProvider internalMetadataProvider) { - if (internalMetadataProvider is null) - { - throw new ArgumentNullException(nameof(internalMetadataProvider)); - } - - var generationData = internalMetadataProvider.GetGenerationData(SpdxConstants.SPDX30ManifestInfo); - - var sbomToolName = internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolName); - var sbomToolVersion = internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolVersion); + var sbomToolName = (string)internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolName); + var sbomToolVersion = (string)internalMetadataProvider.GetMetadata(MetadataKey.SBOMToolVersion); var packageName = internalMetadataProvider.GetPackageName(); var packageVersion = internalMetadataProvider.GetPackageVersion(); @@ -640,7 +644,7 @@ public IDictionary GetMetadataDictionary(IInternalMetadataProvid Created = internalMetadataProvider.GetGenerationTimestamp(), CreatedBy = new List { - $"{internalMetadataProvider.GetPackageSupplier()}", + internalMetadataProvider.GetPackageSupplier(), }, CreatedUsing = new List { @@ -648,15 +652,6 @@ public IDictionary GetMetadataDictionary(IInternalMetadataProvid } }; - return new Dictionary - { - { Constants.SPDXVersionHeaderName, Version }, - { Constants.DataLicenseHeaderName, Constants.DataLicenceValue }, - { Constants.SPDXIDHeaderName, Constants.SPDXDocumentIdValue }, - { Constants.DocumentNameHeaderName, documentName }, - { Constants.DocumentNamespaceHeaderName, internalMetadataProvider.GetDocumentNamespace() }, - { Constants.CreationInfoHeaderName, creationInfo }, - { Constants.DocumentDescribesHeaderName, new string[] { generationData.RootPackageId } } - }; + return (sbomToolName, sbomToolVersion, packageName, packageVersion, documentName, creationInfo); } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs index 0e79b51db..1f7d5112d 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/ContextsResult.cs @@ -8,11 +8,11 @@ namespace Microsoft.Sbom.Parser; public record ContextsResult : ParserStateResult { - public ContextsResult(ParserStateResult result, List jsonList) + public ContextsResult(ParserStateResult result, List contexts) : base(result.FieldName, result.Result, result.ExplicitField, result.YieldReturn) { - Contexts = jsonList; + Contexts = contexts; } - public IEnumerable Contexts { get; set; } + public IEnumerable Contexts { get; set; } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs index c0ce167a8..2f2ea0403 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs @@ -60,7 +60,11 @@ public SPDX30Parser( { this.jsonSerializerOptions = jsonSerializerOptions ?? new JsonSerializerOptions { - Converters = { new ElementSerializer(), new JsonStringEnumConverter() }, + Converters = + { + new ElementSerializer(), + new JsonStringEnumConverter() + }, }; var handlers = new Dictionary @@ -69,28 +73,7 @@ public SPDX30Parser( { GraphProperty, new PropertyHandler(ParameterType.Array) }, }; - if (!string.IsNullOrEmpty(this.RequiredComplianceStandard)) - { - if (!Enum.TryParse(this.RequiredComplianceStandard, true, out var complianceStandardAsEnum)) - { - throw new ParserException($"{this.RequiredComplianceStandard} compliance standard is not supported."); - } - else - { - switch (complianceStandardAsEnum) - { - case ComplianceStandard.NTIA: - this.EntitiesToEnforceComplianceStandardsFor = this.entitiesWithDifferentNTIARequirements; - break; - default: - throw new ParserException($"{this.RequiredComplianceStandard} compliance standard is not supported."); - } - } - } - else - { - Console.WriteLine("No required compliance standard."); - } + (_, this.EntitiesToEnforceComplianceStandardsFor) = GetEntitiesToEnforceComplianceStandard(this.RequiredComplianceStandard); if (bufferSize is null) { @@ -116,7 +99,7 @@ public SPDX30Parser( { result = parser.Next(); - if (result is not null && result.Result is not null) + if (result?.Result is not null) { var fieldName = result.FieldName; this.observedFieldNames.Add(fieldName); @@ -125,12 +108,10 @@ public SPDX30Parser( switch (fieldName) { case ContextProperty: - var contextResult = new ContextsResult(result, jsonList); - ValidateContext(contextResult); - result = contextResult; + result = ConvertToContexts(jsonList, result); break; case GraphProperty: - var elementsResult = ConvertToElements(jsonList, ref result, this.RequiredComplianceStandard, this.EntitiesToEnforceComplianceStandardsFor); + var elementsResult = ConvertToElements(jsonList, result); this.Metadata = this.SetMetadata(elementsResult); result = elementsResult; break; @@ -158,11 +139,38 @@ public SPDX30Parser( return null; } + public SpdxMetadata GetMetadata() + { + if (!this.parsingComplete) + { + throw new ParserException($"{nameof(this.GetMetadata)} can only be called after Parsing is complete to ensure that a whole object is returned."); + } + + // TODO: Eventually this return type should be changed to SpdxMetadata to be consistent with naming. + return this.Metadata; + } + + public ManifestInfo[] RegisterManifest() => new ManifestInfo[] { SpdxConstants.SPDX30ManifestInfo }; + + private ContextsResult ConvertToContexts(List? jsonList, ParserStateResult? result) + { + if (jsonList != null && jsonList.All(e => e is string)) + { + var contextsResult = new ContextsResult(result, jsonList.Cast().ToList()); + ValidateContext(contextsResult); + return contextsResult; + } + else + { + throw new InvalidDataException("The context property must be a list of strings."); + } + } + private void ValidateContext(ContextsResult result) { if (result.Contexts == null || !result.Contexts.Any() || result.Contexts.Count() > 1) { - throw new ParserException($"The context property is either empty or has more than one string."); + throw new ParserException($"The context property is invalid. It should only have one string."); } } @@ -171,7 +179,7 @@ private void ValidateContext(ContextsResult result) /// /// /// - public ElementsResult ConvertToElements(List? jsonList, ref ParserStateResult? result, string? requiredComplianceStandard, IReadOnlyCollection? entitiesWithDifferentNTIARequirements) + private ElementsResult ConvertToElements(List? jsonList, ParserStateResult? result) { var elementsResult = new ElementsResult(result); var elementsList = new List(); @@ -188,63 +196,22 @@ public ElementsResult ConvertToElements(List? jsonList, ref ParserStateR return elementsResult; } - var complianceStandardAsEnum = ComplianceStandard.None; - if (!string.IsNullOrEmpty(this.RequiredComplianceStandard)) - { - if (!Enum.TryParse(this.RequiredComplianceStandard, true, out complianceStandardAsEnum)) - { - throw new ParserException($"{this.RequiredComplianceStandard} compliance standard is not supported."); - } - else - { - switch (complianceStandardAsEnum) - { - case ComplianceStandard.NTIA: - this.EntitiesToEnforceComplianceStandardsFor = this.entitiesWithDifferentNTIARequirements; - break; - default: - throw new ParserException($"{this.RequiredComplianceStandard} compliance standard is not supported."); - } - } - } - else - { - Console.WriteLine("No required compliance standard."); - } + // Default to no compliance standard. + (var complianceStandardAsEnum, this.EntitiesToEnforceComplianceStandardsFor) = GetEntitiesToEnforceComplianceStandard(this.RequiredComplianceStandard); foreach (JsonObject jsonObject in jsonList) { - if (jsonObject == null || !jsonObject.Any()) - { - continue; - } - - var entityType = GetEntityType(jsonObject, complianceStandardAsEnum); - - object? deserializedObject = null; - try - { - deserializedObject = JsonSerializer.Deserialize(jsonObject.ToString(), entityType, jsonSerializerOptions); - } - catch (Exception e) - { - throw new ParserException(e.Message); - } + var parsedJsonObject = ParseJsonObject(jsonObject, complianceStandardAsEnum); + var entityType = parsedJsonObject?.GetType(); - if (deserializedObject != null) + if (parsedJsonObject is not null) { - var deserializedElement = (Element)deserializedObject; - - // Deduplication of elements by checking SPDX ID + var deserializedElement = (Element)parsedJsonObject; var spdxId = deserializedElement.SpdxId; - if (!elementsSpdxIdList.TryGetValue(spdxId, out _)) + + if (IsUniqueElement(spdxId, elementsSpdxIdList)) { elementsList.Add(deserializedElement); - elementsSpdxIdList.Add(spdxId); - } - else - { - Console.WriteLine($"Duplicate element with SPDX ID {spdxId} found. Skipping."); } switch (entityType?.Name) @@ -269,13 +236,7 @@ public ElementsResult ConvertToElements(List? jsonList, ref ParserStateR } } - // Validate if elements meet required compliance standards - switch (complianceStandardAsEnum) - { - case ComplianceStandard.NTIA: - ValidateNTIARequirements(elementsList); - break; - } + ValidateElementsBasedOnComplianceStandard(complianceStandardAsEnum, elementsList); elementsResult.Elements = elementsList; elementsResult.Files = filesList; @@ -283,7 +244,7 @@ public ElementsResult ConvertToElements(List? jsonList, ref ParserStateR return elementsResult; } - public Type GetEntityType(JsonObject jsonObject, ComplianceStandard? requiredComplianceStandard) + private Type GetEntityType(JsonObject jsonObject, ComplianceStandard? requiredComplianceStandard) { var assembly = typeof(Element).Assembly; var typeFromSbom = jsonObject["type"]?.ToString(); @@ -317,7 +278,7 @@ public Type GetEntityType(JsonObject jsonObject, ComplianceStandard? requiredCom return type; } - public void ValidateNTIARequirements(List elementsList) + private void ValidateNTIARequirements(List elementsList) { ValidateSbomDocCreationForNTIA(elementsList); ValidateSbomFilesForNTIA(elementsList); @@ -329,10 +290,10 @@ public void ValidateNTIARequirements(List elementsList) /// /// /// - public void ValidateSbomDocCreationForNTIA(List elementsList) + private void ValidateSbomDocCreationForNTIA(List elementsList) { - var spdxDocumentElements = elementsList.Where(element => element is SpdxDocument); - if (spdxDocumentElements.Count() != 1) + var spdxDocumentElements = elementsList.Where(element => element is SpdxDocument).ToList(); + if (spdxDocumentElements.Count != 1) { throw new ParserException("SBOM document is not NTIA compliant because it must only contain one SpdxDocument element."); } @@ -350,7 +311,7 @@ public void ValidateSbomDocCreationForNTIA(List elementsList) /// /// /// - public void ValidateSbomFilesForNTIA(List elementsList) + private void ValidateSbomFilesForNTIA(List elementsList) { var fileElements = elementsList.Where(element => element is NTIAFile); foreach (var fileElement in fileElements) @@ -358,8 +319,7 @@ public void ValidateSbomFilesForNTIA(List elementsList) var fileSpdxId = fileElement.SpdxId; var fileHasSha256Hash = fileElement.VerifiedUsing. - Any(packageVerificationCode => packageVerificationCode.Algorithm == - HashAlgorithm.sha256); + Any(packageVerificationCode => packageVerificationCode.Algorithm == HashAlgorithm.sha256); if (!fileHasSha256Hash) { @@ -373,7 +333,7 @@ public void ValidateSbomFilesForNTIA(List elementsList) /// /// /// - public void ValidateSbomPackagesForNTIA(List elementsList) + private void ValidateSbomPackagesForNTIA(List elementsList) { var packageElements = elementsList.Where(element => element is Package); foreach (var packageElement in packageElements) @@ -381,8 +341,7 @@ public void ValidateSbomPackagesForNTIA(List elementsList) var packageSpdxId = packageElement.SpdxId; var packageHasSha256Hash = packageElement.VerifiedUsing. - Any(packageVerificationCode => packageVerificationCode.Algorithm == - HashAlgorithm.sha256); + Any(packageVerificationCode => packageVerificationCode.Algorithm == HashAlgorithm.sha256); if (!packageHasSha256Hash) { @@ -395,7 +354,7 @@ public void ValidateSbomPackagesForNTIA(List elementsList) /// Sets metadata based on parsed SBOM elements. /// /// - public SpdxMetadata SetMetadata(ElementsResult result) + private SpdxMetadata SetMetadata(ElementsResult result) { var metadata = new SpdxMetadata(); var spdxDocumentElement = (SpdxDocument?)result.Elements.FirstOrDefault(element => element.Type == "SpdxDocument"); @@ -450,16 +409,95 @@ public SpdxMetadata SetMetadata(ElementsResult result) return metadata; } - public SpdxMetadata GetMetadata() + private (ComplianceStandard, IReadOnlyCollection) GetEntitiesToEnforceComplianceStandard(string? requiredComplianceStandard) { - if (!this.parsingComplete) + if (!string.IsNullOrEmpty(requiredComplianceStandard)) { - throw new ParserException($"{nameof(this.GetMetadata)} can only be called after Parsing is complete to ensure that a whole object is returned."); + if (!Enum.TryParse(requiredComplianceStandard, true, out var complianceStandardAsEnum)) + { + throw new ParserException($"{requiredComplianceStandard} compliance standard is not supported."); + } + else + { + switch (complianceStandardAsEnum) + { + case ComplianceStandard.NTIA: + return (complianceStandardAsEnum, this.entitiesWithDifferentNTIARequirements); + default: + throw new ParserException($"{requiredComplianceStandard} compliance standard is not supported."); + } + } } + else + { + Console.WriteLine("No required compliance standard."); + return (ComplianceStandard.None, Array.Empty()); + } + } - // TODO: Eventually this return type should be changed to SpdxMetadata to be consistent with naming. - return this.Metadata; + /// + /// Validate if elements meet required compliance standards + /// + /// + /// + private void ValidateElementsBasedOnComplianceStandard(ComplianceStandard complianceStandardAsEnum, List elementsList) + { + switch (complianceStandardAsEnum) + { + case ComplianceStandard.NTIA: + ValidateNTIARequirements(elementsList); + break; + case ComplianceStandard.None: + Console.WriteLine("No compliance standard to enforce."); + break; + default: + Console.WriteLine($"Unexpected compliance standard {complianceStandardAsEnum}."); + break; + } } - public ManifestInfo[] RegisterManifest() => new ManifestInfo[] { SpdxConstants.SPDX30ManifestInfo }; + private object? ParseJsonObject(JsonObject jsonObject, ComplianceStandard complianceStandardAsEnum) + { + if (jsonObject is null || !jsonObject.Any()) + { + return null; + } + else + { + var entityType = GetEntityType(jsonObject, complianceStandardAsEnum); + + object? deserializedObject = null; + try + { + deserializedObject = JsonSerializer.Deserialize(jsonObject.ToString(), entityType, jsonSerializerOptions); + } + catch (Exception e) + { + throw new ParserException(e.Message); + } + + return deserializedObject; + } + } + + /// + /// Handle deduplication of elements by checking SPDX ID + /// + /// + /// + /// + /// + private bool IsUniqueElement(string spdxId, HashSet elementsSpdxIdList) + { + if (!elementsSpdxIdList.TryGetValue(spdxId, out _)) + { + elementsSpdxIdList.Add(spdxId); + return true; + } + else + { + Console.WriteLine($"Duplicate element with SPDX ID {spdxId} found. Skipping."); + return false; + } + } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs index 0d0f8e3ce..aa9852b05 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs @@ -25,7 +25,7 @@ public static SbomFile ToSbomFile(this File spdxFile) { var checksums = spdxFile.VerifiedUsing?.Select(c => c.ToSbomChecksum()); - if (checksums == null || !checksums.Any() || checksums.All(c => c.Algorithm != AlgorithmName.SHA256)) + if (checksums is null || !checksums.Any() || checksums.All(c => c.Algorithm != AlgorithmName.SHA256)) { throw new ParserException("File hash is missing a SHA256 value"); } diff --git a/test/Microsoft.Sbom.Api.Tests/Config/Validators/ManifestInfoValidatorTests.cs b/test/Microsoft.Sbom.Api.Tests/Config/Validators/ManifestInfoValidatorTests.cs new file mode 100644 index 000000000..a73a3dfd7 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Config/Validators/ManifestInfoValidatorTests.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Config.Validators; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using PowerArgs; + +namespace Microsoft.Sbom.Api.Tests.Config.Validators; + +[TestClass] +public class ManifestInfoValidatorTests +{ + private readonly Mock mockAssemblyConfig = new Mock(); + + [TestMethod] + public void InvalidManifestInfoThrows() + { + var invalidManifestInfo = new ManifestInfo + { + Name = "SPDX", + Version = "asdf" + }; + + var validator = new ManifestInfoValidator(mockAssemblyConfig.Object); + Assert.ThrowsException(() => validator.ValidateInternal("property", invalidManifestInfo, null)); + } + + [TestMethod] + public void ValidManifestInfoPasses() + { + var validManifestInfo = new ManifestInfo + { + Name = "SPDX", + Version = "3.0" + }; + + var validator = new ManifestInfoValidator(mockAssemblyConfig.Object); + validator.ValidateInternal("property", validManifestInfo, null); + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs index 9f49a3e3b..0d84c8ab2 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs @@ -40,7 +40,6 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Newtonsoft.Json.Linq; -using Spectre.Console; using Checksum = Microsoft.Sbom.Contracts.Checksum; using Generator30 = Microsoft.Sbom.Parsers.Spdx30SbomParser.Generator; using IComponentDetector = Microsoft.Sbom.Api.Utils.IComponentDetector; diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs index 57d225fc4..a28e28f75 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/SbomParserBasedValidationWorkflowTests.cs @@ -513,8 +513,6 @@ public async Task SbomParserBasedValidationWorkflowTests_SetsComplianceStandard_ sbomConfigs.Setup(c => c.Get(SpdxConstants.SPDX30ManifestInfo)).Returns(sbomConfig); fileSystemMock.Setup(f => f.OpenRead("/root/_manifest/spdx_3.0/manifest.spdx.json")).Returns(Stream.Null); - //fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) - // .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); fileSystemMock.Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())) .Returns((string root, string relativePath) => $"{root}/{relativePath}"); fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())) diff --git a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs index 07dda45d4..428066e4d 100644 --- a/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs +++ b/test/Microsoft.Sbom.Parsers.Spdx30SbomParser.Tests/Parser/SbomParserTestsBase.cs @@ -40,7 +40,7 @@ public ParserResults Parse(SPDX30Parser parser, Stream? stream = null, bool clos switch (result.FieldName) { case SPDX30Parser.ContextProperty: - results.FormatEnforcedSPDX3Result.Context = (string?)((ContextsResult)result).Contexts.FirstOrDefault(); + results.FormatEnforcedSPDX3Result.Context = (result as ContextsResult)?.Contexts.FirstOrDefault(); break; case SPDX30Parser.GraphProperty: var elementsResult = (ElementsResult)result; From 9fb11294bd97c39a3700bf50499b4b313b1d3d6e Mon Sep 17 00:00:00 2001 From: ppandrate Date: Thu, 13 Feb 2025 19:36:15 -0800 Subject: [PATCH 13/15] working API changes --- .../Executors/Spdx2SerializationStrategy.cs | 14 ++-- .../Executors/Spdx3SerializationStrategy.cs | 61 ++++++++++------- .../Manifest/BaseManifestConfigHandler.cs | 66 +++++++++++++++++++ .../SPDX22ManifestConfigHandler.cs | 51 +++----------- .../SPDX30ManifestConfigHandler.cs | 62 +++++++++++++++++ .../Output/MetadataBuilder.cs | 21 +++--- .../Helpers/JsonDocumentCollection.cs | 5 ++ .../Helpers/RelationshipsArrayGenerator.cs | 5 +- .../IManifestGenerator.cs | 8 +++ .../Generator.cs | 32 +++++++++ .../GenerateSbomTask.cs | 3 +- .../RelationshipsArrayGeneratorTest.cs | 29 ++++---- 12 files changed, 253 insertions(+), 104 deletions(-) create mode 100644 src/Microsoft.Sbom.Api/Manifest/BaseManifestConfigHandler.cs create mode 100644 src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX30ManifestConfigHandler.cs diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs index 6d44377ab..cba60217b 100644 --- a/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs +++ b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs @@ -81,27 +81,27 @@ public async Task> WriteJsonObjectsToSbomAsync( // Files section var filesGenerateResult = await fileArrayGenerator.GenerateAsync(); filesGenerateResult.Errors.AddRange(filesGenerateResult.Errors); - WriteJsonObjectsFromGenerationResult(filesGenerateResult); + WriteJsonObjectsFromGenerationResult(filesGenerateResult, fileArrayGenerator.SbomConfig); // Packages section var packagesGenerateResult = await packageArrayGenerator.GenerateAsync(); packagesGenerateResult.Errors.AddRange(packagesGenerateResult.Errors); - WriteJsonObjectsFromGenerationResult(packagesGenerateResult); + WriteJsonObjectsFromGenerationResult(packagesGenerateResult, packageArrayGenerator.SbomConfig); // External Document Reference section var externalDocumentReferenceGenerateResult = await externalDocumentReferenceGenerator.GenerateAsync(); externalDocumentReferenceGenerateResult.Errors.AddRange(externalDocumentReferenceGenerateResult.Errors); - WriteJsonObjectsFromGenerationResult(externalDocumentReferenceGenerateResult); + WriteJsonObjectsFromGenerationResult(externalDocumentReferenceGenerateResult, externalDocumentReferenceGenerator.SbomConfig); // Relationships section var relationshipGenerateResult = await relationshipsArrayGenerator.GenerateAsync(); relationshipGenerateResult.Errors.AddRange(relationshipGenerateResult.Errors); - WriteJsonObjectsFromGenerationResult(relationshipGenerateResult); + WriteJsonObjectsFromGenerationResult(relationshipGenerateResult, relationshipsArrayGenerator.SbomConfig); return errors; } - private void WriteJsonObjectsFromGenerationResult(GenerationResult generationResult) + private void WriteJsonObjectsFromGenerationResult(GenerationResult generationResult, ISbomConfig sbomConfig) { foreach (var serializer in generationResult.SerializerToJsonDocuments.Keys) { @@ -109,8 +109,8 @@ private void WriteJsonObjectsFromGenerationResult(GenerationResult generationRes { serializer.Write(jsonDocument); } - - serializer.EndJsonArray(); } + + sbomConfig.JsonSerializer.EndJsonArray(); } } diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs index 41d541084..7bd492900 100644 --- a/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs +++ b/src/Microsoft.Sbom.Api/Executors/Spdx3SerializationStrategy.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; using System.Collections.Generic; using System.Text.Json; using System.Threading.Tasks; @@ -54,6 +53,10 @@ public async Task> WriteJsonObjectsToSbomAsync( relationshipsArrayGenerator.SpdxManifestVersion = spdxManifestVersion; externalDocumentReferenceGenerator.SpdxManifestVersion = spdxManifestVersion; + WriteContext(sbomConfig); + + sbomConfig.JsonSerializer.StartJsonArray(SpdxConstants.SPDXGraphHeaderName); + // Files section var generateResult = await fileArrayGenerator.GenerateAsync(); WriteElementsToSbom(generateResult); @@ -73,6 +76,8 @@ public async Task> WriteJsonObjectsToSbomAsync( generateResult.Errors.AddRange(relationshipGenerateResult.Errors); WriteElementsToSbom(relationshipGenerateResult); + sbomConfig.JsonSerializer.EndJsonArray(); + return generateResult.Errors; } @@ -81,40 +86,50 @@ private void WriteElementsToSbom(GenerationResult generateResult) // Write the JSON objects to the SBOM foreach (var serializer in generateResult.SerializerToJsonDocuments.Keys) { - // Write context - serializer.StartJsonArray(SpdxConstants.SPDXContextHeaderName); - var document = JsonDocument.Parse(SpdxConstants.SPDX3ContextValue); - serializer.Write(document); - serializer.EndJsonArray(); - // Deduplication of elements by checking SPDX ID var elementsSpdxIdList = new HashSet(); - serializer.StartJsonArray(SpdxConstants.SPDXGraphHeaderName); - var jsonDocuments = generateResult.SerializerToJsonDocuments[serializer]; foreach (var jsonDocument in jsonDocuments) { - foreach (var element in jsonDocument.RootElement.EnumerateArray()) + if (jsonDocument.RootElement.ValueKind == JsonValueKind.Object) { - if (element.TryGetProperty("spdxId", out var spdxIdField)) + WriteElement(serializer, jsonDocument.RootElement, elementsSpdxIdList); + } + else + { + foreach (var element in jsonDocument.RootElement.EnumerateArray()) { - var spdxId = spdxIdField.GetString(); - - if (elementsSpdxIdList.TryGetValue(spdxId, out _)) - { - Console.WriteLine($"Duplicate element with SPDX ID {spdxId} found. Skipping."); - } - else - { - serializer.Write(element); - elementsSpdxIdList.Add(spdxId); - } + WriteElement(serializer, element, elementsSpdxIdList); } } } + } + } - serializer.EndJsonArray(); + private void WriteContext(ISbomConfig sbomConfig) + { + sbomConfig.JsonSerializer.StartJsonArray(SpdxConstants.SPDXContextHeaderName); + var document = JsonDocument.Parse(SpdxConstants.SPDX3ContextValue); + sbomConfig.JsonSerializer.Write(document); + sbomConfig.JsonSerializer.EndJsonArray(); + } + + private void WriteElement(IManifestToolJsonSerializer serializer, JsonElement element, HashSet elementsSpdxIdList) + { + if (element.TryGetProperty("spdxId", out var spdxIdField)) + { + var spdxId = spdxIdField.GetString(); + + if (elementsSpdxIdList.TryGetValue(spdxId, out _)) + { + return; + } + else + { + serializer.Write(element); + elementsSpdxIdList.Add(spdxId); + } } } } diff --git a/src/Microsoft.Sbom.Api/Manifest/BaseManifestConfigHandler.cs b/src/Microsoft.Sbom.Api/Manifest/BaseManifestConfigHandler.cs new file mode 100644 index 000000000..a0678a3e4 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Manifest/BaseManifestConfigHandler.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Recorder; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; + +namespace Microsoft.Sbom.Api.Manifest.ManifestConfigHandlers; + +/// +/// Provides the base class for ManifestConfig handlers. +/// +public abstract class BaseManifestConfigHandler : IManifestConfigHandler +{ + protected readonly IMetadataBuilderFactory metadataBuilderFactory; + protected readonly IConfiguration configuration; + protected readonly IFileSystemUtils fileSystemUtils; + + protected BaseManifestConfigHandler( + IConfiguration configuration, + IFileSystemUtils fileSystemUtils, + IMetadataBuilderFactory metadataBuilderFactory) + { + this.metadataBuilderFactory = metadataBuilderFactory ?? throw new ArgumentException(nameof(metadataBuilderFactory)); + this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + } + + protected abstract ManifestInfo ManifestInfo { get; } + + protected string ManifestDirPath => configuration.ManifestDirPath?.Value; + + protected string SbomDirPath => fileSystemUtils.JoinPaths(ManifestDirPath, $"{ManifestInfo.Name.ToLower()}_{ManifestInfo.Version.ToLower()}"); + + protected string SbomFilePath => fileSystemUtils.JoinPaths(SbomDirPath, $"manifest.{ManifestInfo.Name.ToLower()}.json"); + + protected string ManifestJsonSha256FilePath => $"{SbomFilePath}.sha256"; + + protected string CatalogFilePath => fileSystemUtils.JoinPaths(SbomDirPath, SpdxConstants.CatalogFileName); + + protected string BsiFilePath => fileSystemUtils.JoinPaths(SbomDirPath, SpdxConstants.BsiFileName); + + protected IMetadataBuilder MetadataBuilder => metadataBuilderFactory.Get(ManifestInfo); + + public abstract bool TryGetManifestConfig(out ISbomConfig sbomConfig); + + protected ISbomConfig CreateSbomConfig() + { + return new SbomConfig(fileSystemUtils) + { + ManifestInfo = ManifestInfo, + ManifestJsonDirPath = SbomDirPath, + ManifestJsonFilePath = SbomFilePath, + CatalogFilePath = CatalogFilePath, + BsiFilePath = BsiFilePath, + ManifestJsonFileSha256FilePath = ManifestJsonSha256FilePath, + MetadataBuilder = MetadataBuilder, + Recorder = new SbomPackageDetailsRecorder() + }; + } +} diff --git a/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX22ManifestConfigHandler.cs b/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX22ManifestConfigHandler.cs index 3a53a6018..d80666e23 100644 --- a/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX22ManifestConfigHandler.cs +++ b/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX22ManifestConfigHandler.cs @@ -1,12 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System; -using Microsoft.Sbom.Api.Manifest.Configuration; -using Microsoft.Sbom.Api.Recorder; using Microsoft.Sbom.Common; using Microsoft.Sbom.Common.Config; using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; namespace Microsoft.Sbom.Api.Manifest.ManifestConfigHandlers; @@ -14,55 +12,22 @@ namespace Microsoft.Sbom.Api.Manifest.ManifestConfigHandlers; /// /// Provides the ManifestConfig for the SPDX 2.2 format. /// -public class SPDX22ManifestConfigHandler : IManifestConfigHandler +public class SPDX22ManifestConfigHandler : BaseManifestConfigHandler { - private readonly IMetadataBuilderFactory metadataBuilderFactory; - private readonly IConfiguration configuration; - private readonly IFileSystemUtils fileSystemUtils; - - private string ManifestDirPath => configuration.ManifestDirPath?.Value; - - // directory path for SPDX 2.2 is - // root/_manifest/spdx_2.2/ - private string SbomDirPath => fileSystemUtils.JoinPaths(ManifestDirPath, $"{SpdxConstants.SPDX22ManifestInfo.Name.ToLower()}_{SpdxConstants.SPDX22ManifestInfo.Version.ToLower()}"); - - // sbom file path is manifest.spdx.json in the sbom directory. - private string SbomFilePath => fileSystemUtils.JoinPaths(SbomDirPath, $"manifest.{SpdxConstants.SPDX22ManifestInfo.Name.ToLower()}.json"); - - // sha file is sbom file + .sha256 - private string ManifestJsonSha256FilePath => $"{SbomFilePath}.sha256"; - - // catalog file is always manifest.cat - private string CatalogFilePath => fileSystemUtils.JoinPaths(SbomDirPath, SpdxConstants.CatalogFileName); - - // bsi.json file contains build session metadata and is always bsi.json - private string BsiJsonFilePath => fileSystemUtils.JoinPaths(SbomDirPath, SpdxConstants.BsiFileName); - - private IMetadataBuilder MetadataBuilder => metadataBuilderFactory.Get(SpdxConstants.SPDX22ManifestInfo); - public SPDX22ManifestConfigHandler( IConfiguration configuration, IFileSystemUtils fileSystemUtils, IMetadataBuilderFactory metadataBuilderFactory) + : base(configuration, fileSystemUtils, metadataBuilderFactory) { - this.metadataBuilderFactory = metadataBuilderFactory ?? throw new ArgumentException(nameof(metadataBuilderFactory)); - this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); - this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); } - public bool TryGetManifestConfig(out ISbomConfig sbomConfig) + /// + protected override ManifestInfo ManifestInfo => SpdxConstants.SPDX22ManifestInfo; + + public override bool TryGetManifestConfig(out ISbomConfig sbomConfig) { - sbomConfig = new SbomConfig(fileSystemUtils) - { - ManifestInfo = SpdxConstants.SPDX22ManifestInfo, - ManifestJsonDirPath = SbomDirPath, - ManifestJsonFilePath = SbomFilePath, - CatalogFilePath = CatalogFilePath, - BsiFilePath = BsiJsonFilePath, - ManifestJsonFileSha256FilePath = ManifestJsonSha256FilePath, - MetadataBuilder = MetadataBuilder, - Recorder = new SbomPackageDetailsRecorder() - }; + sbomConfig = CreateSbomConfig(); // For generation the default behavior is to always return true // as we generate all the current formats of SBOM. Only override if the -mi diff --git a/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX30ManifestConfigHandler.cs b/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX30ManifestConfigHandler.cs new file mode 100644 index 000000000..72b506937 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX30ManifestConfigHandler.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using SpdxConstants = Microsoft.Sbom.Constants.SpdxConstants; + +namespace Microsoft.Sbom.Api.Manifest.ManifestConfigHandlers; + +/// +/// Provides the ManifestConfig for the SPDX 3.0 format. +/// +public class SPDX30ManifestConfigHandler : BaseManifestConfigHandler +{ + public SPDX30ManifestConfigHandler( + IConfiguration configuration, + IFileSystemUtils fileSystemUtils, + IMetadataBuilderFactory metadataBuilderFactory) + : base(configuration, fileSystemUtils, metadataBuilderFactory) + { + } + + /// + protected override ManifestInfo ManifestInfo => SpdxConstants.SPDX30ManifestInfo; + + public override bool TryGetManifestConfig(out ISbomConfig sbomConfig) + { + sbomConfig = CreateSbomConfig(); + + // For generation the default behavior is to always return true + // as we generate all the current formats of SBOM. Only override if the -mi + // argument is specified. + if (configuration.ManifestToolAction == ManifestToolActions.Generate) + { + if (configuration.ManifestInfo?.Value != null + && !configuration.ManifestInfo.Value.Contains(SpdxConstants.SPDX30ManifestInfo)) + { + return false; + } + + return true; + } + + if (configuration.ManifestToolAction == ManifestToolActions.Validate) + { + // We can only validate one format at a time, so check if its this one and return true/false. + if (configuration.ManifestInfo?.Value != null + && configuration.ManifestInfo.Value.Count == 1 + && configuration.ManifestInfo.Value.Contains(SpdxConstants.SPDX30ManifestInfo)) + { + return true; + } + + return false; + } + + sbomConfig = null; + return false; + } +} diff --git a/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs b/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs index 38c90acf6..22bc25c0e 100644 --- a/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs +++ b/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs @@ -8,7 +8,6 @@ using Microsoft.Sbom.Api.Utils; using Microsoft.Sbom.Extensions; using Microsoft.Sbom.Extensions.Entities; -using Microsoft.Sbom.Parsers.Spdx22SbomParser; using Serilog; namespace Microsoft.Sbom.Api.Output; @@ -55,13 +54,13 @@ public bool TryGetFilesArrayHeaderName(out string headerName) { try { - headerName = ((Generator)manifestGenerator).FilesArrayHeaderName; + headerName = manifestGenerator.FilesArrayHeaderName; return true; } catch (NotSupportedException) { headerName = null; - logger.Warning("Files array not suppored on this SBOM format."); + logger.Warning("Files array not supported on this SBOM format."); return false; } } @@ -70,13 +69,13 @@ public bool TryGetPackageArrayHeaderName(out string headerName) { try { - headerName = ((Generator)manifestGenerator).PackagesArrayHeaderName; + headerName = manifestGenerator.PackagesArrayHeaderName; return true; } catch (NotSupportedException) { headerName = null; - logger.Warning("Packages array not suppored on this SBOM format."); + logger.Warning("Packages array not supported on this SBOM format."); return false; } } @@ -85,13 +84,13 @@ public bool TryGetExternalRefArrayHeaderName(out string headerName) { try { - headerName = ((Generator)manifestGenerator).ExternalDocumentRefArrayHeaderName; + headerName = manifestGenerator.ExternalDocumentRefArrayHeaderName; return true; } catch (NotSupportedException) { headerName = null; - logger.Warning("External Document Reference array not suppored on this SBOM format."); + logger.Warning("External Document Reference array not supported on this SBOM format."); return false; } } @@ -100,8 +99,7 @@ public bool TryGetRootPackageJson(IInternalMetadataProvider internalMetadataProv { try { - generationResult = manifestGenerator - .GenerateRootPackage(internalMetadataProvider); + generationResult = manifestGenerator.GenerateRootPackage(internalMetadataProvider); if (generationResult == null) { @@ -122,8 +120,7 @@ public bool TryGetCreationInfoJson(IInternalMetadataProvider internalMetadataPro { try { - generationResult = manifestGenerator - .GenerateJsonDocument(internalMetadataProvider); + generationResult = manifestGenerator.GenerateJsonDocument(internalMetadataProvider); if (generationResult == null) { @@ -144,7 +141,7 @@ public bool TryGetRelationshipsHeaderName(out string headerName) { try { - headerName = ((Generator)manifestGenerator).RelationshipsArrayHeaderName; + headerName = manifestGenerator.RelationshipsArrayHeaderName; return headerName != null; } catch (NotSupportedException) diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/JsonDocumentCollection.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/JsonDocumentCollection.cs index c18f6b275..b1854712d 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/JsonDocumentCollection.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/JsonDocumentCollection.cs @@ -10,6 +10,11 @@ public class JsonDocumentCollection { public Dictionary> SerializersToJson { get; } + public JsonDocumentCollection() + { + SerializersToJson = new Dictionary>(); + } + public void AddJsonDocument(T key, JsonDocument document) { if (SerializersToJson.TryGetValue(key, out var jsonDocuments)) diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs index be2b579ca..bffe009b5 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs @@ -28,7 +28,7 @@ public class RelationshipsArrayGenerator : IJsonArrayGenerator GenerateAsync() @@ -60,6 +59,8 @@ public async Task GenerateAsync() // Write the relationship array only if supported if (serializationStrategy.AddToRelationshipsSupportingConfig(relationshipsArraySupportingConfigs, this.SbomConfig)) { + var generationData = this.SbomConfig?.Recorder.GetGenerationData(); + var jsonChannelsArray = new ChannelReader[] { // Packages relationships diff --git a/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs b/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs index 82573d0e2..5d716ee16 100644 --- a/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs +++ b/src/Microsoft.Sbom.Extensions/IManifestGenerator.cs @@ -104,6 +104,14 @@ public interface IManifestGenerator /// string Version { get; } + string FilesArrayHeaderName { get; } + + string PackagesArrayHeaderName { get; } + + string RelationshipsArrayHeaderName { get; } + + string ExternalDocumentRefArrayHeaderName { get; } + /// /// Return a dictionary of items that need to be added to the header of /// the generated SBOM. diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs index 0f96fd19e..948749676 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Generator.cs @@ -38,6 +38,38 @@ public class Generator : IManifestGenerator Converters = { new ElementSerializer() }, }; + public string FilesArrayHeaderName + { + get + { + throw new NotSupportedException("Files array not supported for SBOMs generated with SPDX 3.0."); + } + } + + public string PackagesArrayHeaderName + { + get + { + throw new NotSupportedException("Packages array not supported for SBOMs generated with SPDX 3.0."); + } + } + + public string RelationshipsArrayHeaderName + { + get + { + throw new NotSupportedException("Relationships array not supported for SBOMs generated with SPDX 3.0."); + } + } + + public string ExternalDocumentRefArrayHeaderName + { + get + { + throw new NotSupportedException("External document ref array not supported for SBOMs generated with SPDX 3.0."); + } + } + /// /// Generates all SPDX elements related to a single file. /// diff --git a/src/Microsoft.Sbom.Targets/GenerateSbomTask.cs b/src/Microsoft.Sbom.Targets/GenerateSbomTask.cs index 74007aac5..6a701c7fa 100644 --- a/src/Microsoft.Sbom.Targets/GenerateSbomTask.cs +++ b/src/Microsoft.Sbom.Targets/GenerateSbomTask.cs @@ -54,7 +54,8 @@ public GenerateSbom() .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton()) + .AddSingleton() + .AddSingleton()) .Build(); this.Generator = host.Services.GetRequiredService(); } diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs index a2475c009..5141be562 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs @@ -53,24 +53,12 @@ public class RelationshipsArrayGeneratorTest [TestInitialize] public void Setup() { - recorder = new SbomPackageDetailsRecorder(); - relationships = new List(); - relationshipGeneratorMock.Setup(r => r.Run(It.IsAny>(), It.IsAny())) - .Callback, ManifestInfo>((relationship, manifestInfo) => - { - while (relationship.MoveNext()) - { - relationships.Add(relationship.Current); - } - }); - relationshipGeneratorMock.CallBase = true; - relationshipsArrayGenerator = new RelationshipsArrayGenerator(relationshipGeneratorMock.Object, new ChannelUtils(), loggerMock.Object, recorderMock.Object); - manifestGeneratorProvider.Init(); metadataBuilder = new MetadataBuilder( mockLogger.Object, manifestGeneratorProvider, SpdxConstants.TestManifestInfo, recorderMock.Object); + recorder = new SbomPackageDetailsRecorder(); sbomConfig = new SbomConfig(fileSystemUtilsMock.Object) { ManifestInfo = SpdxConstants.TestManifestInfo, @@ -80,9 +68,18 @@ public void Setup() Recorder = recorder, }; - // Set up the relationshipsArrayGenerator sbomConfig, this is normally done in SBOM generation workflow. - // Since we are testing the RelationshipsArrayGenerator, we need to set it up manually. - relationshipsArrayGenerator.SbomConfig = sbomConfig; + relationships = new List(); + relationshipGeneratorMock.Setup(r => r.Run(It.IsAny>(), It.IsAny())) + .Callback, ManifestInfo>((relationship, manifestInfo) => + { + while (relationship.MoveNext()) + { + relationships.Add(relationship.Current); + } + }); + relationshipGeneratorMock.CallBase = true; + relationshipsArrayGenerator = new RelationshipsArrayGenerator(relationshipGeneratorMock.Object, new ChannelUtils(), loggerMock.Object, recorderMock.Object); + manifestGeneratorProvider.Init(); fileSystemUtilsMock.Setup(f => f.CreateDirectory(ManifestJsonDirPath)); fileSystemUtilsMock.Setup(f => f.OpenWrite(JsonFilePath)).Returns(new MemoryStream()); From b91f763c37cad29e92e0f179275a7f0e85a56140 Mon Sep 17 00:00:00 2001 From: ppandrate Date: Fri, 14 Feb 2025 15:22:28 -0800 Subject: [PATCH 14/15] bug fixes and cleanup --- .../Config/ApiConfigurationBuilder.cs | 1 - .../Config/Args/ValidationArgs.cs | 2 +- src/Microsoft.Sbom.Api/Config/ConfigFile.cs | 5 +++++ src/Microsoft.Sbom.Api/ConfigurationProfile.cs | 3 +-- .../Executors/ConcurrentSha256HashValidator.cs | 2 +- .../Executors/Spdx2SerializationStrategy.cs | 4 ++-- .../Executors/Spdx3SerializationStrategy.cs | 16 ++++++++-------- .../Config/IConfiguration.cs | 2 +- .../Entities/Element.cs | 4 ++-- .../Parser/SPDX30Parser.cs | 6 +----- .../Utils/SPDXToSbomFormatConverterExtensions.cs | 6 ------ src/Microsoft.Sbom.Targets/GenerateSbomTask.cs | 9 ++++++--- src/Microsoft.Sbom.Tool/ValidationService.cs | 5 +++-- .../Helpers/RelationshipsArrayGeneratorTest.cs | 5 ++++- 14 files changed, 35 insertions(+), 35 deletions(-) diff --git a/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs b/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs index 81042c5f0..3b234548a 100644 --- a/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs +++ b/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs @@ -143,7 +143,6 @@ public static InputConfiguration GetConfiguration( IgnoreMissing = GetConfigurationSetting(ignoreMissing), Parallelism = GetConfigurationSetting(sanitizedRuntimeConfiguration.WorkflowParallelism), ManifestInfo = ConvertSbomSpecificationToManifestInfo(specifications), - ComplianceStandard = GetConfigurationSetting(string.Empty), }; SetVerbosity(sanitizedRuntimeConfiguration, configuration); diff --git a/src/Microsoft.Sbom.Api/Config/Args/ValidationArgs.cs b/src/Microsoft.Sbom.Api/Config/Args/ValidationArgs.cs index 3e8be3a45..6a4ab18ec 100644 --- a/src/Microsoft.Sbom.Api/Config/Args/ValidationArgs.cs +++ b/src/Microsoft.Sbom.Api/Config/Args/ValidationArgs.cs @@ -87,5 +87,5 @@ public class ValidationArgs : GenerationAndValidationCommonArgs /// [ArgDescription("The compliance standard to validate against")] [ArgShortcut("cs")] - public ComplianceStandard ComplianceStandard { get; set; } + public string ComplianceStandard { get; set; } } diff --git a/src/Microsoft.Sbom.Api/Config/ConfigFile.cs b/src/Microsoft.Sbom.Api/Config/ConfigFile.cs index 69074c646..fdcb99dd9 100644 --- a/src/Microsoft.Sbom.Api/Config/ConfigFile.cs +++ b/src/Microsoft.Sbom.Api/Config/ConfigFile.cs @@ -181,4 +181,9 @@ public class ConfigFile /// If set to true, we will attempt to parse metadata information of packages detected in the SBOM from the local package cache. /// public bool? EnablePackageMetadataParsing { get; set; } + + /// + /// The compliance standard to validate against + /// + public string ComplianceStandard { get; set; } } diff --git a/src/Microsoft.Sbom.Api/ConfigurationProfile.cs b/src/Microsoft.Sbom.Api/ConfigurationProfile.cs index e5bbe88de..3c555beba 100644 --- a/src/Microsoft.Sbom.Api/ConfigurationProfile.cs +++ b/src/Microsoft.Sbom.Api/ConfigurationProfile.cs @@ -40,8 +40,7 @@ public ConfigurationProfile() .ForMember(c => c.NamespaceUriUniquePart, o => o.Ignore()) .ForMember(c => c.NamespaceUriBase, o => o.Ignore()) .ForMember(c => c.DeleteManifestDirIfPresent, o => o.Ignore()) - .ForMember(c => c.PackageSupplier, o => o.Ignore()) - .ForMember(c => c.ComplianceStandard, o => o.Ignore()); + .ForMember(c => c.PackageSupplier, o => o.Ignore()); CreateMap() #pragma warning disable CS0618 // 'Configuration.ManifestPath' is obsolete: 'This field is not provided by the user or configFile, set by system' diff --git a/src/Microsoft.Sbom.Api/Executors/ConcurrentSha256HashValidator.cs b/src/Microsoft.Sbom.Api/Executors/ConcurrentSha256HashValidator.cs index 358017575..5aa8c1852 100644 --- a/src/Microsoft.Sbom.Api/Executors/ConcurrentSha256HashValidator.cs +++ b/src/Microsoft.Sbom.Api/Executors/ConcurrentSha256HashValidator.cs @@ -79,7 +79,7 @@ private async Task Validate(InternalSbomFileInfo internalFileInfo, Channel> WriteJsonObjectsToSbomAsync( relationshipsArrayGenerator.SpdxManifestVersion = spdxManifestVersion; externalDocumentReferenceGenerator.SpdxManifestVersion = spdxManifestVersion; + // Holds the SPDX IDs of all the elements that have been written to the SBOM. Used for deduplication. + var elementsSpdxIdList = new HashSet(); + WriteContext(sbomConfig); sbomConfig.JsonSerializer.StartJsonArray(SpdxConstants.SPDXGraphHeaderName); // Files section var generateResult = await fileArrayGenerator.GenerateAsync(); - WriteElementsToSbom(generateResult); + WriteElementsToSbom(generateResult, elementsSpdxIdList); // Packages section var packagesGenerateResult = await packageArrayGenerator.GenerateAsync(); generateResult.Errors.AddRange(packagesGenerateResult.Errors); - WriteElementsToSbom(packagesGenerateResult); + WriteElementsToSbom(packagesGenerateResult, elementsSpdxIdList); // External Document Reference section var externalDocumentReferenceGenerateResult = await externalDocumentReferenceGenerator.GenerateAsync(); generateResult.Errors.AddRange(externalDocumentReferenceGenerateResult.Errors); - WriteElementsToSbom(externalDocumentReferenceGenerateResult); + WriteElementsToSbom(externalDocumentReferenceGenerateResult, elementsSpdxIdList); // Relationships section var relationshipGenerateResult = await relationshipsArrayGenerator.GenerateAsync(); generateResult.Errors.AddRange(relationshipGenerateResult.Errors); - WriteElementsToSbom(relationshipGenerateResult); + WriteElementsToSbom(relationshipGenerateResult, elementsSpdxIdList); sbomConfig.JsonSerializer.EndJsonArray(); return generateResult.Errors; } - private void WriteElementsToSbom(GenerationResult generateResult) + private void WriteElementsToSbom(GenerationResult generateResult, HashSet elementsSpdxIdList) { // Write the JSON objects to the SBOM foreach (var serializer in generateResult.SerializerToJsonDocuments.Keys) { - // Deduplication of elements by checking SPDX ID - var elementsSpdxIdList = new HashSet(); - var jsonDocuments = generateResult.SerializerToJsonDocuments[serializer]; foreach (var jsonDocument in jsonDocuments) { diff --git a/src/Microsoft.Sbom.Common/Config/IConfiguration.cs b/src/Microsoft.Sbom.Common/Config/IConfiguration.cs index 842fb0bf5..f8349b2ca 100644 --- a/src/Microsoft.Sbom.Common/Config/IConfiguration.cs +++ b/src/Microsoft.Sbom.Common/Config/IConfiguration.cs @@ -209,7 +209,7 @@ public interface IConfiguration ConfigurationSetting SbomDir { get; set; } /// - /// Gets or sets the directory containing the sbom(s) to redact. + /// The compliance standard to validate against. /// ConfigurationSetting ComplianceStandard { get; set; } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Element.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Element.cs index 06a38706e..4a1a06581 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Element.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Entities/Element.cs @@ -10,9 +10,9 @@ namespace Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; /// Base domain class from which all other SPDX-3.0 domain classes derive. /// https://spdx.github.io/spdx-spec/v3.0.1/model/Core/Classes/Element/ /// -public abstract class Element +public class Element { - protected Element() + public Element() { CreationInfoDetails = "_:creationinfo"; Type = GetType().Name; diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs index 2f2ea0403..8d05f46a9 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Parser/SPDX30Parser.cs @@ -230,7 +230,6 @@ private ElementsResult ConvertToElements(List? jsonList, ParserStateResu elementsResult.RelationshipsCount += 1; break; default: - Console.WriteLine($"Unrecognized entity type: {entityType?.Name}"); break; } } @@ -313,7 +312,7 @@ private void ValidateSbomDocCreationForNTIA(List elementsList) /// private void ValidateSbomFilesForNTIA(List elementsList) { - var fileElements = elementsList.Where(element => element is NTIAFile); + var fileElements = elementsList.Where(element => element is Parsers.Spdx30SbomParser.Entities.File); foreach (var fileElement in fileElements) { var fileSpdxId = fileElement.SpdxId; @@ -430,7 +429,6 @@ private SpdxMetadata SetMetadata(ElementsResult result) } else { - Console.WriteLine("No required compliance standard."); return (ComplianceStandard.None, Array.Empty()); } } @@ -448,7 +446,6 @@ private void ValidateElementsBasedOnComplianceStandard(ComplianceStandard compli ValidateNTIARequirements(elementsList); break; case ComplianceStandard.None: - Console.WriteLine("No compliance standard to enforce."); break; default: Console.WriteLine($"Unexpected compliance standard {complianceStandardAsEnum}."); @@ -496,7 +493,6 @@ private bool IsUniqueElement(string spdxId, HashSet elementsSpdxIdList) } else { - Console.WriteLine($"Duplicate element with SPDX ID {spdxId} found. Skipping."); return false; } } diff --git a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs index aa9852b05..448960396 100644 --- a/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs +++ b/src/Microsoft.Sbom.Parsers.Spdx30SbomParser/Utils/SPDXToSbomFormatConverterExtensions.cs @@ -4,7 +4,6 @@ using System.Linq; using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Contracts.Enums; -using Microsoft.Sbom.JsonAsynchronousNodeKit.Exceptions; using Microsoft.Sbom.Parsers.Spdx30SbomParser.Entities; using SbomChecksum = Microsoft.Sbom.Contracts.Checksum; @@ -25,11 +24,6 @@ public static SbomFile ToSbomFile(this File spdxFile) { var checksums = spdxFile.VerifiedUsing?.Select(c => c.ToSbomChecksum()); - if (checksums is null || !checksums.Any() || checksums.All(c => c.Algorithm != AlgorithmName.SHA256)) - { - throw new ParserException("File hash is missing a SHA256 value"); - } - // Not setting LicenseConcluded and LicenseInfoInFiles since the whole SBOM is required to set these values. return new SbomFile { diff --git a/src/Microsoft.Sbom.Targets/GenerateSbomTask.cs b/src/Microsoft.Sbom.Targets/GenerateSbomTask.cs index 6a701c7fa..708fbb6e7 100644 --- a/src/Microsoft.Sbom.Targets/GenerateSbomTask.cs +++ b/src/Microsoft.Sbom.Targets/GenerateSbomTask.cs @@ -19,7 +19,8 @@ namespace Microsoft.Sbom.Targets; using Microsoft.Sbom.Contracts.Interfaces; using Microsoft.Sbom.Extensions; using Microsoft.Sbom.Extensions.DependencyInjection; -using Microsoft.Sbom.Parsers.Spdx22SbomParser; +using SPDX22 = Microsoft.Sbom.Parsers.Spdx22SbomParser; +using SPDX30= Microsoft.Sbom.Parsers.Spdx30SbomParser; /// /// MSBuild task for generating SBOMs from build output. @@ -50,10 +51,12 @@ public GenerateSbom() .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() + .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton() - .AddSingleton() + .AddSingleton() + .AddSingleton() .AddSingleton() .AddSingleton()) .Build(); diff --git a/src/Microsoft.Sbom.Tool/ValidationService.cs b/src/Microsoft.Sbom.Tool/ValidationService.cs index 73eb4f51b..f2b60e24b 100644 --- a/src/Microsoft.Sbom.Tool/ValidationService.cs +++ b/src/Microsoft.Sbom.Tool/ValidationService.cs @@ -2,6 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Hosting; @@ -41,13 +42,13 @@ public async Task StartAsync(CancellationToken cancellationToken) bool result; try { - if (configuration.ManifestInfo.Value.Contains(SpdxConstants.SPDX22ManifestInfo)) + if (configuration.ManifestInfo.Value.Any(SpdxConstants.SupportedSpdxManifests.Contains)) { result = await parserValidationWorkflow.RunAsync(); } else { - throw new ConfigurationException($"Validation only supports the SPDX2.2 format."); + throw new ConfigurationException($"Validation only supports the SPDX2.2 and SPDX3.0 format."); } await recorder.FinalizeAndLogTelemetryAsync(); diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs index 5141be562..4900a5d29 100644 --- a/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs @@ -78,7 +78,10 @@ public void Setup() } }); relationshipGeneratorMock.CallBase = true; - relationshipsArrayGenerator = new RelationshipsArrayGenerator(relationshipGeneratorMock.Object, new ChannelUtils(), loggerMock.Object, recorderMock.Object); + relationshipsArrayGenerator = new RelationshipsArrayGenerator(relationshipGeneratorMock.Object, new ChannelUtils(), loggerMock.Object, recorderMock.Object) + { + SbomConfig = sbomConfig + }; manifestGeneratorProvider.Init(); fileSystemUtilsMock.Setup(f => f.CreateDirectory(ManifestJsonDirPath)); From 08afffd6c4724d744f27206a11049caac24c4c9a Mon Sep 17 00:00:00 2001 From: ppandrate Date: Fri, 14 Feb 2025 16:40:43 -0800 Subject: [PATCH 15/15] fix manifestinfovalidator, remove unnecessary changes --- .../Config/ConfigSanitizer.cs | 4 +- .../Validators/ManifestInfoValidator.cs | 2 +- .../Executors/Spdx2SerializationStrategy.cs | 48 ++++++++++++++++--- .../Filters/DownloadedRootPathFilter.cs | 6 +-- .../Helpers/PackageArrayGenerator.cs | 2 +- .../Helpers/RelationshipsArrayGenerator.cs | 2 - .../IManifestGenerator.cs | 16 +++++++ .../Utility/GeneratedSbomValidator.cs | 2 +- 8 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs b/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs index 3a17c6911..fcba2b6cc 100644 --- a/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs +++ b/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs @@ -83,8 +83,8 @@ public IConfiguration SanitizeConfig(IConfiguration configuration) if (configuration2 is not null) { // Prevent null value for LicenseInformationTimeoutInSeconds. - // Values of (0, SpdxConstants.MaxLicenseFetchTimeoutInSeconds] are allowed. Negative values are replaced with the default, and - // the higher values are truncated to the maximum of Common.SpdxConstants.MaxLicenseFetchTimeoutInSeconds + // Values of (0, MaxLicenseFetchTimeoutInSeconds] are allowed. Negative values are replaced with the default, and + // the higher values are truncated to the maximum of MaxLicenseFetchTimeoutInSeconds if (configuration2.LicenseInformationTimeoutInSeconds is null) { configuration2.LicenseInformationTimeoutInSeconds = new(SbomConstants.DefaultLicenseFetchTimeoutInSeconds, SettingSource.Default); diff --git a/src/Microsoft.Sbom.Api/Config/Validators/ManifestInfoValidator.cs b/src/Microsoft.Sbom.Api/Config/Validators/ManifestInfoValidator.cs index 69ced2804..b0bb29494 100644 --- a/src/Microsoft.Sbom.Api/Config/Validators/ManifestInfoValidator.cs +++ b/src/Microsoft.Sbom.Api/Config/Validators/ManifestInfoValidator.cs @@ -28,7 +28,7 @@ public ManifestInfoValidator(Type supportedAttribute, IAssemblyConfig assemblyCo public override void ValidateInternal(string paramName, object paramValue, Attribute attribute) { - if (paramValue is not null && !SpdxConstants.SupportedSpdxManifests.Contains(paramValue as ManifestInfo)) + if (paramValue is not null && paramValue is ManifestInfo manifestInfo && !SpdxConstants.SupportedSpdxManifests.Contains(paramValue as ManifestInfo)) { throw new ValidationArgException($"The value of {paramName} must be a valid ManifestInfo. Supported SPDX versions include 2.2 and 3.0."); } diff --git a/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs index 2e160b146..456877550 100644 --- a/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs +++ b/src/Microsoft.Sbom.Api/Executors/Spdx2SerializationStrategy.cs @@ -78,27 +78,63 @@ public async Task> WriteJsonObjectsToSbomAsync( var errors = new List(); - // Files section + await WriteFiles(fileArrayGenerator); + + await WritePackages(packageArrayGenerator); + + await WriteExternalDocRefs(externalDocumentReferenceGenerator); + + await WriteRelationships(relationshipsArrayGenerator); + + return errors; + } + + /// + /// Write to Files section + /// + /// + /// + private async Task WriteFiles(IJsonArrayGenerator fileArrayGenerator) + { var filesGenerateResult = await fileArrayGenerator.GenerateAsync(); filesGenerateResult.Errors.AddRange(filesGenerateResult.Errors); WriteJsonObjectsFromGenerationResult(filesGenerateResult, fileArrayGenerator.SbomConfig); + } - // Packages section + /// + /// Write to Packages section + /// + /// + /// + private async Task WritePackages(IJsonArrayGenerator packageArrayGenerator) + { var packagesGenerateResult = await packageArrayGenerator.GenerateAsync(); packagesGenerateResult.Errors.AddRange(packagesGenerateResult.Errors); WriteJsonObjectsFromGenerationResult(packagesGenerateResult, packageArrayGenerator.SbomConfig); + } - // External Document Reference section + /// + /// Write to External Document Reference section + /// + /// + /// + private async Task WriteExternalDocRefs(IJsonArrayGenerator externalDocumentReferenceGenerator) + { var externalDocumentReferenceGenerateResult = await externalDocumentReferenceGenerator.GenerateAsync(); externalDocumentReferenceGenerateResult.Errors.AddRange(externalDocumentReferenceGenerateResult.Errors); WriteJsonObjectsFromGenerationResult(externalDocumentReferenceGenerateResult, externalDocumentReferenceGenerator.SbomConfig); + } - // Relationships section + /// + /// Write to Relationships section + /// + /// + /// + private async Task WriteRelationships(IJsonArrayGenerator relationshipsArrayGenerator) + { var relationshipGenerateResult = await relationshipsArrayGenerator.GenerateAsync(); relationshipGenerateResult.Errors.AddRange(relationshipGenerateResult.Errors); WriteJsonObjectsFromGenerationResult(relationshipGenerateResult, relationshipsArrayGenerator.SbomConfig); - - return errors; } private void WriteJsonObjectsFromGenerationResult(GenerationResult generationResult, ISbomConfig sbomConfig) diff --git a/src/Microsoft.Sbom.Api/Filters/DownloadedRootPathFilter.cs b/src/Microsoft.Sbom.Api/Filters/DownloadedRootPathFilter.cs index ea6f7bf7f..6ab1526ed 100644 --- a/src/Microsoft.Sbom.Api/Filters/DownloadedRootPathFilter.cs +++ b/src/Microsoft.Sbom.Api/Filters/DownloadedRootPathFilter.cs @@ -83,11 +83,9 @@ public void Init() validPaths = new HashSet(); var relativeRootPaths = configuration.RootPathFilter.Value.Split(';'); - var r = relativeRootPaths.Select(r => + validPaths.UnionWith(relativeRootPaths.Select(r => new FileInfo(fileSystemUtils.JoinPaths(configuration.BuildDropPath.Value, r)) - .FullName); - - validPaths.UnionWith(r); + .FullName)); foreach (var validPath in validPaths) { diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs index 5a2d752c9..2d2bf272b 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs @@ -38,9 +38,9 @@ public PackageArrayGenerator( ISbomConfigProvider sbomConfigs) { this.log = log ?? throw new ArgumentNullException(nameof(log)); + this.sbomConfigs = sbomConfigs ?? throw new ArgumentNullException(nameof(sbomConfigs)); this.sourcesProviders = sourcesProviders ?? throw new ArgumentNullException(nameof(sourcesProviders)); this.recorder = recorder ?? throw new ArgumentNullException(nameof(recorder)); - this.sbomConfigs = sbomConfigs ?? throw new ArgumentNullException(nameof(sbomConfigs)); } public async Task GenerateAsync() diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs index bffe009b5..1fb3f3588 100644 --- a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs @@ -28,8 +28,6 @@ public class RelationshipsArrayGenerator : IJsonArrayGenerator string Version { get; } + /// + /// Gets the value of the header to use where the files section of the SBOM will be placed. + /// If this is not supported, this method should throw a . + /// string FilesArrayHeaderName { get; } + /// + /// Gets the value of the header to use where the packages section of the SBOM will be placed. + /// If this is not supported, this method should throw a . + /// string PackagesArrayHeaderName { get; } + /// + /// Gets the value of the header where the relationship data about this SBOM will be placed. + /// If this is not supported, this method should throw a . + /// string RelationshipsArrayHeaderName { get; } + /// + /// Gets the value of the header where the external document reference data about this SBOM will be placed. + /// If this is not supported, this method should throw a . + /// string ExternalDocumentRefArrayHeaderName { get; } /// diff --git a/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs b/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs index c622a7831..62ed5e39a 100644 --- a/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs +++ b/test/Microsoft.Sbom.Targets.Tests/Utility/GeneratedSbomValidator.cs @@ -99,7 +99,7 @@ internal void AssertSbomIsValid(string manifestPath, string buildDropPath, strin } else { - Assert.Fail("An unexpected SBOM specification was used. Please specify a valid SPDX version. Current supported versions are 2.2 or 3.0."); + Assert.Fail("An unexpected SBOM specification was used. Please specify a valid SPDX version. Current supported versions are 2.2 and 3.0."); } }