Merge pull request DependencyTrack#3621 from fnxpt/experimental

support for experimental configurations
rkg-mm · Apr 22, 2024 · ee775e3 · ee775e3
2 parents 6a40954 + 2f4a9d9
commit ee775e3
Show file tree

Hide file tree

Showing 9 changed files with 96 additions and 35 deletions.
diff --git a/docs/_posts/2024-xx-xx-v4.11.0.md b/docs/_posts/2024-xx-xx-v4.11.0.md
@@ -10,14 +10,12 @@ overhauled to be more reliable and efficient. Further, BOM processing is now an
 occurring midway do not cause a partial state to be left behind. De-duplication of components and services is more
 predictable, and log messages emitted during processing contain additional context, making them easier to correlate.
 Because the new implementation can have a big impact on how Dependency-Track behaves regarding BOM uploads,
-it is disabled by default for this release. It may be enabled by setting the environment variable `BOM_PROCESSING_TASK_V2_ENABLED`
-to `true`. Users are highly encouraged to do so.
+it is disabled by default for this release. It may be enabled in the experimental panel in administration.
 * **BOM Validation**. Historically, Dependency-Track did not validate uploaded BOMs and VEXs against the CycloneDX
 schema. While this allowed BOMs to be processed that did not strictly adhere to the schema, it could also lead to confusion
 when uploaded files were accepted, but then failed to be ingested during asynchronous processing. Starting with this
 release, uploaded files will be rejected if they fail schema validation. Note that this may reveal issues in BOM
-generators that currently produce invalid CycloneDX documents. Validation may be turned off by setting the
-environment variable `BOM_VALIDATION_ENABLED` to `false`.
+generators that currently produce invalid CycloneDX documents. Validation may be turned off by disabling it in the experimental panel in administration.
   * *This feature was demoed in our April community meeting! Watch it [here](https://www.youtube.com/watch?v=3iIeajRJK8o&t=450s)*
 * **Global Vulnerability Audit View**. This new interface allows users to discover and filter vulnerabilities that affect
 their portfolio, across all projects. When portfolio access control is enabled, this view is limited to projects a user

diff --git a/src/main/java/org/dependencytrack/common/ConfigKey.java b/src/main/java/org/dependencytrack/common/ConfigKey.java
@@ -39,9 +39,7 @@ public enum ConfigKey implements Config.Key {
     REPO_META_ANALYZER_CACHE_STAMPEDE_BLOCKER_ENABLED("repo.meta.analyzer.cacheStampedeBlocker.enabled", true),
     REPO_META_ANALYZER_CACHE_STAMPEDE_BLOCKER_LOCK_BUCKETS("repo.meta.analyzer.cacheStampedeBlocker.lock.buckets", 1000),
     REPO_META_ANALYZER_CACHE_STAMPEDE_BLOCKER_MAX_ATTEMPTS("repo.meta.analyzer.cacheStampedeBlocker.max.attempts", 10),
-    SYSTEM_REQUIREMENT_CHECK_ENABLED("system.requirement.check.enabled", true),
-    BOM_PROCESSING_TASK_V2_ENABLED("bom.processing.task.v2.enabled", false),
-    BOM_VALIDATION_ENABLED("bom.validation.enabled", true);
+    SYSTEM_REQUIREMENT_CHECK_ENABLED("system.requirement.check.enabled", true);
 
     private final String propertyName;
     private final Object defaultValue;

diff --git a/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java b/src/main/java/org/dependencytrack/event/EventSubsystemInitializer.java
@@ -18,15 +18,12 @@
  */
 package org.dependencytrack.event;
 
-import alpine.Config;
 import alpine.common.logging.Logger;
 import alpine.event.LdapSyncEvent;
 import alpine.event.framework.EventService;
 import alpine.event.framework.SingleThreadedEventService;
 import alpine.server.tasks.LdapSyncTask;
 import org.dependencytrack.RequirementsVerifier;
-import org.dependencytrack.common.ConfigKey;
-import org.dependencytrack.tasks.BomUploadProcessingTask;
 import org.dependencytrack.tasks.BomUploadProcessingTaskV2;
 import org.dependencytrack.tasks.CallbackTask;
 import org.dependencytrack.tasks.ClearComponentAnalysisCacheTask;
@@ -87,11 +84,8 @@ public void contextInitialized(final ServletContextEvent event) {
         if (RequirementsVerifier.failedValidation()) {
             return;
         }
-        if (Config.getInstance().getPropertyAsBoolean(ConfigKey.BOM_PROCESSING_TASK_V2_ENABLED)) {
-            EVENT_SERVICE.subscribe(BomUploadEvent.class, BomUploadProcessingTaskV2.class);
-        } else {
-            EVENT_SERVICE.subscribe(BomUploadEvent.class, BomUploadProcessingTask.class);
-        }
+
+        EVENT_SERVICE.subscribe(BomUploadEvent.class, BomUploadProcessingTaskV2.class);
         EVENT_SERVICE.subscribe(VexUploadEvent.class, VexUploadProcessingTask.class);
         EVENT_SERVICE.subscribe(LdapSyncEvent.class, LdapSyncTask.class);
         EVENT_SERVICE.subscribe(InternalAnalysisEvent.class, InternalAnalysisTask.class);
@@ -135,11 +129,8 @@ public void contextDestroyed(final ServletContextEvent event) {
         LOGGER.info("Shutting down asynchronous event subsystem");
         TaskScheduler.getInstance().shutdown();
 
-        if (Config.getInstance().getPropertyAsBoolean(ConfigKey.BOM_PROCESSING_TASK_V2_ENABLED)) {
-            EVENT_SERVICE.unsubscribe(BomUploadProcessingTaskV2.class);
-        } else {
-            EVENT_SERVICE.unsubscribe(BomUploadProcessingTask.class);
-        }
+
+        EVENT_SERVICE.unsubscribe(BomUploadProcessingTaskV2.class);
         EVENT_SERVICE.unsubscribe(VexUploadProcessingTask.class);
         EVENT_SERVICE.unsubscribe(LdapSyncTask.class);
         EVENT_SERVICE.unsubscribe(InternalAnalysisTask.class);

diff --git a/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java b/src/main/java/org/dependencytrack/model/ConfigPropertyConstants.java
@@ -108,7 +108,9 @@ public enum ConfigPropertyConstants {
     TASK_SCHEDULER_COMPONENT_ANALYSIS_CACHE_CLEAR_CADENCE("task-scheduler", "component.analysis.cache.clear.cadence", "24", PropertyType.INTEGER, "Cleanup cadence (in hours) for component analysis cache"),
     SEARCH_INDEXES_CONSISTENCY_CHECK_ENABLED("search-indexes", "consistency.check.enabled", "true", PropertyType.BOOLEAN, "Flag to enable lucene indexes periodic consistency check"),
     SEARCH_INDEXES_CONSISTENCY_CHECK_CADENCE("search-indexes", "consistency.check.cadence", "4320", PropertyType.INTEGER, "Lucene indexes consistency check cadence (in minutes)"),
-    SEARCH_INDEXES_CONSISTENCY_CHECK_DELTA_THRESHOLD("search-indexes", "consistency.check.delta.threshold", "20", PropertyType.INTEGER, "Threshold used to trigger an index rebuild when comparing database table and corresponding lucene index (in percentage). It must be an integer between 1 and 100");
+    SEARCH_INDEXES_CONSISTENCY_CHECK_DELTA_THRESHOLD("search-indexes", "consistency.check.delta.threshold", "20", PropertyType.INTEGER, "Threshold used to trigger an index rebuild when comparing database table and corresponding lucene index (in percentage). It must be an integer between 1 and 100"),
+    BOM_PROCESSING_TASK_V2_ENABLED("experimental", "bom.processing.task.v2.enabled", "false", PropertyType.BOOLEAN, "Flag to enable BOM UPLOAD V2"),
+    BOM_VALIDATION_ENABLED("artifact", "bom.validation.enabled", "true", PropertyType.BOOLEAN, "Flag to control bom validation");
 
     private String groupName;
     private String propertyName;

diff --git a/src/main/java/org/dependencytrack/resources/v1/BomResource.java b/src/main/java/org/dependencytrack/resources/v1/BomResource.java
@@ -18,7 +18,6 @@
  */
 package org.dependencytrack.resources.v1;
 
-import alpine.Config;
 import alpine.common.logging.Logger;
 import alpine.event.framework.Event;
 import alpine.server.auth.PermissionRequired;
@@ -35,9 +34,9 @@
 import org.cyclonedx.CycloneDxMediaType;
 import org.cyclonedx.exception.GeneratorException;
 import org.dependencytrack.auth.Permissions;
-import org.dependencytrack.common.ConfigKey;
 import org.dependencytrack.event.BomUploadEvent;
 import org.dependencytrack.model.Component;
+import org.dependencytrack.model.ConfigPropertyConstants;
 import org.dependencytrack.model.Project;
 import org.dependencytrack.model.validation.ValidUuid;
 import org.dependencytrack.parser.cyclonedx.CycloneDXExporter;
@@ -468,8 +467,10 @@ private Response process(QueryManager qm, Project project, List<FormDataBodyPart
     }
 
     static void validate(final byte[] bomBytes) {
-        if (!Config.getInstance().getPropertyAsBoolean(ConfigKey.BOM_VALIDATION_ENABLED)) {
-            return;
+        try (QueryManager qm = new QueryManager()) {
+            if (!qm.isEnabled(ConfigPropertyConstants.BOM_VALIDATION_ENABLED)) {
+                return;
+            }
         }
 
         try {

diff --git a/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTaskV2.java b/src/main/java/org/dependencytrack/tasks/BomUploadProcessingTaskV2.java
@@ -41,6 +41,7 @@
 import org.dependencytrack.model.Bom;
 import org.dependencytrack.model.Component;
 import org.dependencytrack.model.ComponentIdentity;
+import org.dependencytrack.model.ConfigPropertyConstants;
 import org.dependencytrack.model.DependencyMetrics;
 import org.dependencytrack.model.FindingAttribution;
 import org.dependencytrack.model.License;
@@ -144,6 +145,15 @@ public void inform(final Event e) {
             return;
         }
 
+        //EXPERIMENTAL: FUTURE RELEASES SHOULD JUST REMOVE THE FOLLOWING BLOCK
+        try (QueryManager qm = new QueryManager()) {
+            if (!qm.isEnabled(ConfigPropertyConstants.BOM_PROCESSING_TASK_V2_ENABLED)) {
+                new BomUploadProcessingTask().inform(event);
+                return;
+            }
+        }
+        //EXPERIMENTAL
+
         final var ctx = new Context(event.getChainIdentifier(), event.getProject(), event.getBom());
         try (var ignoredMdcProjectUuid = MDC.putCloseable(MDC_PROJECT_UUID, ctx.project.getUuid().toString());
              var ignoredMdcProjectName = MDC.putCloseable(MDC_PROJECT_NAME, ctx.project.getName());

diff --git a/src/test/java/org/dependencytrack/resources/v1/BomResourceTest.java b/src/test/java/org/dependencytrack/resources/v1/BomResourceTest.java
@@ -64,6 +64,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatNoException;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.dependencytrack.model.ConfigPropertyConstants.BOM_VALIDATION_ENABLED;
 
 public class BomResourceTest extends ResourceTest {
 
@@ -871,6 +872,14 @@ public void uploadBomInvalidParentTest() throws Exception {
     public void uploadBomInvalidJsonTest() {
         initializeWithPermissions(Permissions.BOM_UPLOAD);
 
+        qm.createConfigProperty(
+                BOM_VALIDATION_ENABLED.getGroupName(),
+                BOM_VALIDATION_ENABLED.getPropertyName(),
+                "true",
+                BOM_VALIDATION_ENABLED.getPropertyType(),
+                null
+        );
+
         final var project = new Project();
         project.setName("acme-app");
         project.setVersion("1.0.0");
@@ -919,6 +928,14 @@ public void uploadBomInvalidJsonTest() {
     public void uploadBomInvalidXmlTest() {
         initializeWithPermissions(Permissions.BOM_UPLOAD);
 
+        qm.createConfigProperty(
+          BOM_VALIDATION_ENABLED.getGroupName(),
+          BOM_VALIDATION_ENABLED.getPropertyName(),
+          "true",
+          BOM_VALIDATION_ENABLED.getPropertyType(),
+          null
+        );
+
         final var project = new Project();
         project.setName("acme-app");
         project.setVersion("1.0.0");

diff --git a/src/test/java/org/dependencytrack/resources/v1/VexResourceTest.java b/src/test/java/org/dependencytrack/resources/v1/VexResourceTest.java
@@ -49,6 +49,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatNoException;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.dependencytrack.model.ConfigPropertyConstants.BOM_VALIDATION_ENABLED;
 
 public class VexResourceTest extends ResourceTest {
 
@@ -219,6 +220,14 @@ public void exportProjectAsCycloneDxTest() {
     public void uploadVexInvalidJsonTest() {
         initializeWithPermissions(Permissions.BOM_UPLOAD);
 
+        qm.createConfigProperty(
+          BOM_VALIDATION_ENABLED.getGroupName(),
+          BOM_VALIDATION_ENABLED.getPropertyName(),
+          "true",
+          BOM_VALIDATION_ENABLED.getPropertyType(),
+          null
+        );
+
         final var project = new Project();
         project.setName("acme-app");
         project.setVersion("1.0.0");
@@ -267,6 +276,14 @@ public void uploadVexInvalidJsonTest() {
     public void uploadVexInvalidXmlTest() {
         initializeWithPermissions(Permissions.BOM_UPLOAD);
 
+        qm.createConfigProperty(
+          BOM_VALIDATION_ENABLED.getGroupName(),
+          BOM_VALIDATION_ENABLED.getPropertyName(),
+          "true",
+          BOM_VALIDATION_ENABLED.getPropertyType(),
+          null
+        );
+
         final var project = new Project();
         project.setName("acme-app");
         project.setVersion("1.0.0");

diff --git a/src/test/java/org/dependencytrack/tasks/BomUploadProcessingTaskTest.java b/src/test/java/org/dependencytrack/tasks/BomUploadProcessingTaskTest.java
@@ -72,6 +72,8 @@
 import static org.assertj.core.api.Assertions.fail;
 import static org.awaitility.Awaitility.await;
 import static org.dependencytrack.assertion.Assertions.assertConditionWithTimeout;
+import static org.dependencytrack.model.ConfigPropertyConstants.BOM_VALIDATION_ENABLED;
+import static org.dependencytrack.model.ConfigPropertyConstants.BOM_PROCESSING_TASK_V2_ENABLED;
 
 @RunWith(Parameterized.class)
 public class BomUploadProcessingTaskTest extends PersistenceCapableTest {
@@ -100,15 +102,18 @@ public void inform(final Notification notification) {
     @Parameterized.Parameters(name = "{index}: {0}")
     public static Collection<Object[]> data() {
         return Arrays.asList(new Object[][]{
-                {BomUploadProcessingTask.class.getSimpleName(), (Supplier<alpine.event.framework.Subscriber>) BomUploadProcessingTask::new},
+                {BomUploadProcessingTask.class.getSimpleName(), (Supplier<alpine.event.framework.Subscriber>) BomUploadProcessingTaskV2::new},
                 {BomUploadProcessingTaskV2.class.getSimpleName(), (Supplier<alpine.event.framework.Subscriber>) BomUploadProcessingTaskV2::new}
         });
     }
 
+    private final String ignoredBomUploadProcessingTaskName;
     private final Supplier<alpine.event.framework.Subscriber> bomUploadProcessingTaskSupplier;
 
     public BomUploadProcessingTaskTest(final String ignoredBomUploadProcessingTaskName,
                                        final Supplier<alpine.event.framework.Subscriber> bomUploadProcessingTaskSupplier) {
+
+        this.ignoredBomUploadProcessingTaskName = ignoredBomUploadProcessingTaskName;
         this.bomUploadProcessingTaskSupplier = bomUploadProcessingTaskSupplier;
     }
 
@@ -119,6 +124,14 @@ public void setUp() {
         EventService.getInstance().subscribe(VulnerabilityAnalysisEvent.class, EventSubscriber.class);
         NotificationService.getInstance().subscribe(new Subscription(NotificationSubscriber.class));
 
+        qm.createConfigProperty(
+          BOM_PROCESSING_TASK_V2_ENABLED.getGroupName(),
+          BOM_PROCESSING_TASK_V2_ENABLED.getPropertyName(),
+          (this.ignoredBomUploadProcessingTaskName == BomUploadProcessingTaskV2.class.getSimpleName()) ? "true" : "false",
+          BOM_PROCESSING_TASK_V2_ENABLED.getPropertyType(),
+          null
+        );
+
         // Enable processing of CycloneDX BOMs
         qm.createConfigProperty(ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getGroupName(),
                 ConfigPropertyConstants.ACCEPT_ARTIFACT_CYCLONEDX.getPropertyName(), "true",
@@ -287,7 +300,7 @@ public void informTest() throws Exception {
     @Test
     public void informWithEmptyBomTest() throws Exception {
         // Known to now work with old task implementation.
-        if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) {
+        if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) {
             return;
         }
 
@@ -308,6 +321,20 @@ public void informWithEmptyBomTest() throws Exception {
 
     @Test
     public void informWithInvalidCycloneDxBomTest() throws Exception {
+
+        // Known to now work with old task implementation.
+        if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) {
+          return;
+        }
+
+        qm.createConfigProperty(
+          BOM_VALIDATION_ENABLED.getGroupName(),
+          BOM_VALIDATION_ENABLED.getPropertyName(),
+          "true",
+          BOM_VALIDATION_ENABLED.getPropertyType(),
+          null
+        );
+
         final Project project = qm.createProject("Acme Example", null, "1.0", null, null, null, true, false);
 
         final byte[] bomBytes = """
@@ -344,7 +371,7 @@ public void informWithInvalidCycloneDxBomTest() throws Exception {
     @Test
     public void informWithNonExistentProjectTest() throws Exception {
         // Known to now work with old task implementation.
-        if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) {
+        if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) {
             return;
         }
 
@@ -368,7 +395,7 @@ public void informWithNonExistentProjectTest() throws Exception {
     @Test
     public void informWithComponentsUnderMetadataBomTest() throws Exception {
         // Known to now work with old task implementation.
-        if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) {
+        if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) {
             return;
         }
 
@@ -404,7 +431,7 @@ public void informWithComponentsUnderMetadataBomTest() throws Exception {
     @Test
     public void informWithExistingDuplicateComponentsTest() {
         // Known to now work with old task implementation.
-        if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) {
+        if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) {
             return;
         }
 
@@ -490,7 +517,7 @@ public void informWithExistingDuplicateComponentsTest() {
     @Test
     public void informWithBloatedBomTest() throws Exception {
         // Known to now work with old task implementation.
-        if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) {
+        if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) {
             return;
         }
 
@@ -546,7 +573,7 @@ public void informWithBloatedBomTest() throws Exception {
     @Test
     public void informWithCustomLicenseResolutionTest() throws Exception {
         // Known to now work with old task implementation.
-        if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) {
+        if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) {
             return;
         }
 
@@ -1021,7 +1048,7 @@ public void informWithExistingComponentPropertiesAndBomWithComponentProperties()
     @Test // https://github.com/DependencyTrack/dependency-track/issues/1905
     public void informIssue1905Test() throws Exception {
         // Known to now work with old task implementation.
-        if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) {
+        if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) {
             return;
         }
 
@@ -1060,7 +1087,7 @@ public void informIssue1905Test() throws Exception {
     @Test // https://github.com/DependencyTrack/dependency-track/issues/2519
     public void informIssue2519Test() throws Exception {
         // Known to now work with old task implementation.
-        if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) {
+        if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) {
             return;
         }
 
@@ -1138,7 +1165,7 @@ public void informIssue3309Test() {
     @Test // https://github.com/DependencyTrack/dependency-track/issues/3371
     public void informIssue3371Test() throws Exception {
         // Known to now work with old task implementation.
-        if (bomUploadProcessingTaskSupplier.get() instanceof BomUploadProcessingTask) {
+        if (ignoredBomUploadProcessingTaskName == BomUploadProcessingTask.class.getSimpleName()) {
             return;
         }