From 77b6bcbcd5ee6c700d968a583407e5c5f525016d Mon Sep 17 00:00:00 2001 From: nscuro Date: Mon, 5 Aug 2024 14:33:58 +0200 Subject: [PATCH] Add e2e test for S3 BOM upload storage Relates to #633 Signed-off-by: nscuro --- .../org/dependencytrack/e2e/AbstractE2ET.java | 2 +- .../BomUploadProcessingWithS3StorageE2ET.java | 127 ++++++++++++++++++ .../e2e/VulnerabilityPolicyE2ET.java | 2 +- scripts/create-topics.sh | 1 + 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 e2e/src/test/java/org/dependencytrack/e2e/BomUploadProcessingWithS3StorageE2ET.java diff --git a/e2e/src/test/java/org/dependencytrack/e2e/AbstractE2ET.java b/e2e/src/test/java/org/dependencytrack/e2e/AbstractE2ET.java index be46d4a5a..7817379b0 100644 --- a/e2e/src/test/java/org/dependencytrack/e2e/AbstractE2ET.java +++ b/e2e/src/test/java/org/dependencytrack/e2e/AbstractE2ET.java @@ -317,7 +317,7 @@ private ApiServerClient initializeApiServerClient() { } @AfterEach - void afterEach() { + void afterEach() throws Exception { ApiServerAuthInterceptor.reset(); Optional.ofNullable(vulnAnalyzerContainer).ifPresent(GenericContainer::stop); diff --git a/e2e/src/test/java/org/dependencytrack/e2e/BomUploadProcessingWithS3StorageE2ET.java b/e2e/src/test/java/org/dependencytrack/e2e/BomUploadProcessingWithS3StorageE2ET.java new file mode 100644 index 000000000..7f0d791b5 --- /dev/null +++ b/e2e/src/test/java/org/dependencytrack/e2e/BomUploadProcessingWithS3StorageE2ET.java @@ -0,0 +1,127 @@ +/* + * This file is part of Dependency-Track. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + * Copyright (c) OWASP Foundation. All Rights Reserved. + */ +package org.dependencytrack.e2e; + +import io.minio.ListObjectsArgs; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.Result; +import io.minio.messages.Item; +import org.apache.commons.io.IOUtils; +import org.dependencytrack.apiserver.model.BomProcessingResponse; +import org.dependencytrack.apiserver.model.BomUploadRequest; +import org.dependencytrack.apiserver.model.WorkflowTokenResponse; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.MinIOContainer; +import org.testcontainers.utility.DockerImageName; + +import java.time.Duration; +import java.util.Base64; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +class BomUploadProcessingWithS3StorageE2ET extends AbstractE2ET { + + private static final String BOM_UPLOAD_BUCKET_NAME = "bom-upload"; + + private MinIOContainer minioContainer; + private MinioClient minioClient; + + @Override + @BeforeEach + void beforeEach() throws Exception { + minioContainer = new MinIOContainer(DockerImageName.parse("minio/minio:RELEASE.2023-12-14T18-51-57Z")) + .withNetworkAliases("minio") + .withNetwork(internalNetwork); + minioContainer.start(); + + minioClient = MinioClient.builder() + .endpoint(minioContainer.getS3URL()) + .credentials(minioContainer.getUserName(), minioContainer.getPassword()) + .build(); + createBucket(); + + super.beforeEach(); + } + + @Override + protected void customizeApiServerContainer(final GenericContainer container) { + container + .withEnv("BOM_UPLOAD_STORAGE_DEFAULT_EXTENSION", "s3") + .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_S3_ENABLED", "true") + .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_S3_ENDPOINT", "http://minio:9000") + .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_S3_BUCKET", BOM_UPLOAD_BUCKET_NAME) + .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_S3_ACCESS_KEY", minioContainer.getUserName()) + .withEnv("BOM_UPLOAD_STORAGE_EXTENSION_S3_SECRET_KEY", minioContainer.getPassword()); + } + + @Override + @AfterEach + void afterEach() throws Exception { + if (minioClient != null) { + minioClient.close(); + } + Optional.ofNullable(minioContainer).ifPresent(GenericContainer::stop); + + super.afterEach(); + } + + @Test + void test() throws Exception { + // Parse and base64 encode a BOM. + final byte[] bomBytes = IOUtils.resourceToByteArray("/dtrack-apiserver-4.5.0.bom.json"); + final String bomBase64 = Base64.getEncoder().encodeToString(bomBytes); + + // Upload the BOM. + final WorkflowTokenResponse response = apiServerClient.uploadBom(new BomUploadRequest("foo", "bar", true, bomBase64)); + assertThat(response.token()).isNotEmpty(); + + // Wait up to 15sec for the BOM processing to complete. + await("BOM processing") + .atMost(Duration.ofSeconds(15)) + .pollDelay(Duration.ofMillis(250)) + .untilAsserted(() -> { + final BomProcessingResponse processingResponse = apiServerClient.isBomBeingProcessed(response.token()); + assertThat(processingResponse.processing()).isFalse(); + }); + + verifyBomDeleted(); + } + + private void createBucket() throws Exception { + minioClient.makeBucket(MakeBucketArgs.builder() + .bucket(BOM_UPLOAD_BUCKET_NAME) + .build()); + } + + private void verifyBomDeleted() { + final Iterable> results = minioClient.listObjects(ListObjectsArgs.builder() + .bucket(BOM_UPLOAD_BUCKET_NAME) + .recursive(true) + .build()); + + assertThat(results).isEmpty(); + } + +} diff --git a/e2e/src/test/java/org/dependencytrack/e2e/VulnerabilityPolicyE2ET.java b/e2e/src/test/java/org/dependencytrack/e2e/VulnerabilityPolicyE2ET.java index 1fdd8d5be..8a885b855 100644 --- a/e2e/src/test/java/org/dependencytrack/e2e/VulnerabilityPolicyE2ET.java +++ b/e2e/src/test/java/org/dependencytrack/e2e/VulnerabilityPolicyE2ET.java @@ -172,7 +172,7 @@ protected void customizeVulnAnalyzerContainer(final GenericContainer containe @Override @AfterEach - void afterEach() { + void afterEach() throws Exception { Optional.ofNullable(minioContainer).ifPresent(GenericContainer::stop); super.afterEach(); diff --git a/scripts/create-topics.sh b/scripts/create-topics.sh index 721ff6c4f..0d596e726 100644 --- a/scripts/create-topics.sh +++ b/scripts/create-topics.sh @@ -77,6 +77,7 @@ for topic_name in "${vuln_analysis_topics[@]}"; do create_topic "$topic_name" "${VULN_ANALYSIS_TOPICS_PARTITIONS:-3}" "retention.ms=${VULN_ANALYSIS_TOPICS_RETENTION_MS:-43200000}" done +create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.event.bom-uploaded" "${EVENT_BOM_UPLOADED_TOPIC_PARTITIONS:-3}" "retention.ms=${EVENT_BOM_UPLOADED_TOPIC_RETENTION_MS:-43200000}" create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vulnerability.mirror.command" "1" "retention.ms=${VULN_MIRROR_TOPICS_RETENTION_MS:-43200000}" create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vulnerability.mirror.state" "1" "cleanup.policy=compact segment.bytes=67108864 max.compaction.lag.ms=1" create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vulnerability.digest" "1" "cleanup.policy=compact segment.bytes=134217728"