From caa15fe072d601bdf22dba0121de3e3eb6e7540d Mon Sep 17 00:00:00 2001 From: Paul Latzelsperger Date: Thu, 28 Dec 2023 13:44:23 +0100 Subject: [PATCH] move tests to use the JUnit runner --- .github/workflows/verify.yaml | 17 +---- build.gradle.kts | 7 ++ .../end2end-test/catalog-runtime/README.md | 0 .../filesystem/CatalogExtension.java | 48 -------------- .../filesystem/FileBasedNodeDirectory.java | 65 ------------------- ...rg.eclipse.edc.spi.system.ServiceExtension | 1 - system-tests/end2end-test/docker-compose.yml | 44 ------------- .../e2e-junit-runner/build.gradle.kts | 1 + .../org/eclipse/edc/end2end/Endpoint.java | 4 ++ .../edc/end2end/FederatedCatalogTest.java | 60 ++++++++++++++++- .../edc/end2end/ManagementApiClient.java | 12 ++-- .../end2end-test/resources/nodes-dc.json | 9 --- 12 files changed, 82 insertions(+), 186 deletions(-) delete mode 100644 system-tests/end2end-test/catalog-runtime/README.md delete mode 100644 system-tests/end2end-test/catalog-runtime/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/CatalogExtension.java delete mode 100644 system-tests/end2end-test/catalog-runtime/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/FileBasedNodeDirectory.java delete mode 100644 system-tests/end2end-test/catalog-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension delete mode 100644 system-tests/end2end-test/docker-compose.yml create mode 100644 system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/Endpoint.java delete mode 100644 system-tests/end2end-test/resources/nodes-dc.json diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml index 6aafe9e5..44f148c0 100644 --- a/.github/workflows/verify.yaml +++ b/.github/workflows/verify.yaml @@ -73,7 +73,7 @@ jobs: - name: Component Tests uses: ./.github/actions/run-tests with: - command: ./gradlew test jacocoTestReport -DincludeTags="ComponentTest" + command: ./gradlew test jacocoTestReport -DincludeTags="ComponentTest" -PverboseTest=true End-To-End-Tests: runs-on: ubuntu-latest @@ -83,23 +83,12 @@ jobs: - name: 'Build Launchers' run: | - ./gradlew -p system-tests/end2end-test clean shadowJar - - - name: 'Install docker compose plugin' - run: | - DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker} - mkdir -p $DOCKER_CONFIG/cli-plugins - curl -SL https://github.com/docker/compose/releases/download/v2.17.3/docker-compose-linux-x86_64 -o $DOCKER_CONFIG/cli-plugins/docker-compose - chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose - - - name: 'Start FCC Runtime in Docker' - run: docker compose -f system-tests/end2end-test/docker-compose.yml up --build --wait - timeout-minutes: 10 + ./gradlew compileJava compileTestJava - name: 'End to End Integration Tests' uses: ./.github/actions/run-tests with: - command: ./gradlew system-tests:end2end-test:e2e-junit:test -DincludeTags="EndToEndTest" + command: ./gradlew test jacocoTestReport -DincludeTags="EndToEndTest" -PverboseTest=true API-Tests: env: diff --git a/build.gradle.kts b/build.gradle.kts index 66087503..79480cc6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -61,4 +61,11 @@ allprojects { configDirectory.set(rootProject.file("resources")) } + // EdcRuntimeExtension uses this to determine the runtime classpath of the module to run. + tasks.register("printClasspath") { + doLast { + println(sourceSets["main"].runtimeClasspath.asPath) + } + } + } diff --git a/system-tests/end2end-test/catalog-runtime/README.md b/system-tests/end2end-test/catalog-runtime/README.md deleted file mode 100644 index e69de29b..00000000 diff --git a/system-tests/end2end-test/catalog-runtime/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/CatalogExtension.java b/system-tests/end2end-test/catalog-runtime/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/CatalogExtension.java deleted file mode 100644 index 6b65419f..00000000 --- a/system-tests/end2end-test/catalog-runtime/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/CatalogExtension.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.catalog.node.directory.filesystem; - -import org.eclipse.edc.crawler.spi.TargetNodeDirectory; -import org.eclipse.edc.runtime.metamodel.annotation.Inject; -import org.eclipse.edc.runtime.metamodel.annotation.Provider; -import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.spi.system.ServiceExtension; -import org.eclipse.edc.spi.system.ServiceExtensionContext; -import org.eclipse.edc.spi.types.TypeManager; -import org.eclipse.edc.util.concurrency.LockManager; - -import java.io.File; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static java.lang.String.format; -import static java.util.Optional.ofNullable; - -public class CatalogExtension implements ServiceExtension { - - private static final String FILE_LOCATION_SETTING = "fcc.directory.file"; - - @Inject - private TypeManager typeManager; - - @Provider - public TargetNodeDirectory createFileSystemDirectory(ServiceExtensionContext context) { - var setting = ofNullable(context.getSetting(FILE_LOCATION_SETTING, null)) - .orElseThrow(() -> new EdcException(format("Config property [%s] not found, will ABORT!", FILE_LOCATION_SETTING))); - - File nodeFile = new File(setting); - return new FileBasedNodeDirectory(nodeFile, new LockManager(new ReentrantReadWriteLock()), typeManager.getMapper()); - } - -} diff --git a/system-tests/end2end-test/catalog-runtime/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/FileBasedNodeDirectory.java b/system-tests/end2end-test/catalog-runtime/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/FileBasedNodeDirectory.java deleted file mode 100644 index a0f55db5..00000000 --- a/system-tests/end2end-test/catalog-runtime/src/main/java/org/eclipse/edc/catalog/node/directory/filesystem/FileBasedNodeDirectory.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (c) 2022 Microsoft Corporation - * - * This program and the accompanying materials are made available under the - * terms of the Apache License, Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - * - * Contributors: - * Microsoft Corporation - initial API and implementation - * - */ - -package org.eclipse.edc.catalog.node.directory.filesystem; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.eclipse.edc.crawler.spi.TargetNode; -import org.eclipse.edc.crawler.spi.TargetNodeDirectory; -import org.eclipse.edc.spi.EdcException; -import org.eclipse.edc.util.concurrency.LockManager; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.List; - -/** - * File-based node directory, solely intended for use in testing, specifically with docker-compose - */ -public class FileBasedNodeDirectory implements TargetNodeDirectory { - - private static final TypeReference> NODE_LIST_TYPE = new TypeReference<>() { - }; - private final List nodes = new ArrayList<>(); - private final LockManager lockManager; - private final ObjectMapper objectMapper; - - public FileBasedNodeDirectory(File nodeFile, LockManager lockManager, ObjectMapper objectMapper) { - this.lockManager = lockManager; - this.objectMapper = objectMapper; - readAll(nodeFile); - } - - @Override - public List getAll() { - return lockManager.readLock(() -> nodes); - } - - @Override - public void insert(TargetNode node) { - lockManager.writeLock(() -> nodes.add(node)); - } - - private void readAll(File nodeFile) { - try { - var content = Files.readString(nodeFile.toPath()); - nodes.addAll(objectMapper.readValue(content, NODE_LIST_TYPE)); - } catch (IOException e) { - throw new EdcException(e); - } - } -} diff --git a/system-tests/end2end-test/catalog-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/system-tests/end2end-test/catalog-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension deleted file mode 100644 index 27e69f26..00000000 --- a/system-tests/end2end-test/catalog-runtime/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension +++ /dev/null @@ -1 +0,0 @@ -org.eclipse.edc.catalog.node.directory.filesystem.CatalogExtension \ No newline at end of file diff --git a/system-tests/end2end-test/docker-compose.yml b/system-tests/end2end-test/docker-compose.yml deleted file mode 100644 index 09ade068..00000000 --- a/system-tests/end2end-test/docker-compose.yml +++ /dev/null @@ -1,44 +0,0 @@ -services: - - # standalone FCC runtime - fcc-runtime: - container_name: federated-catalog-runtime - build: - context: catalog-runtime/ - args: - JVM_ARGS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005" - environment: - EDC_CATALOG_CACHE_EXECUTION_DELAY_SECONDS: 0 - EDC_CATALOG_CACHE_EXECUTION_PERIOD_SECONDS: 2 - EDC_CATALOG_CACHE_PARTITION_NUM_CRAWLERS: 5 - EDC_IDS_ID: urn:connector:fcc - EDC_WEB_REST_CORS_ENABLED: "true" - EDC_WEB_REST_CORS_HEADERS: "origin,content-type,accept,authorization,x-api-key" - FCC_DIRECTORY_FILE: /resources/nodes-dc.json - depends_on: [ connector1 ] - volumes: - - ./resources:/resources - ports: - - "8181:8181" - - "5005:5005" - - - # dummy connector instance 1 - connector1: - container_name: connector1 - build: - context: connector-runtime/ - args: - JVM_ARGS: "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5006" - - environment: - IDS_WEBHOOK_ADDRESS: http://connector1:8282 - EDC_CONNECTOR_NAME: connector1 - EDC_IDS_ID: urn:connector:1 - EDC_WEB_REST_CORS_ENABLED: "true" - EDC_WEB_REST_CORS_HEADERS: "origin,content-type,accept,authorization,x-api-key" - EDC_DSP_CALLBACK_ADDRESS: "http://connector1:8282/api/v1/dsp" - ports: - - "8282:8282" - - "9192:9191" - - "5006:5006" diff --git a/system-tests/end2end-test/e2e-junit-runner/build.gradle.kts b/system-tests/end2end-test/e2e-junit-runner/build.gradle.kts index 16bd44de..d3011675 100644 --- a/system-tests/end2end-test/e2e-junit-runner/build.gradle.kts +++ b/system-tests/end2end-test/e2e-junit-runner/build.gradle.kts @@ -18,6 +18,7 @@ plugins { dependencies { testImplementation(project(":spi:federated-catalog-spi")) + testImplementation(project(":core:federated-catalog-core")) testImplementation(libs.awaitility) testImplementation(libs.edc.api.management) testImplementation(libs.edc.core.transform) diff --git a/system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/Endpoint.java b/system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/Endpoint.java new file mode 100644 index 00000000..735fffd9 --- /dev/null +++ b/system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/Endpoint.java @@ -0,0 +1,4 @@ +package org.eclipse.edc.end2end; + +public record Endpoint(String path, String port) { +} diff --git a/system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/FederatedCatalogTest.java b/system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/FederatedCatalogTest.java index 69f7ad7c..0f0a1681 100644 --- a/system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/FederatedCatalogTest.java +++ b/system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/FederatedCatalogTest.java @@ -18,6 +18,8 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import jakarta.json.Json; import jakarta.json.JsonBuilderFactory; +import org.eclipse.edc.catalog.directory.InMemoryNodeDirectory; +import org.eclipse.edc.catalog.spi.CatalogConstants; import org.eclipse.edc.core.transform.TypeTransformerRegistryImpl; import org.eclipse.edc.core.transform.transformer.from.JsonObjectFromCatalogTransformer; import org.eclipse.edc.core.transform.transformer.from.JsonObjectFromDataServiceTransformer; @@ -29,18 +31,24 @@ import org.eclipse.edc.core.transform.transformer.to.JsonObjectToDatasetTransformer; import org.eclipse.edc.core.transform.transformer.to.JsonObjectToDistributionTransformer; import org.eclipse.edc.core.transform.transformer.to.JsonValueToGenericTypeTransformer; +import org.eclipse.edc.crawler.spi.TargetNode; +import org.eclipse.edc.crawler.spi.TargetNodeDirectory; import org.eclipse.edc.jsonld.TitaniumJsonLd; import org.eclipse.edc.jsonld.util.JacksonJsonLd; import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.EdcRuntimeExtension; import org.eclipse.edc.spi.monitor.Monitor; import org.eclipse.edc.spi.result.Result; import org.eclipse.edc.transform.spi.TypeTransformerRegistry; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.extension.RegisterExtension; import java.time.Duration; import java.util.Base64; +import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.UUID; @@ -57,9 +65,43 @@ class FederatedCatalogTest { public static final Duration TIMEOUT = ofSeconds(60); + private static final Endpoint CONNECTOR_MANAGEMENT = new Endpoint("/management", "8081"); + private static final Endpoint CONNECTOR_PROTOCOL = new Endpoint("/api/v1/dsp", "8082"); + private static final Endpoint CONNECTOR_DEFAULT = new Endpoint("/api/v1/", "8080"); + + private static final Endpoint CATALOG_MANAGEMENT = new Endpoint("/management", "8091"); + private static final Endpoint CATALOG_PROTOCOL = new Endpoint("/api/v1/dsp", "8092"); + private static final Endpoint CATALOG_DEFAULT = new Endpoint("/api/v1/", "8090"); + + @RegisterExtension + static EdcRuntimeExtension connector = new EdcRuntimeExtension(":system-tests:end2end-test:connector-runtime", "connector", + configOf("edc.connector.name", "connector1", + "edc.web.rest.cors.enabled", "true", + "web.http.port", CONNECTOR_DEFAULT.port(), + "web.http.path", CONNECTOR_DEFAULT.path(), + "web.http.protocol.port", CONNECTOR_PROTOCOL.port(), + "web.http.protocol.path", CONNECTOR_PROTOCOL.path(), + "web.http.management.port", CONNECTOR_MANAGEMENT.port(), + "web.http.management.path", CONNECTOR_MANAGEMENT.path(), + "edc.web.rest.cors.headers", "origin,content-type,accept,authorization,x-api-key", + "edc.dsp.callback.address", "http://localhost:%s%s".formatted(CONNECTOR_PROTOCOL.port(), CONNECTOR_PROTOCOL.path()))); + + @RegisterExtension + static EdcRuntimeExtension catalog = new EdcRuntimeExtension(":system-tests:end2end-test:catalog-runtime", "catalog", + configOf("edc.catalog.cache.execution.delay.seconds", "0", + "edc.catalog.cache.execution.period.seconds", "2", + "edc.catalog.cache.partition.num.crawlers", "5", + "edc.web.rest.cors.enabled", "true", + "web.http.port", CATALOG_DEFAULT.port(), + "web.http.path", CATALOG_DEFAULT.path(), + "web.http.protocol.port", CATALOG_PROTOCOL.port(), + "web.http.protocol.path", CATALOG_PROTOCOL.path(), + "web.http.management.port", CATALOG_MANAGEMENT.port(), + "web.http.management.path", CATALOG_MANAGEMENT.path(), + "edc.web.rest.cors.headers", "origin,content-type,accept,authorization,x-api-key")); private final TypeTransformerRegistry typeTransformerRegistry = new TypeTransformerRegistryImpl(); private final ObjectMapper mapper = JacksonJsonLd.createObjectMapper(); - private final ManagementApiClient apiClient = new ManagementApiClient(mapper, new TitaniumJsonLd(mock(Monitor.class)), typeTransformerRegistry); + private final ManagementApiClient apiClient = new ManagementApiClient(CATALOG_MANAGEMENT, CONNECTOR_MANAGEMENT, mapper, new TitaniumJsonLd(mock(Monitor.class)), typeTransformerRegistry); @BeforeEach void setUp() { @@ -77,6 +119,10 @@ void setUp() { jsonObjectToOdrlTransformers().forEach(typeTransformerRegistry::register); typeTransformerRegistry.register(new JsonObjectToDistributionTransformer()); typeTransformerRegistry.register(new JsonValueToGenericTypeTransformer(mapper)); + + var directory = new InMemoryNodeDirectory(); + directory.insert(new TargetNode("connector", "http://localhost:%s%s".formatted(CONNECTOR_PROTOCOL.port(), CONNECTOR_PROTOCOL.path()), List.of(CatalogConstants.DATASPACE_PROTOCOL))); + catalog.registerServiceMock(TargetNodeDirectory.class, directory); } @Test @@ -121,6 +167,18 @@ void crawl_whenOfferAvailable_shouldContainOffer(TestInfo testInfo) { }); } + private static Map configOf(String... keyValuePairs) { + if (keyValuePairs.length % 2 != 0) { + throw new IllegalArgumentException("Must have an even number of key value pairs, was " + keyValuePairs.length); + } + + var map = new HashMap(); + for (int i = 0; i < keyValuePairs.length - 1; i += 2) { + map.put(keyValuePairs[i], keyValuePairs[i + 1]); + } + return map; + } + private String getError(Result r) { return ofNullable(r.getFailureDetail()).orElse("No error"); } diff --git a/system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/ManagementApiClient.java b/system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/ManagementApiClient.java index c05c2ca2..cd929d6b 100644 --- a/system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/ManagementApiClient.java +++ b/system-tests/end2end-test/e2e-junit-runner/src/test/java/org/eclipse/edc/end2end/ManagementApiClient.java @@ -37,19 +37,23 @@ import static java.lang.String.format; class ManagementApiClient { - private static final String MANAGEMENT_BASE_URL = "http://localhost:9192/management"; - private static final String CATALOG_BASE_URL = "http://localhost:8181/api"; private static final TypeReference>> LIST_TYPE_REFERENCE = new TypeReference<>() { }; private static final MediaType JSON = MediaType.parse("application/json"); + private final String MANAGEMENT_BASE_URL; + private final String CATALOG_BASE_URL; private final ObjectMapper mapper; private final JsonLd jsonLdService; private final TypeTransformerRegistry typeTransformerRegistry; - ManagementApiClient(ObjectMapper mapper, JsonLd jsonLdService, TypeTransformerRegistry typeTransformerRegistry) { + ManagementApiClient(Endpoint catalogManagement, Endpoint connectorManagement, + ObjectMapper mapper, JsonLd jsonLdService, + TypeTransformerRegistry typeTransformerRegistry) { this.mapper = mapper; this.jsonLdService = jsonLdService; this.typeTransformerRegistry = typeTransformerRegistry; + MANAGEMENT_BASE_URL = "http://localhost:%s%s".formatted(connectorManagement.port(), connectorManagement.path()); + CATALOG_BASE_URL = "http://localhost:%s%s".formatted(catalogManagement.port(), catalogManagement.path()); } Result postAsset(JsonObject entry) { @@ -84,7 +88,7 @@ List getContractOffers() { } } - private static String catalog(String path) { + private String catalog(String path) { return CATALOG_BASE_URL + path; } diff --git a/system-tests/end2end-test/resources/nodes-dc.json b/system-tests/end2end-test/resources/nodes-dc.json deleted file mode 100644 index 760041a3..00000000 --- a/system-tests/end2end-test/resources/nodes-dc.json +++ /dev/null @@ -1,9 +0,0 @@ -[ - { - "name": "connector1", - "url": "http://connector1:8282/api/v1/dsp", - "supportedProtocols": [ - "dataspace-protocol-http" - ] - } -]