diff --git a/advanced/advanced-01-open-telemetry/resources/start-transfer.json b/advanced/advanced-01-open-telemetry/resources/start-transfer.json index 1dc97872..354566ce 100644 --- a/advanced/advanced-01-open-telemetry/resources/start-transfer.json +++ b/advanced/advanced-01-open-telemetry/resources/start-transfer.json @@ -6,7 +6,6 @@ "connectorId": "provider", "counterPartyAddress": "http://provider:19194/protocol", "contractId": "{{contract-agreement-id}}", - "assetId": "assetId", "protocol": "dataspace-protocol-http", "transferType": "HttpData-PULL" } diff --git a/federated-catalog/fc-00-basic/README.md b/federated-catalog/fc-00-basic/README.md index ffbd7b14..8dca991d 100644 --- a/federated-catalog/fc-00-basic/README.md +++ b/federated-catalog/fc-00-basic/README.md @@ -70,7 +70,7 @@ Use the following command to build a connector jar. #### Run the connector Execute the following to run the connector jar. ```shell -java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar +java -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar ``` --- @@ -95,4 +95,4 @@ curl -d @transfer/transfer-01-negotiation/resources/create-policy.json \ curl -d @transfer/transfer-01-negotiation/resources/create-contract-definition.json \ -H 'content-type: application/json' http://localhost:19193/management/v3/contractdefinitions \ -s | jq -``` \ No newline at end of file +``` diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 4ed8db2d..8be27d68 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -57,6 +57,7 @@ edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http edc-transfer-pull-http-dynamic-receiver = { module = "org.eclipse.edc:transfer-pull-http-dynamic-receiver", version.ref = "edc" } edc-vault-hashicorp = { module = "org.eclipse.edc:vault-hashicorp", version.ref = "edc" } edc-validator-data-address-http-data = { module = "org.eclipse.edc:validator-data-address-http-data", version.ref = "edc" } +edc-web-spi = { module = "org.eclipse.edc:web-spi", version.ref = "edc" } jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" } junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "jupiter" } junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "jupiter" } @@ -81,4 +82,5 @@ edc-fc-ext-api = { module = "org.eclipse.edc:federated-catalog-api", version.ref edc-fc-spi-crawler = { module = "org.eclipse.edc:crawler-spi", version.ref = "edc" } [plugins] -shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" } +shadow = { id = "com.gradleup.shadow", version = "8.3.5" } + diff --git a/settings.gradle.kts b/settings.gradle.kts index aec13b97..2d584bda 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -41,6 +41,8 @@ include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-consumer") include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-provider") include(":transfer:transfer-05-file-transfer-cloud:transfer-file-cloud") +include(":transfer:transfer-06-custom-proxy-data-plane:provider-proxy-data-plane") + include(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime") include(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime") include(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime") diff --git a/system-tests/build.gradle.kts b/system-tests/build.gradle.kts index dfb6652b..4569a88f 100644 --- a/system-tests/build.gradle.kts +++ b/system-tests/build.gradle.kts @@ -58,6 +58,8 @@ dependencies { testCompileOnly(project(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-consumer")) testCompileOnly(project(":transfer:transfer-05-file-transfer-cloud:transfer-file-cloud")) + testCompileOnly(project(":transfer:transfer-06-custom-proxy-data-plane:provider-proxy-data-plane")) + testCompileOnly(project(":federated-catalog:fc-00-basic:fixed-node-resolver")) testCompileOnly(project(":federated-catalog:fc-01-embedded:fc-connector")) testCompileOnly(project(":federated-catalog:fc-02-standalone:standalone-fc")) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/common/FederatedCatalogCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/FederatedCatalogCommon.java index 1e67b731..d43539a1 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/common/FederatedCatalogCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/common/FederatedCatalogCommon.java @@ -25,7 +25,6 @@ import static io.restassured.RestAssured.given; import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; -import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_KEY; import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_VALUE; import static org.eclipse.edc.samples.util.ConfigPropertiesLoader.fromPropertiesFile; @@ -44,12 +43,6 @@ public class FederatedCatalogCommon { private static final String STANDALONE_FC_CONFIG_PROPERTIES_FILE_PATH = "federated-catalog/fc-02-standalone/standalone-fc/config.properties"; private static final String FC_CONNECTOR_CONFIG_PROPERTIES_FILE_PATH = "federated-catalog/fc-01-embedded/fc-connector/config.properties"; - private static final String EDC_KEYSTORE = "edc.keystore"; - private static final String EDC_KEYSTORE_PASSWORD = "edc.keystore.password"; - private static final String EDC_FS_CONFIG = "edc.fs.config"; - private static final String CERT_PFX_FILE_PATH = "transfer/transfer-00-prerequisites/resources/certs/cert.pfx"; - private static final String KEYSTORE_PASSWORD = "123456"; - private static final String CRAWLER_EXECUTION_DELAY = "edc.catalog.cache.execution.delay.seconds"; public static final int CRAWLER_EXECUTION_DELAY_VALUE = 5; private static final String CRAWLER_EXECUTION_PERIOD = "edc.catalog.cache.execution.period.seconds"; @@ -79,8 +72,6 @@ private static RuntimeExtension getRuntime( return new RuntimePerClassExtension(new EmbeddedRuntime(moduleName, modulePath) .configurationProvider(fromPropertiesFile(configPropertiesFilePath)) .configurationProvider(() -> ConfigFactory.fromMap(Map.of( - EDC_KEYSTORE, getFileFromRelativePath(CERT_PFX_FILE_PATH).getAbsolutePath(), - EDC_KEYSTORE_PASSWORD, KEYSTORE_PASSWORD, CRAWLER_EXECUTION_DELAY, Integer.toString(CRAWLER_EXECUTION_DELAY_VALUE), CRAWLER_EXECUTION_PERIOD, Integer.toString(CRAWLER_EXECUTION_PERIOD_VALUE))) ) diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java index 143a02bd..88bd0832 100644 --- a/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java +++ b/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java @@ -18,11 +18,6 @@ import org.eclipse.edc.junit.extensions.RuntimeExtension; import org.eclipse.edc.junit.extensions.RuntimePerClassExtension; import org.eclipse.edc.samples.util.ConfigPropertiesLoader; -import org.eclipse.edc.spi.system.configuration.ConfigFactory; - -import java.util.Map; - -import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath; public class PrerequisitesCommon { public static final String API_KEY_HEADER_KEY = "X-Api-Key"; @@ -33,21 +28,19 @@ public class PrerequisitesCommon { private static final String CONNECTOR_MODULE_PATH = ":transfer:transfer-00-prerequisites:connector"; private static final String PROVIDER = "provider"; private static final String CONSUMER = "consumer"; - private static final String EDC_KEYSTORE = "edc.keystore"; - private static final String EDC_KEYSTORE_PASSWORD = "edc.keystore.password"; - private static final String EDC_FS_CONFIG = "edc.fs.config"; - - private static final String CERT_PFX_FILE_PATH = "transfer/transfer-00-prerequisites/resources/certs/cert.pfx"; - private static final String KEYSTORE_PASSWORD = "123456"; private static final String PROVIDER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties"; private static final String CONSUMER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties"; public static RuntimeExtension getProvider() { - return getConnector(CONNECTOR_MODULE_PATH, PROVIDER, PROVIDER_CONFIG_PROPERTIES_FILE_PATH); + return getProvider(CONNECTOR_MODULE_PATH, PROVIDER_CONFIG_PROPERTIES_FILE_PATH); + } + + public static RuntimeExtension getProvider(String modulePath, String configPath) { + return getConnector(modulePath, PROVIDER, configPath); } public static RuntimeExtension getConsumer() { - return getConnector(CONNECTOR_MODULE_PATH, CONSUMER, CONSUMER_CONFIG_PROPERTIES_FILE_PATH); + return getConsumer(CONNECTOR_MODULE_PATH); } public static RuntimeExtension getConsumer(String modulePath) { @@ -61,10 +54,6 @@ private static RuntimeExtension getConnector( ) { return new RuntimePerClassExtension(new EmbeddedRuntime(moduleName, modulePath) .configurationProvider(ConfigPropertiesLoader.fromPropertiesFile(configPropertiesFilePath)) - .configurationProvider(() -> ConfigFactory.fromMap(Map.of( - EDC_KEYSTORE, getFileFromRelativePath(CERT_PFX_FILE_PATH).getAbsolutePath(), - EDC_KEYSTORE_PASSWORD, KEYSTORE_PASSWORD)) - ) ); } } diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer06customProxyDataPlaneTest.java b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer06customProxyDataPlaneTest.java new file mode 100644 index 00000000..e839e58d --- /dev/null +++ b/system-tests/src/test/java/org/eclipse/edc/samples/transfer/Transfer06customProxyDataPlaneTest.java @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2022 Microsoft Corporation + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Microsoft Corporation - initial test implementation for sample + * Mercedes-Benz Tech Innovation GmbH - refactor test cases + * + */ + +package org.eclipse.edc.samples.transfer; + +import org.apache.http.HttpStatus; +import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcessStates; +import org.eclipse.edc.junit.annotations.EndToEndTest; +import org.eclipse.edc.junit.extensions.RuntimeExtension; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static io.restassured.RestAssured.given; +import static org.apache.http.HttpHeaders.AUTHORIZATION; +import static org.assertj.core.api.Assertions.assertThat; +import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath; +import static org.eclipse.edc.samples.common.NegotiationCommon.runNegotiation; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_KEY; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_VALUE; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.CONSUMER_MANAGEMENT_URL; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer; +import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider; +import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus; +import static org.eclipse.edc.samples.util.TransferUtil.startTransfer; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.not; + +@EndToEndTest +public class Transfer06customProxyDataPlaneTest { + + private static final String SAMPLE_NAME = "transfer-06-custom-proxy-data-plane"; + private static final String START_TRANSFER_FILE_PATH = "transfer/%s/resources/start-transfer.json".formatted(SAMPLE_NAME); + + @RegisterExtension + static RuntimeExtension provider = getProvider( + ":transfer:%s:provider-proxy-data-plane".formatted(SAMPLE_NAME), + "transfer/%s/resources/configuration/provider.properties".formatted(SAMPLE_NAME) + ); + + @RegisterExtension + static RuntimeExtension consumer = getConsumer(); + + @Test + void runSampleSteps() { + var requestBody = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH); + var contractAgreementId = runNegotiation(); + var transferProcessId = startTransfer(requestBody, contractAgreementId); + checkTransferStatus(transferProcessId, TransferProcessStates.STARTED); + + var edr = given() + .when() + .get(CONSUMER_MANAGEMENT_URL + "/v3/edrs/{id}/dataaddress", transferProcessId) + .then() + .log().ifValidationFails() + .statusCode(200) + .extract().body().jsonPath(); + + var result = given() + .header(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE) + .header(AUTHORIZATION, edr.getString("authorization")) + .when() + .get(edr.getString("endpoint")) + .then() + .statusCode(HttpStatus.SC_OK) + .log().ifValidationFails() + .body("[0].name", not(emptyString())) + .extract() + .jsonPath() + .get("[0].name"); + + assertThat(result).isEqualTo("Leanne Graham"); + } + +} diff --git a/transfer/README.md b/transfer/README.md index de66133b..7313a71d 100644 --- a/transfer/README.md +++ b/transfer/README.md @@ -44,4 +44,8 @@ transfer scenario, where a file is transferred not in the local file system, but different cloud providers. In this sample you will set up a provider that offers a file located in an `Azure Blob Storage`, and a consumer that requests to transfer this file to an `AWS S3 bucket`. Terraform is used for creating all required cloud -resources. \ No newline at end of file +resources. + +### [Transfer sample 06](./transfer-06-custom-proxy-data-plane): Implement a custom Http Proxy for PULL transfer + +This sample demonstrates how a custom HTTP Proxy Data Plane can be implemented to address PULL transfer type. diff --git a/transfer/streaming/streaming-01-http-to-http/transfer.json b/transfer/streaming/streaming-01-http-to-http/transfer.json index 5d89d89d..1b1d6a03 100644 --- a/transfer/streaming/streaming-01-http-to-http/transfer.json +++ b/transfer/streaming/streaming-01-http-to-http/transfer.json @@ -8,7 +8,6 @@ "baseUrl": "http://localhost:4000" }, "protocol": "dataspace-protocol-http", - "assetId": "stream-asset", "contractId": "{{contract-agreement-id}}", "connectorId": "provider", "counterPartyAddress": "http://localhost:18182/protocol", diff --git a/transfer/streaming/streaming-02-kafka-to-http/6-transfer.json b/transfer/streaming/streaming-02-kafka-to-http/6-transfer.json index 66e1173e..4ef9b875 100644 --- a/transfer/streaming/streaming-02-kafka-to-http/6-transfer.json +++ b/transfer/streaming/streaming-02-kafka-to-http/6-transfer.json @@ -9,7 +9,6 @@ }, "transferType": "HttpData-PUSH", "protocol": "dataspace-protocol-http", - "assetId": "stream-asset", "contractId": "{{contract-agreement-id}}", "connectorId": "provider", "counterPartyAddress": "http://localhost:18182/protocol" diff --git a/transfer/streaming/streaming-03-kafka-broker/6-transfer.json b/transfer/streaming/streaming-03-kafka-broker/6-transfer.json index db1d7efb..a674296a 100644 --- a/transfer/streaming/streaming-03-kafka-broker/6-transfer.json +++ b/transfer/streaming/streaming-03-kafka-broker/6-transfer.json @@ -7,7 +7,6 @@ "type": "KafkaBroker" }, "protocol": "dataspace-protocol-http", - "assetId": "stream-asset", "contractId": "{{contract-agreement-id}}", "connectorId": "provider", "connectorAddress": "http://localhost:18182/protocol" diff --git a/transfer/transfer-00-prerequisites/README.md b/transfer/transfer-00-prerequisites/README.md index e0820e92..e4a7daab 100644 --- a/transfer/transfer-00-prerequisites/README.md +++ b/transfer/transfer-00-prerequisites/README.md @@ -45,13 +45,13 @@ Inspect the different configuration files below: To run the provider, just run the following command ```bash -java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar +java -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar ``` To run the consumer, just run the following command (different terminal) ```bash -java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar +java -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar ``` Assuming you didn't change the ports in config files, the consumer will listen on the diff --git a/transfer/transfer-00-prerequisites/resources/certs/cert.pfx b/transfer/transfer-00-prerequisites/resources/certs/cert.pfx deleted file mode 100644 index 7ac9c73e..00000000 Binary files a/transfer/transfer-00-prerequisites/resources/certs/cert.pfx and /dev/null differ diff --git a/transfer/transfer-02-consumer-pull/resources/start-transfer.json b/transfer/transfer-02-consumer-pull/resources/start-transfer.json index c9959b7b..ff44b695 100644 --- a/transfer/transfer-02-consumer-pull/resources/start-transfer.json +++ b/transfer/transfer-02-consumer-pull/resources/start-transfer.json @@ -6,7 +6,6 @@ "connectorId": "provider", "counterPartyAddress": "http://localhost:19194/protocol", "contractId": "{{contract-agreement-id}}", - "assetId": "assetId", "protocol": "dataspace-protocol-http", "transferType": "HttpData-PULL" } diff --git a/transfer/transfer-03-provider-push/resources/start-transfer.json b/transfer/transfer-03-provider-push/resources/start-transfer.json index 51e6706b..f0a02b4a 100644 --- a/transfer/transfer-03-provider-push/resources/start-transfer.json +++ b/transfer/transfer-03-provider-push/resources/start-transfer.json @@ -6,7 +6,6 @@ "connectorId": "provider", "counterPartyAddress": "http://localhost:19194/protocol", "contractId": "{{contract-agreement-id}}", - "assetId": "assetId", "protocol": "dataspace-protocol-http", "transferType": "HttpData-PUSH", "dataDestination": { diff --git a/transfer/transfer-04-event-consumer/README.md b/transfer/transfer-04-event-consumer/README.md index 0f3d9cc3..d0f87204 100644 --- a/transfer/transfer-04-event-consumer/README.md +++ b/transfer/transfer-04-event-consumer/README.md @@ -64,7 +64,7 @@ Run this to build and launch the consumer with listener extension: ```bash ./gradlew transfer:transfer-04-event-consumer:consumer-with-listener:build -java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-04-event-consumer/consumer-with-listener/build/libs/connector.jar +java -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-04-event-consumer/consumer-with-listener/build/libs/connector.jar ```` ### 2. Negotiate a new contract diff --git a/transfer/transfer-05-file-transfer-cloud/resources/start-transfer.json b/transfer/transfer-05-file-transfer-cloud/resources/start-transfer.json index 3034f77e..0fa94fce 100644 --- a/transfer/transfer-05-file-transfer-cloud/resources/start-transfer.json +++ b/transfer/transfer-05-file-transfer-cloud/resources/start-transfer.json @@ -6,7 +6,6 @@ "connectorId": "provider", "counterPartyAddress": "http://localhost:19194/protocol", "contractId": "{{contract-agreement-id}}", - "assetId": "1", "protocol": "dataspace-protocol-http", "transferType": "AmazonS3-PUSH", "dataDestination": { @@ -16,4 +15,4 @@ "objectName": "test-document.txt", "endpointOverride": "http://localhost:9000" } -} \ No newline at end of file +} diff --git a/transfer/transfer-06-custom-proxy-data-plane/README.md b/transfer/transfer-06-custom-proxy-data-plane/README.md new file mode 100644 index 00000000..59e2e920 --- /dev/null +++ b/transfer/transfer-06-custom-proxy-data-plane/README.md @@ -0,0 +1,192 @@ +# Implement a simple "Consumer Pull" Http transfer flow + +The purpose of this sample is to show a data exchange between two connectors, one representing the +data provider and the other, the consumer. It's based on a "consumer pull" use case that you can find +more details +on [consumer pull and provider push transfers documentation](https://eclipse-edc.github.io/documentation/for-adopters/control-plane/#consumer-pull-and-provider-push-transfers). + +This sample consists of the following steps: + +* Provide a simple Http Proxy data plane implementation +* Perform a file transfer initiated by the consumer +* The provider will send an EndpointDataReference to the consumer +* The consumer will call the endpoint and fetch the data + +## Prerequisites + +The following steps assume you already built the base connector in the [Prerequisites](../transfer-00-prerequisites/README.md), +that will be used for the "consumer" instance. + +## Implement a simple Http Proxy data plane + +In the [provider-proxy-data-plane](provider-proxy-data-plane) there's a connector based on the [basic one](../transfer-00-prerequisites/connector) +but with the `data-plane-api-v2` module excluded, because that one has been deprecated: + +```kotlin + runtimeOnly(project(":transfer:transfer-00-prerequisites:connector")) { + exclude("org.eclipse.edc", "data-plane-public-api-v2") + } +``` + +The proxy implementation provided in this runtime is really bare-bone and it supports only `GET`. Needless to say, it is +only for sample purposes, for a production environment you should rely on a proper proxy implementation. + +Here's the rundown of the implementation: + +### `CustomProxyDataPlaneExtension` + +Is the extension that takes care to inject the needed services and: +- registers a `PortMapping` on the `PortMappingRegistry`: this tells the underlying Web Server (jetty) to expose an endpoint + under the configured port and path +- adds an endpoint generator function on the `generatorFunction`, this will ensure that all the `EndpointDataReference`s + generated by the connector will point to the data-plane proxy public endpoint. Note that this endpoint can be configured, + because in a real world scenario it will need to point to a public endpoint exposed on the internet. +- registers the `ProxyController`, that is the actual proxy implementation. + +[Please check out the code](provider-proxy-data-plane/src/main/java/org/eclipse/edc/sample/extension/proxy/CustomProxyDataPlaneExtension.java). + +### `ProxyController` + +This is modeled as a classic Jersey controller, implementing only, as stated above, only the `GET` method. +On every call the proxy will: +- verify the `Authorization` token +- extract the source `DataAddress` from the token using the `authorizationService` +- create and send an http request to the source server +- pipe the response body stream to the output, adding status code and content-type information. + +[Please check out the code](provider-proxy-data-plane/src/main/java/org/eclipse/edc/sample/extension/proxy/ProxyController.java). + +## Run the sample + +Running this sample consists of multiple steps, that are executed one by one and following the same +order. + +### Build the provider connector + +```bash +./gradlew transfer:transfer-06-custom-proxy-data-plane:provider-proxy-data-plane:build +``` + +### Run the connectors + +To run the provider, just run the following command + +```bash +java -Dedc.fs.config=transfer/transfer-06-custom-proxy-data-plane/resources/configuration/provider.properties \ + -jar transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/build/libs/connector.jar +``` + +To run the consumer, just run the following command (different terminal). Note that the consumer is the same that was built +in the [prerequisites sample](../transfer-00-prerequisites) + +```bash +java -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties \ + -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar +``` + +### Negotiate the contract + +You can follow the instructions in the [`transfer-01-negotiation`](../transfer-01-negotiation/README.md) to setup a +Contract Negotiation between the two parts. + +### Start the transfer + +In the [request body](resources/start-transfer.json), we need to specify which asset we want transferred, the ID of the +contract agreement, the address of the provider connector and where we want the file transferred. +Before executing the request, insert the `contractAgreementId` from the previous chapter. Then run: + +```bash +curl -X POST "http://localhost:29193/management/v3/transferprocesses" \ + -H "Content-Type: application/json" \ + -d @transfer/transfer-06-custom-proxy-data-plane/resources/start-transfer.json \ + -s | jq +``` + +> the "HttpData-PULL" transfer type is used for the consumer pull method, and it means that it will be up to +> the consumer to request the data to the provider and that the request will be a proxy for the datasource + +Then, we will get a UUID in the response. This time, this is the ID of the `TransferProcess` (process id) created on the +consumer side, because like the contract negotiation, the data transfer is handled in a state machine and performed +asynchronously. + +Sample output: + +```json +{ + ... + "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", + "createdAt": 1674078357807, + ... +} +``` + +### Check the transfer status + +Due to the nature of the transfer, it will be very fast and most likely already done by the time you +read the UUID. + +```bash +curl http://localhost:29193/management/v3/transferprocesses/ | jq +``` + +You should see the Transfer Process in `STARTED` state: + +```json +{ + ... + "@id": "591bb609-1edb-4a6b-babe-50f1eca3e1e9", + "state": "STARTED", + ... +} + +``` + +> Note that for the consumer pull scenario the TP will stay in STARTED state after the data has been transferred successfully. +> It might get eventually get shifted to TERMINATED or DEPROVISIONED by other components, but this is not scope of this sample. + +### Check the data + +At this step, an EndpointDataReference would have been generated by the provider and sent to the consumer. The latter +stored it in a cache, so we can obtain it using the transfer process id: +```bash +curl http://localhost:29193/management/v3/edrs//dataaddress | jq +``` + +```json +{ + "@type": "DataAddress", + "type": "https://w3id.org/idsa/v4.1/HTTP", + "endpoint": "http://localhost:19291/public", + "authType": "bearer", + "endpointType": "https://w3id.org/idsa/v4.1/HTTP", + "authorization": "eyJraWQiOiJwdWJsaWMta2V5IiwiYWxnIjoiUlMyNTYifQ.eyJpc3MiOiJwcm92aWRlciIsImF1ZCI6ImNvbnN1bWVyIiwic3ViIjoicHJvdmlkZXIiLCJpYXQiOjE3MTc3NjkyMzEyOTYsImp0aSI6IjM2M2RhMGU4LWZmOGItNDY1My05YjQwLWY4MjdlMWMzOGMzYyJ9.WOVPz6m7XzIrbiMTfLqOXacGYz8Xk_-iQu7gmxoIgDFYsgo0da2Iv51EsugIpqbodPsmB0kK7zkyrmsFOfAASAq7fjsy4gQF-u5egYwoGpcxjYaJJdQa5lkwjC0fRxdVFVwZwrOaT5Mg-vGA9HssTEnlA64q-O0ae_aTH5ToflmPDM3FhAgL55I3odM5ysM2POEJY6pgOxIV9XjuhZFl_i_iTiUCZy__oQUZiYk58wKoqfK758Sy1WzpH-eyZCDUi_Z3n6cJB80_0ZThoPhtiFH7Tl9DfStnjsCoaeqMLFnTXp0s8h4ZGFmjfBc-72aAdRQqqLDT8WXNg3Csv5B56Q", + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/", + "edc": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + } +} + +``` + +Once this json is read, use a tool like postman or curl to execute the following query, to read the +data + +```bash +curl --location --request GET 'http://localhost:19291/public/' --header 'Authorization: ' +``` + +At the end, and to be sure that you correctly achieved the pull, you can check if the data you get +is the same as the one you can get at https://jsonplaceholder.typicode.com/users + + +Since we configured the `HttpData` with `proxyPath`, we could also ask for a specific user with: + +```bash +curl --location --request GET 'http://localhost:19291/public/1' --header 'Authorization: ' +``` + +And the data returned will be the same as in https://jsonplaceholder.typicode.com/users/1 + +Your first data transfer has been completed successfully. +Continue with the [next chapter](../transfer-03-provider-push/README.md) to run through a "provider push" scenario. diff --git a/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/build.gradle.kts b/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/build.gradle.kts new file mode 100644 index 00000000..cee4c5f1 --- /dev/null +++ b/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/build.gradle.kts @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2025 Cofinity-X + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Cofinity-X - initial API and implementation + * + */ + + +plugins { + `java-library` + id("application") + alias(libs.plugins.shadow) +} + +dependencies { + implementation(libs.edc.data.plane.spi) + implementation(libs.edc.web.spi) + + runtimeOnly(project(":transfer:transfer-00-prerequisites:connector")) { + exclude("org.eclipse.edc", "data-plane-public-api-v2") + } +} + +application { + mainClass.set("$group.boot.system.runtime.BaseRuntime") +} + +var distTar = tasks.getByName("distTar") +var distZip = tasks.getByName("distZip") + +tasks.withType { + dependsOn(":transfer:transfer-00-prerequisites:connector:shadowJar") +} + +tasks.withType { + mergeServiceFiles() + archiveFileName.set("connector.jar") + dependsOn(distTar, distZip) +} diff --git a/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/src/main/java/org/eclipse/edc/sample/extension/proxy/CustomProxyDataPlaneExtension.java b/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/src/main/java/org/eclipse/edc/sample/extension/proxy/CustomProxyDataPlaneExtension.java new file mode 100644 index 00000000..81cb8ca8 --- /dev/null +++ b/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/src/main/java/org/eclipse/edc/sample/extension/proxy/CustomProxyDataPlaneExtension.java @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025 Cofinity-X + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Cofinity-X - initial API and implementation + * + */ + +package org.eclipse.edc.sample.extension.proxy; + +import org.eclipse.edc.connector.dataplane.spi.Endpoint; +import org.eclipse.edc.connector.dataplane.spi.iam.DataPlaneAuthorizationService; +import org.eclipse.edc.connector.dataplane.spi.iam.PublicEndpointGeneratorService; +import org.eclipse.edc.runtime.metamodel.annotation.Configuration; +import org.eclipse.edc.runtime.metamodel.annotation.Inject; +import org.eclipse.edc.runtime.metamodel.annotation.Setting; +import org.eclipse.edc.runtime.metamodel.annotation.Settings; +import org.eclipse.edc.spi.system.ServiceExtension; +import org.eclipse.edc.spi.system.ServiceExtensionContext; +import org.eclipse.edc.web.spi.WebService; +import org.eclipse.edc.web.spi.configuration.PortMapping; +import org.eclipse.edc.web.spi.configuration.PortMappingRegistry; + +import static org.eclipse.edc.web.spi.configuration.ApiContext.PUBLIC; + +public class CustomProxyDataPlaneExtension implements ServiceExtension { + + private static final int DEFAULT_PUBLIC_PORT = 8185; + private static final String DEFAULT_PUBLIC_PATH = "/api/public"; + + @Configuration + private PublicApiConfiguration apiConfiguration; + @Setting(description = "Base url of the public API endpoint without the trailing slash. This should point to the public endpoint configured.", + key = "edc.dataplane.proxy.public.endpoint") + private String proxyPublicEndpoint; + + @Inject + private PortMappingRegistry portMappingRegistry; + @Inject + private PublicEndpointGeneratorService generatorService; + @Inject + private WebService webService; + @Inject + private DataPlaneAuthorizationService authorizationService; + + @Override + public void initialize(ServiceExtensionContext context) { + portMappingRegistry.register(new PortMapping(PUBLIC, apiConfiguration.port(), apiConfiguration.path())); + + generatorService.addGeneratorFunction("HttpData", dataAddress -> Endpoint.url(proxyPublicEndpoint)); + + webService.registerResource(PUBLIC, new ProxyController(authorizationService)); + } + + @Settings + record PublicApiConfiguration( + @Setting(key = "web.http." + PUBLIC + ".port", description = "Port for " + PUBLIC + " api context", defaultValue = DEFAULT_PUBLIC_PORT + "") + int port, + @Setting(key = "web.http." + PUBLIC + ".path", description = "Path for " + PUBLIC + " api context", defaultValue = DEFAULT_PUBLIC_PATH) + String path + ) { + + } +} diff --git a/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/src/main/java/org/eclipse/edc/sample/extension/proxy/ProxyController.java b/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/src/main/java/org/eclipse/edc/sample/extension/proxy/ProxyController.java new file mode 100644 index 00000000..e751e98f --- /dev/null +++ b/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/src/main/java/org/eclipse/edc/sample/extension/proxy/ProxyController.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2025 Cofinity-X + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + * + * Contributors: + * Cofinity-X - initial API and implementation + * + */ + +package org.eclipse.edc.sample.extension.proxy; + +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.Response; +import org.eclipse.edc.connector.dataplane.spi.iam.DataPlaneAuthorizationService; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; + +import static jakarta.ws.rs.core.HttpHeaders.AUTHORIZATION; +import static jakarta.ws.rs.core.HttpHeaders.CONTENT_TYPE; +import static jakarta.ws.rs.core.MediaType.APPLICATION_OCTET_STREAM; +import static jakarta.ws.rs.core.MediaType.WILDCARD; +import static jakarta.ws.rs.core.Response.Status.FORBIDDEN; +import static jakarta.ws.rs.core.Response.Status.UNAUTHORIZED; +import static java.util.Collections.emptyMap; +import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE; + +@Path("{any:.*}") +@Consumes(WILDCARD) +@Produces(WILDCARD) +public class ProxyController { + + private final DataPlaneAuthorizationService authorizationService; + + public ProxyController(DataPlaneAuthorizationService authorizationService) { + this.authorizationService = authorizationService; + } + + private final HttpClient httpClient = HttpClient.newHttpClient(); + + @GET + public Response proxyGet(@Context ContainerRequestContext requestContext) { + var token = requestContext.getHeaderString(AUTHORIZATION); + if (token == null) { + return Response.status(UNAUTHORIZED).build(); + } + + var authorization = authorizationService.authorize(token, emptyMap()); + if (authorization.failed()) { + return Response.status(FORBIDDEN).build(); + } + + var sourceDataAddress = authorization.getContent(); + + try { + var targetUrl = sourceDataAddress.getStringProperty(EDC_NAMESPACE + "baseUrl") + "/" + requestContext.getUriInfo().getPath(); + var request = HttpRequest.newBuilder() + .uri(URI.create(targetUrl)) + .method(requestContext.getMethod(), HttpRequest.BodyPublishers.ofInputStream(requestContext::getEntityStream)) + .build(); + + var response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); + return Response.status(response.statusCode()) + .header(CONTENT_TYPE, response.headers().firstValue(CONTENT_TYPE).orElse(APPLICATION_OCTET_STREAM)) + .entity(response.body()) + .build(); + } catch (IOException | InterruptedException e) { + return Response.status(Response.Status.BAD_GATEWAY) + .entity("{\"error\": \"Failed to contact backend service\"}") + .build(); + } + } + +} diff --git a/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension b/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension new file mode 100644 index 00000000..d1f67d99 --- /dev/null +++ b/transfer/transfer-06-custom-proxy-data-plane/provider-proxy-data-plane/src/main/resources/META-INF/services/org.eclipse.edc.spi.system.ServiceExtension @@ -0,0 +1 @@ +org.eclipse.edc.sample.extension.proxy.CustomProxyDataPlaneExtension diff --git a/transfer/transfer-06-custom-proxy-data-plane/resources/configuration/provider.properties b/transfer/transfer-06-custom-proxy-data-plane/resources/configuration/provider.properties new file mode 100644 index 00000000..b74ec8e0 --- /dev/null +++ b/transfer/transfer-06-custom-proxy-data-plane/resources/configuration/provider.properties @@ -0,0 +1,17 @@ +edc.participant.id=provider +edc.dsp.callback.address=http://localhost:19194/protocol +web.http.port=19191 +web.http.path=/api +web.http.management.port=19193 +web.http.management.path=/management +web.http.protocol.port=19194 +web.http.protocol.path=/protocol +edc.transfer.proxy.token.signer.privatekey.alias=private-key +edc.transfer.proxy.token.verifier.publickey.alias=public-key +web.http.public.port=19291 +web.http.public.path=/public +web.http.control.port=19192 +web.http.control.path=/control +web.http.version.port=19195 +web.http.version.path=/version +edc.dataplane.proxy.public.endpoint=http://localhost:19291/public diff --git a/transfer/transfer-06-custom-proxy-data-plane/resources/start-transfer.json b/transfer/transfer-06-custom-proxy-data-plane/resources/start-transfer.json new file mode 100644 index 00000000..ff44b695 --- /dev/null +++ b/transfer/transfer-06-custom-proxy-data-plane/resources/start-transfer.json @@ -0,0 +1,11 @@ +{ + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "TransferRequestDto", + "connectorId": "provider", + "counterPartyAddress": "http://localhost:19194/protocol", + "contractId": "{{contract-agreement-id}}", + "protocol": "dataspace-protocol-http", + "transferType": "HttpData-PULL" +}