feat: http proxy sample (eclipse-edc#363)

* build: update shadow plugin group id * docs: add samples for proxy data-plane * feat: add transfer-06 sample for custom proxy component
Think-iT-Labs · Feb 11, 2025 · 1f79c6d · 1f79c6d
1 parent 647e41a
commit 1f79c6d
Show file tree

Hide file tree

Showing 25 changed files with 534 additions and 41 deletions.
diff --git a/advanced/advanced-01-open-telemetry/resources/start-transfer.json b/advanced/advanced-01-open-telemetry/resources/start-transfer.json
@@ -6,7 +6,6 @@
   "connectorId": "provider",
   "counterPartyAddress": "http://provider:19194/protocol",
   "contractId": "{{contract-agreement-id}}",
-  "assetId": "assetId",
   "protocol": "dataspace-protocol-http",
   "transferType": "HttpData-PULL"
 }
diff --git a/federated-catalog/fc-00-basic/README.md b/federated-catalog/fc-00-basic/README.md
@@ -70,7 +70,7 @@ Use the following command to build a connector jar.
 #### Run the connector
 Execute the following to run the connector jar.
 ```shell
-java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar
+java -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar
 ```
 
 ---
@@ -95,4 +95,4 @@ curl -d @transfer/transfer-01-negotiation/resources/create-policy.json \
 curl -d @transfer/transfer-01-negotiation/resources/create-contract-definition.json \
   -H 'content-type: application/json' http://localhost:19193/management/v3/contractdefinitions \
   -s | jq
-```
+```
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
@@ -57,6 +57,7 @@ edc-transfer-pull-http-receiver = { module = "org.eclipse.edc:transfer-pull-http
 edc-transfer-pull-http-dynamic-receiver = { module = "org.eclipse.edc:transfer-pull-http-dynamic-receiver", version.ref = "edc" }
 edc-vault-hashicorp = { module = "org.eclipse.edc:vault-hashicorp", version.ref = "edc" }
 edc-validator-data-address-http-data = { module = "org.eclipse.edc:validator-data-address-http-data", version.ref = "edc" }
+edc-web-spi = { module = "org.eclipse.edc:web-spi", version.ref = "edc" }
 jakarta-rsApi = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version.ref = "rsApi" }
 junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api", version.ref = "jupiter" }
 junit-jupiter-engine = { module = "org.junit.jupiter:junit-jupiter-engine", version.ref = "jupiter" }
@@ -81,4 +82,5 @@ edc-fc-ext-api = { module = "org.eclipse.edc:federated-catalog-api", version.ref
 edc-fc-spi-crawler = { module = "org.eclipse.edc:crawler-spi", version.ref = "edc" }
 
 [plugins]
-shadow = { id = "com.github.johnrengelman.shadow", version = "8.1.1" }
+shadow = { id = "com.gradleup.shadow", version = "8.3.5" }
+
diff --git a/settings.gradle.kts b/settings.gradle.kts
@@ -41,6 +41,8 @@ include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-consumer")
 include(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-provider")
 include(":transfer:transfer-05-file-transfer-cloud:transfer-file-cloud")
 
+include(":transfer:transfer-06-custom-proxy-data-plane:provider-proxy-data-plane")
+
 include(":transfer:streaming:streaming-01-http-to-http:streaming-01-runtime")
 include(":transfer:streaming:streaming-02-kafka-to-http:streaming-02-runtime")
 include(":transfer:streaming:streaming-03-kafka-broker:streaming-03-runtime")

diff --git a/system-tests/build.gradle.kts b/system-tests/build.gradle.kts
@@ -58,6 +58,8 @@ dependencies {
     testCompileOnly(project(":transfer:transfer-05-file-transfer-cloud:cloud-transfer-consumer"))
     testCompileOnly(project(":transfer:transfer-05-file-transfer-cloud:transfer-file-cloud"))
 
+    testCompileOnly(project(":transfer:transfer-06-custom-proxy-data-plane:provider-proxy-data-plane"))
+
     testCompileOnly(project(":federated-catalog:fc-00-basic:fixed-node-resolver"))
     testCompileOnly(project(":federated-catalog:fc-01-embedded:fc-connector"))
     testCompileOnly(project(":federated-catalog:fc-02-standalone:standalone-fc"))

diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/common/FederatedCatalogCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/FederatedCatalogCommon.java
@@ -25,7 +25,6 @@
 
 import static io.restassured.RestAssured.given;
 import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath;
-import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath;
 import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_KEY;
 import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_VALUE;
 import static org.eclipse.edc.samples.util.ConfigPropertiesLoader.fromPropertiesFile;
@@ -44,12 +43,6 @@ public class FederatedCatalogCommon {
     private static final String STANDALONE_FC_CONFIG_PROPERTIES_FILE_PATH = "federated-catalog/fc-02-standalone/standalone-fc/config.properties";
     private static final String FC_CONNECTOR_CONFIG_PROPERTIES_FILE_PATH = "federated-catalog/fc-01-embedded/fc-connector/config.properties";
 
-    private static final String EDC_KEYSTORE = "edc.keystore";
-    private static final String EDC_KEYSTORE_PASSWORD = "edc.keystore.password";
-    private static final String EDC_FS_CONFIG = "edc.fs.config";
-    private static final String CERT_PFX_FILE_PATH = "transfer/transfer-00-prerequisites/resources/certs/cert.pfx";
-    private static final String KEYSTORE_PASSWORD = "123456";
-
     private static final String CRAWLER_EXECUTION_DELAY = "edc.catalog.cache.execution.delay.seconds";
     public static final int CRAWLER_EXECUTION_DELAY_VALUE = 5;
     private static final String CRAWLER_EXECUTION_PERIOD = "edc.catalog.cache.execution.period.seconds";
@@ -79,8 +72,6 @@ private static RuntimeExtension getRuntime(
         return new RuntimePerClassExtension(new EmbeddedRuntime(moduleName, modulePath)
                 .configurationProvider(fromPropertiesFile(configPropertiesFilePath))
                 .configurationProvider(() -> ConfigFactory.fromMap(Map.of(
-                    EDC_KEYSTORE, getFileFromRelativePath(CERT_PFX_FILE_PATH).getAbsolutePath(),
-                    EDC_KEYSTORE_PASSWORD, KEYSTORE_PASSWORD,
                     CRAWLER_EXECUTION_DELAY, Integer.toString(CRAWLER_EXECUTION_DELAY_VALUE),
                     CRAWLER_EXECUTION_PERIOD, Integer.toString(CRAWLER_EXECUTION_PERIOD_VALUE)))
                 )

diff --git a/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java b/system-tests/src/test/java/org/eclipse/edc/samples/common/PrerequisitesCommon.java
@@ -18,11 +18,6 @@
 import org.eclipse.edc.junit.extensions.RuntimeExtension;
 import org.eclipse.edc.junit.extensions.RuntimePerClassExtension;
 import org.eclipse.edc.samples.util.ConfigPropertiesLoader;
-import org.eclipse.edc.spi.system.configuration.ConfigFactory;
-
-import java.util.Map;
-
-import static org.eclipse.edc.samples.common.FileTransferCommon.getFileFromRelativePath;
 
 public class PrerequisitesCommon {
     public static final String API_KEY_HEADER_KEY = "X-Api-Key";
@@ -33,21 +28,19 @@ public class PrerequisitesCommon {
     private static final String CONNECTOR_MODULE_PATH = ":transfer:transfer-00-prerequisites:connector";
     private static final String PROVIDER = "provider";
     private static final String CONSUMER = "consumer";
-    private static final String EDC_KEYSTORE = "edc.keystore";
-    private static final String EDC_KEYSTORE_PASSWORD = "edc.keystore.password";
-    private static final String EDC_FS_CONFIG = "edc.fs.config";
-
-    private static final String CERT_PFX_FILE_PATH = "transfer/transfer-00-prerequisites/resources/certs/cert.pfx";
-    private static final String KEYSTORE_PASSWORD = "123456";
     private static final String PROVIDER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties";
     private static final String CONSUMER_CONFIG_PROPERTIES_FILE_PATH = "transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties";
 
     public static RuntimeExtension getProvider() {
-        return getConnector(CONNECTOR_MODULE_PATH, PROVIDER, PROVIDER_CONFIG_PROPERTIES_FILE_PATH);
+        return getProvider(CONNECTOR_MODULE_PATH, PROVIDER_CONFIG_PROPERTIES_FILE_PATH);
+    }
+
+    public static RuntimeExtension getProvider(String modulePath, String configPath) {
+        return getConnector(modulePath, PROVIDER, configPath);
     }
 
     public static RuntimeExtension getConsumer() {
-        return getConnector(CONNECTOR_MODULE_PATH, CONSUMER, CONSUMER_CONFIG_PROPERTIES_FILE_PATH);
+        return getConsumer(CONNECTOR_MODULE_PATH);
     }
 
     public static RuntimeExtension getConsumer(String modulePath) {
@@ -61,10 +54,6 @@ private static RuntimeExtension getConnector(
     ) {
         return new RuntimePerClassExtension(new EmbeddedRuntime(moduleName, modulePath)
                 .configurationProvider(ConfigPropertiesLoader.fromPropertiesFile(configPropertiesFilePath))
-                .configurationProvider(() -> ConfigFactory.fromMap(Map.of(
-                    EDC_KEYSTORE, getFileFromRelativePath(CERT_PFX_FILE_PATH).getAbsolutePath(),
-                    EDC_KEYSTORE_PASSWORD, KEYSTORE_PASSWORD))
-                )
         );
     }
 }
diff --git a/...ts/src/test/java/org/eclipse/edc/samples/transfer/Transfer06customProxyDataPlaneTest.java b/...ts/src/test/java/org/eclipse/edc/samples/transfer/Transfer06customProxyDataPlaneTest.java
@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2022 Microsoft Corporation
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Microsoft Corporation - initial test implementation for sample
+ *       Mercedes-Benz Tech Innovation GmbH - refactor test cases
+ *
+ */
+
+package org.eclipse.edc.samples.transfer;
+
+import org.apache.http.HttpStatus;
+import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcessStates;
+import org.eclipse.edc.junit.annotations.EndToEndTest;
+import org.eclipse.edc.junit.extensions.RuntimeExtension;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+import static io.restassured.RestAssured.given;
+import static org.apache.http.HttpHeaders.AUTHORIZATION;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.eclipse.edc.samples.common.FileTransferCommon.getFileContentFromRelativePath;
+import static org.eclipse.edc.samples.common.NegotiationCommon.runNegotiation;
+import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_KEY;
+import static org.eclipse.edc.samples.common.PrerequisitesCommon.API_KEY_HEADER_VALUE;
+import static org.eclipse.edc.samples.common.PrerequisitesCommon.CONSUMER_MANAGEMENT_URL;
+import static org.eclipse.edc.samples.common.PrerequisitesCommon.getConsumer;
+import static org.eclipse.edc.samples.common.PrerequisitesCommon.getProvider;
+import static org.eclipse.edc.samples.util.TransferUtil.checkTransferStatus;
+import static org.eclipse.edc.samples.util.TransferUtil.startTransfer;
+import static org.hamcrest.Matchers.emptyString;
+import static org.hamcrest.Matchers.not;
+
+@EndToEndTest
+public class Transfer06customProxyDataPlaneTest {
+
+    private static final String SAMPLE_NAME = "transfer-06-custom-proxy-data-plane";
+    private static final String START_TRANSFER_FILE_PATH = "transfer/%s/resources/start-transfer.json".formatted(SAMPLE_NAME);
+
+    @RegisterExtension
+    static RuntimeExtension provider = getProvider(
+            ":transfer:%s:provider-proxy-data-plane".formatted(SAMPLE_NAME),
+            "transfer/%s/resources/configuration/provider.properties".formatted(SAMPLE_NAME)
+    );
+
+    @RegisterExtension
+    static RuntimeExtension consumer = getConsumer();
+
+    @Test
+    void runSampleSteps() {
+        var requestBody = getFileContentFromRelativePath(START_TRANSFER_FILE_PATH);
+        var contractAgreementId = runNegotiation();
+        var transferProcessId = startTransfer(requestBody, contractAgreementId);
+        checkTransferStatus(transferProcessId, TransferProcessStates.STARTED);
+
+        var edr = given()
+                .when()
+                .get(CONSUMER_MANAGEMENT_URL + "/v3/edrs/{id}/dataaddress", transferProcessId)
+                .then()
+                .log().ifValidationFails()
+                .statusCode(200)
+                .extract().body().jsonPath();
+
+        var result = given()
+                .header(API_KEY_HEADER_KEY, API_KEY_HEADER_VALUE)
+                .header(AUTHORIZATION, edr.getString("authorization"))
+                .when()
+                .get(edr.getString("endpoint"))
+                .then()
+                .statusCode(HttpStatus.SC_OK)
+                .log().ifValidationFails()
+                .body("[0].name", not(emptyString()))
+                .extract()
+                .jsonPath()
+                .get("[0].name");
+
+        assertThat(result).isEqualTo("Leanne Graham");
+    }
+
+}
diff --git a/transfer/README.md b/transfer/README.md
@@ -44,4 +44,8 @@ transfer scenario, where a file is transferred not in the local file system, but
 different cloud providers. In this sample you will set up
 a provider that offers a file located in an `Azure Blob Storage`, and a consumer that requests to
 transfer this file to an `AWS S3 bucket`. Terraform is used for creating all required cloud
-resources.
+resources.
+
+### [Transfer sample 06](./transfer-06-custom-proxy-data-plane): Implement a custom Http Proxy for PULL transfer
+
+This sample demonstrates how a custom HTTP Proxy Data Plane can be implemented to address PULL transfer type.
diff --git a/transfer/streaming/streaming-01-http-to-http/transfer.json b/transfer/streaming/streaming-01-http-to-http/transfer.json
@@ -8,7 +8,6 @@
     "baseUrl": "http://localhost:4000"
   },
   "protocol": "dataspace-protocol-http",
-  "assetId": "stream-asset",
   "contractId": "{{contract-agreement-id}}",
   "connectorId": "provider",
   "counterPartyAddress": "http://localhost:18182/protocol",

diff --git a/transfer/streaming/streaming-02-kafka-to-http/6-transfer.json b/transfer/streaming/streaming-02-kafka-to-http/6-transfer.json
@@ -9,7 +9,6 @@
   },
   "transferType": "HttpData-PUSH",
   "protocol": "dataspace-protocol-http",
-  "assetId": "stream-asset",
   "contractId": "{{contract-agreement-id}}",
   "connectorId": "provider",
   "counterPartyAddress": "http://localhost:18182/protocol"

diff --git a/transfer/streaming/streaming-03-kafka-broker/6-transfer.json b/transfer/streaming/streaming-03-kafka-broker/6-transfer.json
@@ -7,7 +7,6 @@
     "type": "KafkaBroker"
   },
   "protocol": "dataspace-protocol-http",
-  "assetId": "stream-asset",
   "contractId": "{{contract-agreement-id}}",
   "connectorId": "provider",
   "connectorAddress": "http://localhost:18182/protocol"

diff --git a/transfer/transfer-00-prerequisites/README.md b/transfer/transfer-00-prerequisites/README.md
@@ -45,13 +45,13 @@ Inspect the different configuration files below:
 To run the provider, just run the following command
 
 ```bash
-java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar
+java -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/provider-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar
 ```
 
 To run the consumer, just run the following command (different terminal)
 
 ```bash
-java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar
+java -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-00-prerequisites/connector/build/libs/connector.jar
 ```
 
 Assuming you didn't change the ports in config files, the consumer will listen on the

diff --git a/transfer/transfer-00-prerequisites/resources/certs/cert.pfx b/transfer/transfer-00-prerequisites/resources/certs/cert.pfx
diff --git a/transfer/transfer-02-consumer-pull/resources/start-transfer.json b/transfer/transfer-02-consumer-pull/resources/start-transfer.json
@@ -6,7 +6,6 @@
   "connectorId": "provider",
   "counterPartyAddress": "http://localhost:19194/protocol",
   "contractId": "{{contract-agreement-id}}",
-  "assetId": "assetId",
   "protocol": "dataspace-protocol-http",
   "transferType": "HttpData-PULL"
 }
diff --git a/transfer/transfer-03-provider-push/resources/start-transfer.json b/transfer/transfer-03-provider-push/resources/start-transfer.json
@@ -6,7 +6,6 @@
   "connectorId": "provider",
   "counterPartyAddress": "http://localhost:19194/protocol",
   "contractId": "{{contract-agreement-id}}",
-  "assetId": "assetId",
   "protocol": "dataspace-protocol-http",
   "transferType": "HttpData-PUSH",
   "dataDestination": {

diff --git a/transfer/transfer-04-event-consumer/README.md b/transfer/transfer-04-event-consumer/README.md
@@ -64,7 +64,7 @@ Run this to build and launch the consumer with listener extension:
 
 ```bash
 ./gradlew transfer:transfer-04-event-consumer:consumer-with-listener:build
-java -Dedc.keystore=transfer/transfer-00-prerequisites/resources/certs/cert.pfx -Dedc.keystore.password=123456 -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-04-event-consumer/consumer-with-listener/build/libs/connector.jar
+java -Dedc.fs.config=transfer/transfer-00-prerequisites/resources/configuration/consumer-configuration.properties -jar transfer/transfer-04-event-consumer/consumer-with-listener/build/libs/connector.jar
 ````
 
 ### 2. Negotiate a new contract

diff --git a/transfer/transfer-05-file-transfer-cloud/resources/start-transfer.json b/transfer/transfer-05-file-transfer-cloud/resources/start-transfer.json
@@ -6,7 +6,6 @@
   "connectorId": "provider",
   "counterPartyAddress": "http://localhost:19194/protocol",
   "contractId": "{{contract-agreement-id}}",
-  "assetId": "1",
   "protocol": "dataspace-protocol-http",
   "transferType": "AmazonS3-PUSH",
   "dataDestination": {
@@ -16,4 +15,4 @@
     "objectName": "test-document.txt",
     "endpointOverride": "http://localhost:9000"
   }
-}
+}