From f39376f2b3c16cd2ac931641bde871a6ae88bfe2 Mon Sep 17 00:00:00 2001 From: Kervin Hu Date: Thu, 11 Apr 2024 13:46:48 +0800 Subject: [PATCH] chore: commit for v2.6.1 --- CHANGELOG.md | 9 +++++++ README.md | 5 +--- docs/ECR_DEPLOYMENT_CN.md | 4 ++++ docs/ECR_DEPLOYMENT_EN.md | 6 ++++- docs/S3_DEPLOYMENT_CN.md | 3 +++ docs/S3_DEPLOYMENT_EN.md | 4 ++++ .../architecture-details.md | 6 ++--- docs/en-base/index.md | 4 +++- .../features-and-benefits.md | 4 +++- docs/en-base/solution-overview/use-cases.md | 10 ++++---- docs/en-base/user-guide/tutorial-ecr.md | 14 +++++++++-- docs/en-base/user-guide/tutorial-s3.md | 6 ++++- .../architecture-details.md | 6 ++--- docs/zh-base/index.md | 4 +++- .../features-and-benefits.md | 3 +++ docs/zh-base/solution-overview/use-cases.md | 10 ++++---- .../zh-base/user-guide/tutorial-cli-launch.md | 2 +- docs/zh-base/user-guide/tutorial-ecr.md | 24 +++++++++++++------ docs/zh-base/user-guide/tutorial-s3.md | 19 ++++++++------- .../test/requirements-test.txt | 4 ++-- .../lambda/cdk/test/requirements-test.txt | 8 +++---- .../s3/asg-helper/test/requirements-test.txt | 2 +- .../test/requirements-test.txt | 2 +- .../test/requirements-test.txt | 2 +- source/constructs/lib/cfn-step-functions.ts | 4 ++++ .../lib/ecr-plugin/ecr-plugin-stack.ts | 2 +- .../lib/s3-plugin/ec2-finder-stack.ts | 2 +- .../lib/s3-plugin/ec2-worker-stack.ts | 2 +- .../lib/s3-plugin/s3-plugin-stack.ts | 8 +++---- source/constructs/package.json | 8 +++---- .../test/requirements-test.txt | 4 ++-- 31 files changed, 125 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f835be7..633b6e5 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.6.1] - 2024-04-11 + +### Fixed + +- Fix the ECR transfer task creation issue #147 +- Fix the S3 transfer task creation issue #141 +- Fix the Gov Cloud S3 transfer task assets issue #140 +- Fix the error s3 plugin address issue #145 + ## [2.6.0] - 2024-01-18 ### Added diff --git a/README.md b/README.md index a5ae589..dbbac94 100644 --- a/README.md +++ b/README.md @@ -93,10 +93,7 @@ Create your first data transfer task, For the complete user guide, refer to ## FAQ **Q. Which are the supported Regions of this solution?**
-You can deploy this solution in these Regions: N.Virginia (us-east-1), Ohio (us-east-2), N.California (us-west-1), -Oregon (us-west-2), Mumbai (ap-south-1), Seoul (ap-northeast-2), Singapore (ap-southeast-1), Sydney (ap-southeast-2), -Tokyo (ap-northeast-1), Canada (ca-central-1), Frankfurt (eu-central-1), Ireland (eu-west-1), London (eu-west-2), -Paris (eu-west-3), Stockholm (eu-north-1), São Paulo (sa-east-1), Beijing (cn-north-1), Ningxia (cn-northwest-1). +Please refer to [Supported Regions](https://awslabs.github.io/data-transfer-hub/en/plan-deployment/regions/). **Q. If I want to copy data from U.S. Regions to China Regions, where should I deploy the solution?**
You can either deploy the solution in U.S. Regions or deploy the solution in China Regions. diff --git a/docs/ECR_DEPLOYMENT_CN.md b/docs/ECR_DEPLOYMENT_CN.md index b29a79c..17f5570 100644 --- a/docs/ECR_DEPLOYMENT_CN.md +++ b/docs/ECR_DEPLOYMENT_CN.md @@ -66,6 +66,10 @@ [![Launch Stack](./images/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/template?stackName=DTHECRStack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferECRStack.template) + - 部署到 AWS GovCloud (US) 区 + + [![Launch Stack](../images/launch-stack.svg)](https://console.amazonaws-us-gov.com/cloudformation/home#/stacks/create/template?stackName=DTHECRStack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferECRStack.template) + 1.单击**下一步**。 相应地为参数指定值。 如果需要，请更改堆栈名称。 diff --git a/docs/ECR_DEPLOYMENT_EN.md b/docs/ECR_DEPLOYMENT_EN.md index 794ab96..f170e42 100644 --- a/docs/ECR_DEPLOYMENT_EN.md +++ b/docs/ECR_DEPLOYMENT_EN.md @@ -65,7 +65,11 @@ Please follow below steps to deploy this plugin via AWS Cloudformation. - For AWS Global regions [![Launch Stack](./images/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/template?stackName=DTHECRStack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferECRStack.template) - + + - For AWS GovCloud (US) Regions + + [![Launch Stack](../images/launch-stack.svg)](https://console.amazonaws-us-gov.com/cloudformation/home#/stacks/create/template?stackName=DTHECRStack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferECRStack.template) + 1. Click **Next**. Specify values to parameters accordingly. Change the stack name if required. 1. Click **Next**. Configure additional stack options such as tags (Optional). diff --git a/docs/S3_DEPLOYMENT_CN.md b/docs/S3_DEPLOYMENT_CN.md index 2b67754..a2bb456 100644 --- a/docs/S3_DEPLOYMENT_CN.md +++ b/docs/S3_DEPLOYMENT_CN.md @@ -52,6 +52,9 @@ [![Launch Stack](./images/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/template?stackName=DTHS3Stack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferS3Stack.template) + - 部署到 AWS GovCloud (US) 区 + + [![Launch Stack](./images/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/template?stackName=DTHS3Stack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferS3Stack.template) 1. 单击**下一步**。 相应地为参数指定值。 如果需要，请更改堆栈名称。 diff --git a/docs/S3_DEPLOYMENT_EN.md b/docs/S3_DEPLOYMENT_EN.md index 02fe692..7bee122 100644 --- a/docs/S3_DEPLOYMENT_EN.md +++ b/docs/S3_DEPLOYMENT_EN.md @@ -52,6 +52,10 @@ Please follow below steps to deploy this solution via AWS Cloudformation. [![Launch Stack](./images/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/template?stackName=DTHS3Stack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferS3Stack.template) + - For AWS GovCloud (US) Regions + + [![Launch Stack](../images/launch-stack.svg)](https://console.amazonaws-us-gov.com/cloudformation/home#/stacks/create/template?stackName=DTHS3Stack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferS3Stack.template) + 1. Click **Next**. Specify values to parameters accordingly. Change the stack name if required. 1. Click **Next**. Configure additional stack options such as tags (Optional). diff --git a/docs/en-base/architecture-overview/architecture-details.md b/docs/en-base/architecture-overview/architecture-details.md index 35d8d5f..e8fad6d 100644 --- a/docs/en-base/architecture-overview/architecture-details.md +++ b/docs/en-base/architecture-overview/architecture-details.md @@ -33,8 +33,8 @@ This solution provides a simple web console which allows you to create and manag ### Amazon S3 transfer engine Amazon S3 transfer engine runs the Amazon S3 plugin and is used for transferring objects from their sources into S3 buckets. The S3 plugin supports the following features: -- Transfer Amazon S3 objects between AWS China Regions and AWS Regions -- Transfer objects from Alibaba Cloud OSS / Tencent COS / Qiniu Kodo to Amazon S3 +- Transfer Amazon S3 objects between AWS China Regions and AWS Global Regions +- Transfer objects from other cloud providers to Amazon S3 - Transfer objects from S3 Compatible Storage service to Amazon S3 - Support near real time transfer via S3 Event - Support transfer with object metadata @@ -46,7 +46,7 @@ Amazon S3 transfer engine runs the Amazon S3 plugin and is used for transferring Amazon ECR engine runs the Amazon ECR plugin and is used for transferring container images from other container registries. The ECR plugin supports the following features: -- Transfer Amazon ECR images between AWS China Regions and AWS Regions +- Transfer Amazon ECR images between AWS China Regions and AWS Global Regions - Transfer from public container registry (such as Docker Hub, GCR.io, Quay.io) to Amazon ECR - Transfer selected images to Amazon ECR - Transfer all images and tags from Amazon ECR diff --git a/docs/en-base/index.md b/docs/en-base/index.md index 6cc1d48..8da213a 100644 --- a/docs/en-base/index.md +++ b/docs/en-base/index.md @@ -1,4 +1,6 @@ -The Data Transfer Hub solution provides secure, scalable, and trackable data transfer for Amazon Simple Storage Service (Amazon S3) objects and Amazon Elastic Container Registry (Amazon ECR) images. This data transfer helps customers easily create and manage different types (Amazon S3 object and Amazon ECR image) of transfer tasks between AWS [partitions](https://docs.aws.amazon.com/whitepapers/latest/aws-fault-isolation-boundaries/partitions.html) (for example, aws, aws-cn, aws-us-gov), and from other cloud providers to AWS. +The Data Transfer Hub solution provides secure, scalable, and trackable data transfer for Amazon Simple Storage Service (Amazon S3) objects and Amazon Elastic Container Registry (Amazon ECR) images. This data transfer helps customers easily create and manage different types (Amazon S3 object and Amazon ECR image) of transfer tasks between AWS [partitions](https://docs.aws.amazon.com/whitepapers/latest/aws-fault-isolation-boundaries/partitions.html) (for example, aws, aws-cn, aws-us-gov), and from other cloud providers to AWS at your own discretion. + +If you have enabled the Direct Connect service in a specific AWS Region and a specific AWS China Region and have purchased a compliant cross-border dedicated line provided by a qualified operator to connect the AWS Region and their own VPC in the AWS China Region, you can use Data Transfer Hub's console to create a data transfer task and choose to use a dedicated line for data transfer at your own discretion. This implementation guide provides an overview of the Data Transfer Hub solution, its reference architecture and components, considerations for planning the deployment, configuration steps for deploying the Data Transfer Hub solution to the AWS Cloud. diff --git a/docs/en-base/solution-overview/features-and-benefits.md b/docs/en-base/solution-overview/features-and-benefits.md index fbd58d3..5444045 100644 --- a/docs/en-base/solution-overview/features-and-benefits.md +++ b/docs/en-base/solution-overview/features-and-benefits.md @@ -1,4 +1,4 @@ -The solution supports the following key features: +Depending on the availability of your network environment, the solution supports the following key features: - **Inter-Partition and Cross-Cloud Data Transfer**: to promote seamless transfer capabilities in one place - **Auto scaling**: to allow rapid response to changes in file transfer traffic @@ -11,5 +11,7 @@ The solution supports the following key features: If you need to transfer Amazon S3 objects between AWS Regions, we recommend that you use [Cross-Region Replication][crr]. If you want to transfer Amazon S3 objects within the same AWS Region, we recommend using [Same-Region Replication][srr]. +For data transfer between AWS China Region and AWS Global Region, you will be responsible for your compliance with all applicable laws and regulations on cross-border data transfer (including purchasing compliant cross-border dedicated lines provided by qualified operators for data transfer, performing necessary government approval or filing), and shall initiate data transfer at your own discretion. AWS does not assist you with this data transfer. + [crr]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/replication.html#crr-scenario [srr]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/replication.html#srr-scenario \ No newline at end of file diff --git a/docs/en-base/solution-overview/use-cases.md b/docs/en-base/solution-overview/use-cases.md index 1e99991..c9d7ab8 100644 --- a/docs/en-base/solution-overview/use-cases.md +++ b/docs/en-base/solution-overview/use-cases.md @@ -1,10 +1,10 @@ Today, the China market is one of biggest markets in the world. Many international companies are seeking their success in China, as well as a number of Chinese companies are expanding their businesses globally. One of most important steps of in the business is moving their data. -S3 Cross-Region Replication and ECR Cross-Region Replication are popular but customers cannot use them to replicate data into China Regions. With the launch of Data Transfer Hub solution, customers can now create S3 and ECR data transfer tasks between AWS Regions and AWS China Regions in a web portal. Moreover, it supports replicating data from cloud providers to AWS. +S3 Cross-Region Replication and ECR Cross-Region Replication are popular but customers cannot use them to replicate data into China Regions. With the launch of Data Transfer Hub solution, customers can now create S3 and ECR data transfer tasks between AWS Global Regions and AWS China Regions in a web portal. Moreover, it supports replicating data from other cloud providers to AWS. -Data Transfer Hub supports the following use cases: +Depending on the availability of your network environment, Data Transfer Hub supports the following use cases: -* Copy Amazon S3 objects between AWS Regions and AWS China Regions. -* Copy data from other cloud providers’ object storage services (including Alibaba Cloud OSS, Tencent COS, Qiniu Kodo) to Amazon S3. -* Transfer Amazon ECR images between AWS Regions and AWS China Regions. +* Copy Amazon S3 objects between AWS Global Regions and AWS China Regions. +* Copy data from other cloud providers’ object storage services to Amazon S3. +* Transfer Amazon ECR images between AWS Global Regions and AWS China Regions. * Transfer Dockers image from public docker registry (for example, Docker Hub, Google gcr.io, Red Hat Quay.io) to Amazon ECR. diff --git a/docs/en-base/user-guide/tutorial-ecr.md b/docs/en-base/user-guide/tutorial-ecr.md index e722194..b1958b6 100644 --- a/docs/en-base/user-guide/tutorial-ecr.md +++ b/docs/en-base/user-guide/tutorial-ecr.md @@ -1,10 +1,16 @@ The solution allows you to create an Amazon ECR transfer task in the following ways: - [using the web console](#using-the-web-console) -- [using the ECR plugin](#using-the-dth-ecr-plugin) +- [using the ECR plugin](#using-the-ecr-plugin) - [using AWS CLI](#using-aws-cli) -For a comparison between those options, refer to [Create Amazon S3 transfer task](./tutorial-s3.md). +You can make your choice according to your needs. + +- The web console provides an intuitive user interface where you can start, clone or stop a data transfer task with a simple click. The frontend also provides metric monitoring and logging view, so you do not need to switch between different pages. + +- The ECR plugin is a standalone CloudFormation template, and you can easily integrate it into your workflows. Because this option allows deployment without the frontend, it is useful if you want to deploy in AWS China Regions but do not have an ICP licensed domain. + +- AWS CLI can quickly initiate data transfer tasks. Select this option if you want to leverage Data Transfer Hub in your automation scripts. ## Using the web console You can use the web console to create an Amazon ECR transfer task. For more information about how to launch the web console, see [deployment](../../deployment/deployment-overview). @@ -101,6 +107,10 @@ Please follow below steps to deploy this plugin via AWS Cloudformation. - For AWS Global Regions [![Launch Stack](../images/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/template?stackName=DTHECRStack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferECRStack.template) + + - For AWS GovCloud (US) Regions + + [![Launch Stack](../images/launch-stack.svg)](https://console.amazonaws-us-gov.com/cloudformation/home#/stacks/create/template?stackName=DTHECRStack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferECRStack.template) 1. Click **Next**. Specify values to parameters accordingly. Change the stack name if required. diff --git a/docs/en-base/user-guide/tutorial-s3.md b/docs/en-base/user-guide/tutorial-s3.md index 4a9f320..ccb1a2a 100644 --- a/docs/en-base/user-guide/tutorial-s3.md +++ b/docs/en-base/user-guide/tutorial-s3.md @@ -1,7 +1,7 @@ The solution allows you to create an Amazon S3 transfer task in the following ways: - [using the web console](#using-the-web-console) -- [using the S3 plugin](#using-the-dth-s3-plugin) +- [using the S3 plugin](#using-the-s3-plugin) - [using AWS CLI](#using-aws-cli) You can make your choice according to your needs. @@ -164,6 +164,10 @@ Please follow below steps to deploy this solution via AWS Cloudformation. - For AWS Global Regions [![Launch Stack](../images/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/template?stackName=DTHS3Stack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferS3Stack.template) + + - For AWS GovCloud (US) Regions + + [![Launch Stack](../images/launch-stack.svg)](https://console.amazonaws-us-gov.com/cloudformation/home#/stacks/create/template?stackName=DTHS3Stack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferS3Stack.template) 1. Click **Next**. Specify values to parameters accordingly. Change the stack name if required. diff --git a/docs/zh-base/architecture-overview/architecture-details.md b/docs/zh-base/architecture-overview/architecture-details.md index 9fcaf27..6f00182 100644 --- a/docs/zh-base/architecture-overview/architecture-details.md +++ b/docs/zh-base/architecture-overview/architecture-details.md @@ -34,8 +34,8 @@ Amazon S3传输引擎运行Amazon S3插件，用于将对象从其来源传输到S3存储桶中。S3插件支持以下功能： -- 在AWS中国区域和AWS区域之间传输Amazon S3对象 -- 从阿里云OSS/Tencent COS/Qiniu Kodo传输对象到Amazon S3 +- 在AWS中国区域和其它AWS区域之间传输Amazon S3对象 +- 从其它云提供商传输对象到Amazon S3 - 从S3兼容存储服务传输对象到Amazon S3 - 支持通过S3事件进行几乎实时的传输 - 支持传输对象元数据 @@ -47,7 +47,7 @@ Amazon S3传输引擎运行Amazon S3插件，用于将对象从其来源传输 Amazon ECR引擎运行Amazon ECR插件，用于从其他容器注册表传输容器镜像。ECR插件支持以下功能： -- 在AWS中国区域和AWS区域之间传输Amazon ECR镜像 +- 在AWS中国区域和其它AWS区域之间传输Amazon ECR镜像 - 从公共容器注册表（如Docker Hub、GCR.io、Quay.io）传输到Amazon ECR - 传输选定的镜像到Amazon ECR - 从Amazon ECR传输所有镜像和标签 diff --git a/docs/zh-base/index.md b/docs/zh-base/index.md index 5feeffa..505d0f8 100644 --- a/docs/zh-base/index.md +++ b/docs/zh-base/index.md @@ -1,4 +1,6 @@ -**数据传输解决方案**（Data Transfer Hub）为 Amazon Simple Storage Service（Amazon S3）对象和 Amazon Elastic Container Registry（Amazon ECR）镜像提供安全、可扩展和可跟踪的数据传输。通过轻松地在亚马逊云科技（Amazon Web Services，AWS）不同[分区](https://docs.aws.amazon.com/whitepapers/latest/aws-fault-isolation-boundaries/partitions.html)之间（例如，aws, aws-cn, aws-us-gov）或者从其它云服务提供商到 AWS 创建并管理多种类型的传输任务，该数据传输帮助客户拓展全球业务。 +**数据传输解决方案**（Data Transfer Hub）为 Amazon Simple Storage Service（Amazon S3）对象和 Amazon Elastic Container Registry（Amazon ECR）镜像提供安全、可扩展和可跟踪的数据传输。该方案通过自主并轻松地在亚马逊云科技（Amazon Web Services，AWS）不同[分区](https://docs.aws.amazon.com/whitepapers/latest/aws-fault-isolation-boundaries/partitions.html)之间（例如，aws, aws-cn, aws-us-gov）或者从其它云服务提供商到 AWS 创建并管理多种类型的传输任务，从而帮助客户拓展全球业务。 + +对于已经在特定海外区域和特定中国区域均开启了 Direct Connect 服务并已经采购了有资质的运营商提供的合规跨境专线连接该海外区域和该中国区域自用 VPC 的客户，您可以自主通过 Data Transfer Hub 的控制台创建数据传输任务，并自主选择使用专线进行传输。 本实施指南提供了数据传输解决方案的概述、参考架构和组件、规划部署的注意事项以及将数据传输解决方案部署到 AWS 云的配置步骤。 diff --git a/docs/zh-base/solution-overview/features-and-benefits.md b/docs/zh-base/solution-overview/features-and-benefits.md index ac4569e..530f005 100644 --- a/docs/zh-base/solution-overview/features-and-benefits.md +++ b/docs/zh-base/solution-overview/features-and-benefits.md @@ -6,10 +6,13 @@ - 监控：跟踪数据流、诊断问题并确保数据传输过程的整体健康状况 - 开箱即用的部署 +以上功能均受限于客户网络环境的可用性。 !!! note "注意" 如果您需要在AWS区域之间传输Amazon S3对象，我们建议使用[跨区域复制][crr]。如果您想在同一AWS区域内传输Amazon S3对象，我们建议使用[同区域复制][srr]。 +对于 AWS 中国区域和海外区域之间的数据传输，您应遵守适用于您的关于数据跨境传输的所有法律法规（包括采购有资质的运营商提供的合规跨境专线进行数据传输、履行必要的政府审批或备案），并自主启动数据的传输。AWS 并不协助您进行该等数据传输。 + [crr]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/replication.html#crr-scenario [srr]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/replication.html#srr-scenario diff --git a/docs/zh-base/solution-overview/use-cases.md b/docs/zh-base/solution-overview/use-cases.md index 0c542dc..071ab34 100644 --- a/docs/zh-base/solution-overview/use-cases.md +++ b/docs/zh-base/solution-overview/use-cases.md @@ -1,10 +1,10 @@ 当今的中国市场是世界上最大的市场之一。许多国际公司正在寻求在中国取得成功，同样，一些中国公司也在全球范围内扩展业务。在业务中有很重要的一步是对数据的移动。 -虽然 S3 跨区域复制和 ECR 跨区域复制非常流行，但客户无法使用它们将数据复制到中国区域。随着 Data Transfer Hub 解决方案的推出，客户现在可以在 Web 门户中创建 AWS 区域和 AWS 中国区域之间的 S3 和 ECR 数据传输任务。此外，它还支持将数据从云提供商复制到 AWS。 +虽然 S3 跨区域复制和 ECR 跨区域复制非常流行，但客户无法使用它们将数据复制到中国区域。随着 Data Transfer Hub 解决方案的推出，客户现在可以在 Web 门户中创建 AWS 中国区域和其它 AWS 区域之间的 S3 和 ECR 数据传输任务。此外，它还支持将数据从其它云服务提供商复制到 AWS。 -Data Transfer Hub 支持以下用例： +受限于客户网络环境的可用性，Data Transfer Hub 支持以下用例： -* 在 AWS 区域和 AWS 中国区域之间复制 Amazon S3 对象。 -* 将其他云提供商的对象存储服务（包括阿里云 OSS、腾讯 COS、七牛 Kodo）的数据复制到 Amazon S3。 -* 在 AWS 区域和 AWS 中国区域之间传输 Amazon ECR 镜像。 +* 在 AWS 中国区域和其它 AWS 区域之间复制 Amazon S3 对象。 +* 将其它云提供商的对象存储服务的数据复制到 Amazon S3。 +* 在 AWS 中国区域和其它 AWS 区域之间传输 Amazon ECR 镜像。 * 将来自公共 Docker 容器镜像仓库（例如 Docker Hub、Google gcr.io、Red Hat Quay.io）的 Docker 镜像传输到 Amazon ECR。 diff --git a/docs/zh-base/user-guide/tutorial-cli-launch.md b/docs/zh-base/user-guide/tutorial-cli-launch.md index 2f59d4d..866c766 100644 --- a/docs/zh-base/user-guide/tutorial-cli-launch.md +++ b/docs/zh-base/user-guide/tutorial-cli-launch.md @@ -2,7 +2,7 @@ 1. 创建一个具有两个公有子网或两个拥有[NAT 网关][nat] 私有子网的Amazon VPC。 -2. 根据需要替换``为`https://s3.amazonaws.com/solutions-reference/data-transfer-hub/latest/DataTransferS3Stack-ec2.template`。 +2. 根据需要替换``为`https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferS3Stack.template`。 3. 转到您的终端并输入以下命令。详情请参考**参数列表**。 diff --git a/docs/zh-base/user-guide/tutorial-ecr.md b/docs/zh-base/user-guide/tutorial-ecr.md index 507aeaa..1224863 100644 --- a/docs/zh-base/user-guide/tutorial-ecr.md +++ b/docs/zh-base/user-guide/tutorial-ecr.md @@ -1,12 +1,18 @@ 该解决方案允许您通过以下方式创建 Amazon ECR 传输任务： -- 使用控制台传输任务 -- 使用 DTH ECR 插件传输任务 -- 使用 AWS CLI 创建传输任务 +- [使用控制台传输任务](#console-launch) +- [使用 DTH ECR 插件传输任务](#plugin-launch) +- [使用AWS CLI创建传输任务](#cli-launch) -想要了解这几种方式的更多信息，可参考[创建 Amazon S3 传输任务](./tutorial-s3.md)。 +您可以根据您的需要进行选择。 -## 使用控制台传输任务 +- Web 控制台提供直观的用户界面，您只需单击即可启动、克隆或停止数据传输任务。前端还提供指标监控和日志记录视图，因此您无需在不同页面之间切换。 + +- ECR 插件是一个独立的 CloudFormation 模板，您可以轻松地将其集成到您的工作流程中。由于此选项允许在没有前端的情况下进行部署，因此如果您想在 AWS 中国区域部署但没有 ICP 备案的域名，则此选项非常有用。 + +- AWS CLI可以快速启动数据传输任务。如果您想在自动化脚本中使用该解决方案，请选择此选项。 + +## 使用控制台传输任务 您可以在网页控制台创建Amazon ECR数据传输任务。更多信息请参考[部署解决方案](../../deployment/deployment-overview)。 1. 从**创建传输任务**页面，选择**创建新任务**，然后选择**下一步**。 @@ -37,7 +43,7 @@ 任务创建成功后，会出现在任务页面。 -## 使用DTH ECR 插件传输任务 +## 使用DTH ECR 插件传输任务 !!! note "注意" @@ -102,6 +108,10 @@ [![Launch Stack](../images/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/template?stackName=DTHECRStack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferECRStack.template) + - 部署到AWS GovCloud (US) 区 + + [![Launch Stack](../images/launch-stack.svg)](https://console.amazonaws-us-gov.com/cloudformation/home#/stacks/create/template?stackName=DTHECRStack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferECRStack.template) + 1.单击**下一步**。 相应地为参数指定值。 如果需要，请更改堆栈名称。 @@ -111,7 +121,7 @@ 部署预计用时3-5分钟 -## 使用AWS CLI创建传输任务 +## 使用AWS CLI创建传输任务 您可以使用 [AWS CLI][aws-cli] 创建 Amazon ECR传输任务。如果您同时部署了DTH Portal，通过CLI启动的任务将不会出现在您Portal的任务列表界面中。 1. 创建一个具有两个公有子网或两个拥有[NAT 网关][nat] 私有子网的Amazon VPC。 diff --git a/docs/zh-base/user-guide/tutorial-s3.md b/docs/zh-base/user-guide/tutorial-s3.md index a926b4a..ed307c4 100644 --- a/docs/zh-base/user-guide/tutorial-s3.md +++ b/docs/zh-base/user-guide/tutorial-s3.md @@ -1,8 +1,8 @@ 该解决方案允许您通过以下方式创建 Amazon S3 传输任务： -- 使用控制台传输任务 -- 使用DTH S3 插件创建传输任务 -- 使用AWS CLI创建传输任务 +- [使用控制台传输任务](#console-launch) +- [使用DTH S3 插件创建传输任务](#plugin-launch) +- [使用AWS CLI创建传输任务](#cli-launch) 您可以根据您的需要进行选择。 @@ -12,7 +12,7 @@ - AWS CLI可以快速启动数据传输任务。如果您想在自动化脚本中使用该解决方案，请选择此选项。 -## 使用控制台传输任务 +## 使用控制台传输任务 您可以在网页控制台创建Amazon S3数据传输任务。更多信息请参考[部署解决方案](../../deployment/deployment-overview)。 !!! Note "注意" @@ -119,7 +119,7 @@ -## 使用DTH S3 插件创建传输任务 +## 使用DTH S3 插件创建传输任务 !!! Note "注意" 本教程是纯后端版本的部署指南。如需了解详情，请参考该[DTH S3插件介绍](https://github.com/awslabs/data-transfer-hub/blob/main/docs/S3_PLUGIN_CN.md). @@ -149,7 +149,7 @@ 然后下一步指定密钥名称，最后一步点击创建。 -> 注意：如果该AK/SK是针对源桶, 则需要具有桶的**读**权限, 如果是针对目标桶, 则需要具有桶的**读与写**权限。 如果是Amazon S3, 可以参考[配置凭据](./IAM-Policy_CN.md) +> 注意：如果该AK/SK是针对源桶, 则需要具有桶的**读**权限, 如果是针对目标桶, 则需要具有桶的**读与写**权限。 如果是Amazon S3, 可以参考[配置凭据](../tutorial/IAM-Policy_CN.md) **3. 启动AWS Cloudformation部署** @@ -167,7 +167,10 @@ - 部署到AWS海外区 [![Launch Stack](../images/launch-stack.svg)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/template?stackName=DTHS3Stack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferS3Stack.template) + + - 部署到AWS GovCloud (US) 区 + [![Launch Stack](../images/launch-stack.svg)](https://console.amazonaws-us-gov.com/cloudformation/home#/stacks/create/template?stackName=DTHS3Stack&templateURL=https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferS3Stack.template) 1. 单击**下一步**。 相应地为参数指定值。 如果需要，请更改堆栈名称。 @@ -178,12 +181,12 @@ 部署预计用时3-5分钟 -## 使用AWS CLI创建传输任务 +## 使用AWS CLI创建传输任务 您可以使用 [AWS CLI][aws-cli] 创建 Amazon S3 传输任务。如果您同时部署了DTH Portal，通过CLI启动的任务将不会出现在您Portal的任务列表界面中。 1. 创建一个具有两个公有子网或两个拥有[NAT 网关][nat] 私有子网的Amazon VPC。 -2. 根据需要替换``为`https://s3.amazonaws.com/solutions-reference/data-transfer-hub/latest/DataTransferS3Stack-ec2.template`。 +2. 根据需要替换``为`https://solutions-reference.s3.amazonaws.com/data-transfer-hub/latest/DataTransferS3Stack.template`。 3. 转到您的终端并输入以下命令。详情请参考**参数列表**。 diff --git a/source/constructs/lambda/api/task-monitoring/test/requirements-test.txt b/source/constructs/lambda/api/task-monitoring/test/requirements-test.txt index 4b34868..493e829 100644 --- a/source/constructs/lambda/api/task-monitoring/test/requirements-test.txt +++ b/source/constructs/lambda/api/task-monitoring/test/requirements-test.txt @@ -1,6 +1,6 @@ -moto +moto==3.1.18 pytest pytest-cov pyyaml -openapi_spec_validator +openapi_spec_validator==0.3.3 docker \ No newline at end of file diff --git a/source/constructs/lambda/cdk/test/requirements-test.txt b/source/constructs/lambda/cdk/test/requirements-test.txt index 4b34868..85203ff 100644 --- a/source/constructs/lambda/cdk/test/requirements-test.txt +++ b/source/constructs/lambda/cdk/test/requirements-test.txt @@ -1,6 +1,6 @@ -moto -pytest -pytest-cov +moto==3.1.18 +pytest==7.1.2 +pytest-cov==3.0.0 pyyaml -openapi_spec_validator +openapi_spec_validator==0.3.3 docker \ No newline at end of file diff --git a/source/constructs/lambda/plugin/s3/asg-helper/test/requirements-test.txt b/source/constructs/lambda/plugin/s3/asg-helper/test/requirements-test.txt index f95d46d..4387f72 100644 --- a/source/constructs/lambda/plugin/s3/asg-helper/test/requirements-test.txt +++ b/source/constructs/lambda/plugin/s3/asg-helper/test/requirements-test.txt @@ -1,3 +1,3 @@ -moto +moto==3.1.18 pytest pytest-cov \ No newline at end of file diff --git a/source/constructs/lambda/plugin/s3/custom-resource/test/requirements-test.txt b/source/constructs/lambda/plugin/s3/custom-resource/test/requirements-test.txt index 5e06ff7..1cc2afe 100644 --- a/source/constructs/lambda/plugin/s3/custom-resource/test/requirements-test.txt +++ b/source/constructs/lambda/plugin/s3/custom-resource/test/requirements-test.txt @@ -1,4 +1,4 @@ boto3 -moto +moto==3.1.18 pytest pytest-cov diff --git a/source/constructs/lambda/plugin/s3/multi-part-controller/test/requirements-test.txt b/source/constructs/lambda/plugin/s3/multi-part-controller/test/requirements-test.txt index f95d46d..4387f72 100644 --- a/source/constructs/lambda/plugin/s3/multi-part-controller/test/requirements-test.txt +++ b/source/constructs/lambda/plugin/s3/multi-part-controller/test/requirements-test.txt @@ -1,3 +1,3 @@ -moto +moto==3.1.18 pytest pytest-cov \ No newline at end of file diff --git a/source/constructs/lib/cfn-step-functions.ts b/source/constructs/lib/cfn-step-functions.ts index 2100446..a296ea0 100644 --- a/source/constructs/lib/cfn-step-functions.ts +++ b/source/constructs/lib/cfn-step-functions.ts @@ -160,6 +160,9 @@ export class CloudFormationStateMachine extends Construct { "dynamodb:UpdateItem", "dynamodb:DescribeContinuousBackups", "dynamodb:UpdateContinuousBackups", + "dynamodb:UpdateContinuousBackups", + "dynamodb:TagResource", + "dynamodb:ListTagsOfResource", ], resources: [ `arn:${Aws.PARTITION}:dynamodb:${Aws.REGION}:${Aws.ACCOUNT_ID}:table/DTH*`, @@ -214,6 +217,7 @@ export class CloudFormationStateMachine extends Construct { "ecs:RegisterTaskDefinition", "ecs:DeregisterTaskDefinition", "ecs:DescribeTaskDefinition", + "ecs:TagResource", ], resources: [`*`] }), diff --git a/source/constructs/lib/ecr-plugin/ecr-plugin-stack.ts b/source/constructs/lib/ecr-plugin/ecr-plugin-stack.ts index 692d8e0..037533a 100644 --- a/source/constructs/lib/ecr-plugin/ecr-plugin-stack.ts +++ b/source/constructs/lib/ecr-plugin/ecr-plugin-stack.ts @@ -597,7 +597,7 @@ export class DataTransferECRStack extends Stack { 'ecs:RunTask' ], resources: [ - taskDefArnNoVersion + `${taskDefArnNoVersion}*` ] }), new iam.PolicyStatement({ diff --git a/source/constructs/lib/s3-plugin/ec2-finder-stack.ts b/source/constructs/lib/s3-plugin/ec2-finder-stack.ts index deff54a..5511c98 100644 --- a/source/constructs/lib/s3-plugin/ec2-finder-stack.ts +++ b/source/constructs/lib/s3-plugin/ec2-finder-stack.ts @@ -134,7 +134,7 @@ export class Ec2FinderStack extends Construct { assetDomain: 'https://aws-gcr-solutions-assets.s3.cn-northwest-1.amazonaws.com.cn', }, 'aws-us-gov': { - assetDomain: 'https://aws-gcr-solutions-assets.s3.amazonaws.com', + assetDomain: 'https://aws-gcr-solutions-assets.s3.amazonaws.com', }, } }); diff --git a/source/constructs/lib/s3-plugin/ec2-worker-stack.ts b/source/constructs/lib/s3-plugin/ec2-worker-stack.ts index eddd351..5cf0608 100644 --- a/source/constructs/lib/s3-plugin/ec2-worker-stack.ts +++ b/source/constructs/lib/s3-plugin/ec2-worker-stack.ts @@ -172,7 +172,7 @@ export class Ec2WorkerStack extends Construct { assetDomain: 'https://aws-gcr-solutions-assets.s3.cn-northwest-1.amazonaws.com.cn', }, 'aws-us-gov': { - assetDomain: 'https://aws-gcr-solutions-assets.s3.amazonaws.com', + assetDomain: 'https://aws-gcr-solutions-assets.s3.amazonaws.com', }, } }); diff --git a/source/constructs/lib/s3-plugin/s3-plugin-stack.ts b/source/constructs/lib/s3-plugin/s3-plugin-stack.ts index 25481a0..7bdf8b9 100644 --- a/source/constructs/lib/s3-plugin/s3-plugin-stack.ts +++ b/source/constructs/lib/s3-plugin/s3-plugin-stack.ts @@ -665,10 +665,8 @@ export class DataTransferS3Stack extends Stack { }) ); - // Here we create the notification resource by default - // Using cdk condition to enable or disable this notification - // Using cdk Aspects to modify the event type. - // Lambda to enable bucket notification of log source account. + // Here we using Custom resource Lambda to determine whether to enable or disable S3 Event. + // We do not use CloudFormation intrinsic function to create S3 Event because our creation logic is complex. const s3NotificationHelperFn = new lambda.Function( this, "s3NotificationHelperFn", @@ -692,7 +690,7 @@ export class DataTransferS3Stack extends Stack { } } ); - // Create the policy and role for the Lambda to create and delete CloudWatch Log Group Subscription Filter with cross-account scenario + // Create the policy and role for the Lambda to create and delete S3 notification s3NotificationHelperFn.addToRolePolicy( new iam.PolicyStatement({ actions: ["s3:GetBucketNotification", "s3:PutBucketNotification"], diff --git a/source/constructs/package.json b/source/constructs/package.json index 52ad2df..7aa8f2f 100755 --- a/source/constructs/package.json +++ b/source/constructs/package.json @@ -23,8 +23,8 @@ "@types/jest": "29.5.3", "@types/node": "20.11.0", "@types/uuid": "9.0.2", - "aws-cdk": "2.121.1", - "aws-cdk-lib": "2.121.1", + "aws-cdk": "2.136.0", + "aws-cdk-lib": "2.136.0", "jest": "^29.7.0", "ts-jest": "^29.1.1", "ts-node": "^10.2.1", @@ -35,8 +35,8 @@ "@aws-solutions-constructs/aws-cloudfront-s3": "2.45.0", "@aws-solutions-constructs/core": "2.45.0", "@aws-cdk/aws-appsync-alpha": "2.59.0-alpha.0", - "aws-cdk": "2.121.1", - "aws-cdk-lib": "2.121.1", + "aws-cdk": "2.136.0", + "aws-cdk-lib": "2.136.0", "cdk-nag": "2.28.3", "constructs": "10.3.0", "source-map-support": "0.5.21" diff --git a/source/custom-resource/test/requirements-test.txt b/source/custom-resource/test/requirements-test.txt index 4b34868..493e829 100644 --- a/source/custom-resource/test/requirements-test.txt +++ b/source/custom-resource/test/requirements-test.txt @@ -1,6 +1,6 @@ -moto +moto==3.1.18 pytest pytest-cov pyyaml -openapi_spec_validator +openapi_spec_validator==0.3.3 docker \ No newline at end of file