From c747c2ad59d81edf0b76289348806d2a5b8da1af Mon Sep 17 00:00:00 2001 From: HeoKyungseok <87405823+HeoKyungseok@users.noreply.github.com> Date: Mon, 13 Feb 2023 10:14:56 +0900 Subject: [PATCH] =?UTF-8?q?acl=20=EB=B3=B5=EC=82=AC=EB=A5=BC=20source,=20t?= =?UTF-8?q?arget=20conf=EC=97=90=20=EC=A7=80=EC=A0=95=ED=95=98=EB=8F=84?= =?UTF-8?q?=EB=A1=9D=20=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 27 +++++++ src/main/java/ifs_mover/Config.java | 23 +++++- src/main/java/ifs_mover/repository/IfsS3.java | 79 ++++++++++++------- 3 files changed, 101 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 6dbb07a..2b2f0dc 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ source.conf project_id project id for swift project_name project name for swift container list of containers(If it is empty, it means all container) + acl object ACL on, off target.conf endpoint http(https)://IP:Port | region access Access Key ID @@ -76,6 +77,7 @@ target.conf versioning bucket versioning on, off sync target object sync on, off sync_mode target object sync mode, [etag|size|exist] + acl object ACL on, off 주) –o는 향후 개발 예정 ``` @@ -140,6 +142,7 @@ source.conf bucket: Bucket Name prefix: MOVE를 시작할 PREFIX/DIR 이름 정보 move_size: The size of the file that you can move at once. (M bytes) + acl: object ACL 정보 획득 여부(on/off) user_name user name for swift api_key api key for swift auth_endpoint http(https)://IP:port/v3 @@ -168,6 +171,7 @@ target.conf exist : target에 source object가 존재하는 경우 skip sync=on 이고, sync_mode 값이 없는 경우 etag가 기본 값으로 설정된다. * 주의 - type=file인 경우에는 etag로 지정하여도 etag를 검사하지 않는다. type=file 인 경우 source 파일의 etag를 수집하지 않음. + acl: object ACL 복사 여부(on/off) ``` ### 설정 파일 예시 @@ -229,6 +233,27 @@ target.conf prefix= ``` +#### S3 -> S3 (move-test/move_old_objects/* -> /move-test/*) + ACL 정보도 같이 복사하려는 경우 +```sh +source.conf + mountpoint= + endpoint=http://www.s3abc.com:8080 + access=a9dad4ce7233sdfesdfsd + secret=sdfsdfsdfcd408e83e23dab92 + bucket=move-test + prefix=move_old_objects + move_size= + acl=on + +target.conf + endpoint=https://www.s3other.com:8443 + access=a9dad4ce7233sdfesdfsd + secret=sdfsdfsdfcd408e83e23dab92 + bucket=move-test + prefix= + acl=on +``` + #### S3 -> S3 (AWS) (/move-test/0720/* -> /move-test/*) ```sh source.conf @@ -565,3 +590,5 @@ db_pool_size=10 // mariadb 시 db connection pool size ./ifs_mover -t=s3 -source=source.conf -target=target.conf * 더 자세한 실행 방법은 본 문서의 "실행 예시", "설정 파일 예시"를 참조하세요. + + diff --git a/src/main/java/ifs_mover/Config.java b/src/main/java/ifs_mover/Config.java index 09d4a68..18b003b 100644 --- a/src/main/java/ifs_mover/Config.java +++ b/src/main/java/ifs_mover/Config.java @@ -39,6 +39,7 @@ public class Config { private String sync; private String syncCheck; private String type; + private String acl; // for openstack swift private String userName; @@ -52,7 +53,10 @@ public class Config { private boolean isAWS; private boolean isTargetSync; - private SyncMode syncMode; + private SyncMode syncMode; + + // for acl info + private boolean isACL; private final String MOUNT_POINT = "mountpoint"; private final String END_POINT = "endpoint"; @@ -65,6 +69,7 @@ public class Config { private final String VERSIONING = "versioning"; // ON OFF private final String TARGET_SYNC = "sync"; private final String TARGET_SYNC_MODE = "sync_mode"; + private final String ACL = "acl"; // for openstack swift support private final String USER_NAME = "user_name"; @@ -119,6 +124,7 @@ public void configure() { versioning = properties.getProperty(VERSIONING); sync = properties.getProperty(TARGET_SYNC); syncCheck = properties.getProperty(TARGET_SYNC_MODE); + acl = properties.getProperty(ACL); if (mountPoint != null && !mountPoint.isEmpty() && !mountPoint.endsWith("/")) { mountPoint += "/"; @@ -182,6 +188,17 @@ public void configure() { } else { syncMode = SyncMode.UNKNOWN; } + + if (acl != null &&!acl.isEmpty()) { + acl = acl.toLowerCase(); + if (acl.compareTo(ON) == 0) { + isACL = true; + } else { + isACL = false; + } + } else { + isACL = false; + } } public boolean isAWS() { @@ -309,4 +326,8 @@ public void setType(String type) { public String getVersoning() { return versioning; } + + public boolean isACL() { + return isACL; + } } diff --git a/src/main/java/ifs_mover/repository/IfsS3.java b/src/main/java/ifs_mover/repository/IfsS3.java index 2d238a3..c8940a4 100644 --- a/src/main/java/ifs_mover/repository/IfsS3.java +++ b/src/main/java/ifs_mover/repository/IfsS3.java @@ -83,6 +83,7 @@ public class IfsS3 implements Repository, S3 { private String errMessage; private BucketVersioningConfiguration versionConfig; private boolean targetVersioning; + private boolean isACL; private final String HTTPS = "https"; private final String AWS_S3_V4_SIGNER_TYPE = "AWSS3V4SignerType"; @@ -129,6 +130,7 @@ public void setConfig(Config config, boolean isSource) { isTargetSync = false; targetSyncMode = SyncMode.UNKNOWN; } + isACL = config.isACL(); } @Override @@ -902,17 +904,23 @@ public ObjectData getObject(String bucket, String key, String versionId) { // if (versionId == null || versionId.equalsIgnoreCase("null")) { if (versionId == null) { getObjectRequest = new GetObjectRequest(bucket, key); - getObjectAclRequest = new GetObjectAclRequest(bucket, key); + if (isACL) { + getObjectAclRequest = new GetObjectAclRequest(bucket, key); + } } else { getObjectRequest = new GetObjectRequest(bucket, key).withVersionId(versionId); - getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId); + if (isACL) { + getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId); + } } s3Object = client.getObject(getObjectRequest); data.setS3Object(s3Object); data.setMetadata(s3Object.getObjectMetadata()); data.setInputStream(s3Object.getObjectContent()); data.setSize(s3Object.getObjectMetadata().getContentLength()); - data.setAcl(client.getObjectAcl(getObjectAclRequest)); + if (isACL) { + data.setAcl(client.getObjectAcl(getObjectAclRequest)); + } return data; } @@ -926,17 +934,23 @@ public ObjectData getObject(String bucket, String key, String versionId, long st if (versionId == null) { getObjectRequest = new GetObjectRequest(bucket, key).withRange(start); - getObjectAclRequest = new GetObjectAclRequest(bucket, key); + if (isACL) { + getObjectAclRequest = new GetObjectAclRequest(bucket, key); + } } else { getObjectRequest = new GetObjectRequest(bucket, key).withVersionId(versionId).withRange(start); - getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId); + if (isACL) { + getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId); + } } s3Object = client.getObject(getObjectRequest); data.setS3Object(s3Object); data.setMetadata(s3Object.getObjectMetadata()); data.setInputStream(s3Object.getObjectContent()); data.setSize(s3Object.getObjectMetadata().getContentLength()); - data.setAcl(client.getObjectAcl(getObjectAclRequest)); + if (isACL) { + data.setAcl(client.getObjectAcl(getObjectAclRequest)); + } return data; } @@ -949,16 +963,22 @@ public ObjectData getObject(String bucket, String key, String versionId, long st if (versionId == null) { getObjectRequest = new GetObjectRequest(bucket, key).withRange(start, end); - getObjectAclRequest = new GetObjectAclRequest(bucket, key); + if (isACL) { + getObjectAclRequest = new GetObjectAclRequest(bucket, key); + } } else { getObjectRequest = new GetObjectRequest(bucket, key).withVersionId(versionId).withRange(start, end); - getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId); + if (isACL) { + getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId); + } } s3Object = client.getObject(getObjectRequest); data.setS3Object(s3Object); data.setInputStream(s3Object.getObjectContent()); data.setSize(s3Object.getObjectMetadata().getContentLength()); - data.setAcl(client.getObjectAcl(getObjectAclRequest)); + if (isACL) { + data.setAcl(client.getObjectAcl(getObjectAclRequest)); + } return data; } @@ -1259,31 +1279,36 @@ public SyncMode getTargetSyncMode() { @Override public void setAcl(String bucket, String key, String versionId, AccessControlList acl) { // TODO Auto-generated method stub - SetObjectAclRequest setObjectAclRequest = null; - if (acl.getGrantsAsList().size() == 0) { - if (versionId != null) { - setObjectAclRequest = new SetObjectAclRequest(bucket, key, versionId, CannedAccessControlList.Private); - } else { - setObjectAclRequest = new SetObjectAclRequest(bucket, key, CannedAccessControlList.Private); - } - } else { - if (versionId != null) { - setObjectAclRequest = new SetObjectAclRequest(bucket, key, versionId, acl); + if (isACL) { + SetObjectAclRequest setObjectAclRequest = null; + if (acl.getGrantsAsList().size() == 0) { + if (versionId != null) { + setObjectAclRequest = new SetObjectAclRequest(bucket, key, versionId, CannedAccessControlList.Private); + } else { + setObjectAclRequest = new SetObjectAclRequest(bucket, key, CannedAccessControlList.Private); + } } else { - setObjectAclRequest = new SetObjectAclRequest(bucket, key, acl); + if (versionId != null) { + setObjectAclRequest = new SetObjectAclRequest(bucket, key, versionId, acl); + } else { + setObjectAclRequest = new SetObjectAclRequest(bucket, key, acl); + } } + client.setObjectAcl(setObjectAclRequest); } - client.setObjectAcl(setObjectAclRequest); } @Override public AccessControlList getAcl(String bucket, String key, String versionId) { - GetObjectAclRequest getObjectAclRequest = null; - if (versionId != null) { - getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId); - } else { - getObjectAclRequest = new GetObjectAclRequest(bucket, key); + if (isACL) { + GetObjectAclRequest getObjectAclRequest = null; + if (versionId != null) { + getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId); + } else { + getObjectAclRequest = new GetObjectAclRequest(bucket, key); + } + return client.getObjectAcl(getObjectAclRequest); } - return client.getObjectAcl(getObjectAclRequest); + return null; } }