Skip to content

Commit

Permalink
acl 복사를 source, target conf에 지정하도록 수정
Browse files Browse the repository at this point in the history
  • Loading branch information
@HeoKyungseok
HeoKyungseok committed Feb 13, 2023
1 parent 8e611b3 commit c747c2a
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 28 deletions.
27 changes: 27 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ source.conf
project_id project id for swift
project_name project name for swift
container list of containers(If it is empty, it means all container)
acl object ACL on, off
target.conf
endpoint http(https)://IP:Port | region
access Access Key ID
Expand All @@ -76,6 +77,7 @@ target.conf
versioning bucket versioning on, off
sync target object sync on, off
sync_mode target object sync mode, [etag|size|exist]
acl object ACL on, off
주) –o는 향후 개발 예정
```
Expand Down Expand Up @@ -140,6 +142,7 @@ source.conf
bucket: Bucket Name
prefix: MOVE를 시작할 PREFIX/DIR 이름 정보
move_size: The size of the file that you can move at once. (M bytes)
acl: object ACL 정보 획득 여부(on/off)
user_name user name for swift
api_key api key for swift
auth_endpoint http(https)://IP:port/v3
Expand Down Expand Up @@ -168,6 +171,7 @@ target.conf
exist : target에 source object가 존재하는 경우 skip
sync=on 이고, sync_mode 값이 없는 경우 etag가 기본 값으로 설정된다.
* 주의 - type=file인 경우에는 etag로 지정하여도 etag를 검사하지 않는다. type=file 인 경우 source 파일의 etag를 수집하지 않음.
acl: object ACL 복사 여부(on/off)
```
### 설정 파일 예시
Expand Down Expand Up @@ -229,6 +233,27 @@ target.conf
prefix=
```
#### S3 -> S3 (move-test/move_old_objects/* -> /move-test/*) + ACL 정보도 같이 복사하려는 경우
```sh
source.conf
mountpoint=
endpoint=http://www.s3abc.com:8080
access=a9dad4ce7233sdfesdfsd
secret=sdfsdfsdfcd408e83e23dab92
bucket=move-test
prefix=move_old_objects
move_size=
acl=on
target.conf
endpoint=https://www.s3other.com:8443
access=a9dad4ce7233sdfesdfsd
secret=sdfsdfsdfcd408e83e23dab92
bucket=move-test
prefix=
acl=on
```
#### S3 -> S3 (AWS) (/move-test/0720/* -> /move-test/*)
```sh
source.conf
Expand Down Expand Up @@ -565,3 +590,5 @@ db_pool_size=10 // mariadb 시 db connection pool size
<kbd>./ifs_mover -t=s3 -source=source.conf -target=target.conf</kbd>
* 더 자세한 실행 방법은 본 문서의 "실행 예시", "설정 파일 예시"를 참조하세요.
23 changes: 22 additions & 1 deletion src/main/java/ifs_mover/Config.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ public class Config {
private String sync;
private String syncCheck;
private String type;
private String acl;

// for openstack swift
private String userName;
Expand All @@ -52,7 +53,10 @@ public class Config {

private boolean isAWS;
private boolean isTargetSync;
private SyncMode syncMode;
private SyncMode syncMode;

// for acl info
private boolean isACL;

private final String MOUNT_POINT = "mountpoint";
private final String END_POINT = "endpoint";
Expand All @@ -65,6 +69,7 @@ public class Config {
private final String VERSIONING = "versioning"; // ON OFF
private final String TARGET_SYNC = "sync";
private final String TARGET_SYNC_MODE = "sync_mode";
private final String ACL = "acl";

// for openstack swift support
private final String USER_NAME = "user_name";
Expand Down Expand Up @@ -119,6 +124,7 @@ public void configure() {
versioning = properties.getProperty(VERSIONING);
sync = properties.getProperty(TARGET_SYNC);
syncCheck = properties.getProperty(TARGET_SYNC_MODE);
acl = properties.getProperty(ACL);

if (mountPoint != null && !mountPoint.isEmpty() && !mountPoint.endsWith("/")) {
mountPoint += "/";
Expand Down Expand Up @@ -182,6 +188,17 @@ public void configure() {
} else {
syncMode = SyncMode.UNKNOWN;
}

if (acl != null &&!acl.isEmpty()) {
acl = acl.toLowerCase();
if (acl.compareTo(ON) == 0) {
isACL = true;
} else {
isACL = false;
}
} else {
isACL = false;
}
}

public boolean isAWS() {
Expand Down Expand Up @@ -309,4 +326,8 @@ public void setType(String type) {
public String getVersoning() {
return versioning;
}

public boolean isACL() {
return isACL;
}
}
79 changes: 52 additions & 27 deletions src/main/java/ifs_mover/repository/IfsS3.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ public class IfsS3 implements Repository, S3 {
private String errMessage;
private BucketVersioningConfiguration versionConfig;
private boolean targetVersioning;
private boolean isACL;

private final String HTTPS = "https";
private final String AWS_S3_V4_SIGNER_TYPE = "AWSS3V4SignerType";
Expand Down Expand Up @@ -129,6 +130,7 @@ public void setConfig(Config config, boolean isSource) {
isTargetSync = false;
targetSyncMode = SyncMode.UNKNOWN;
}
isACL = config.isACL();
}

@Override
Expand Down Expand Up @@ -902,17 +904,23 @@ public ObjectData getObject(String bucket, String key, String versionId) {
// if (versionId == null || versionId.equalsIgnoreCase("null")) {
if (versionId == null) {
getObjectRequest = new GetObjectRequest(bucket, key);
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
if (isACL) {
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
}
} else {
getObjectRequest = new GetObjectRequest(bucket, key).withVersionId(versionId);
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
if (isACL) {
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
}
}
s3Object = client.getObject(getObjectRequest);
data.setS3Object(s3Object);
data.setMetadata(s3Object.getObjectMetadata());
data.setInputStream(s3Object.getObjectContent());
data.setSize(s3Object.getObjectMetadata().getContentLength());
data.setAcl(client.getObjectAcl(getObjectAclRequest));
if (isACL) {
data.setAcl(client.getObjectAcl(getObjectAclRequest));
}

return data;
}
Expand All @@ -926,17 +934,23 @@ public ObjectData getObject(String bucket, String key, String versionId, long st

if (versionId == null) {
getObjectRequest = new GetObjectRequest(bucket, key).withRange(start);
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
if (isACL) {
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
}
} else {
getObjectRequest = new GetObjectRequest(bucket, key).withVersionId(versionId).withRange(start);
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
if (isACL) {
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
}
}
s3Object = client.getObject(getObjectRequest);
data.setS3Object(s3Object);
data.setMetadata(s3Object.getObjectMetadata());
data.setInputStream(s3Object.getObjectContent());
data.setSize(s3Object.getObjectMetadata().getContentLength());
data.setAcl(client.getObjectAcl(getObjectAclRequest));
if (isACL) {
data.setAcl(client.getObjectAcl(getObjectAclRequest));
}
return data;
}

Expand All @@ -949,16 +963,22 @@ public ObjectData getObject(String bucket, String key, String versionId, long st

if (versionId == null) {
getObjectRequest = new GetObjectRequest(bucket, key).withRange(start, end);
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
if (isACL) {
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
}
} else {
getObjectRequest = new GetObjectRequest(bucket, key).withVersionId(versionId).withRange(start, end);
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
if (isACL) {
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
}
}
s3Object = client.getObject(getObjectRequest);
data.setS3Object(s3Object);
data.setInputStream(s3Object.getObjectContent());
data.setSize(s3Object.getObjectMetadata().getContentLength());
data.setAcl(client.getObjectAcl(getObjectAclRequest));
if (isACL) {
data.setAcl(client.getObjectAcl(getObjectAclRequest));
}
return data;
}

Expand Down Expand Up @@ -1259,31 +1279,36 @@ public SyncMode getTargetSyncMode() {
@Override
public void setAcl(String bucket, String key, String versionId, AccessControlList acl) {
// TODO Auto-generated method stub
SetObjectAclRequest setObjectAclRequest = null;
if (acl.getGrantsAsList().size() == 0) {
if (versionId != null) {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, versionId, CannedAccessControlList.Private);
} else {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, CannedAccessControlList.Private);
}
} else {
if (versionId != null) {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, versionId, acl);
if (isACL) {
SetObjectAclRequest setObjectAclRequest = null;
if (acl.getGrantsAsList().size() == 0) {
if (versionId != null) {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, versionId, CannedAccessControlList.Private);
} else {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, CannedAccessControlList.Private);
}
} else {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, acl);
if (versionId != null) {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, versionId, acl);
} else {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, acl);
}
}
client.setObjectAcl(setObjectAclRequest);
}
client.setObjectAcl(setObjectAclRequest);
}

@Override
public AccessControlList getAcl(String bucket, String key, String versionId) {
GetObjectAclRequest getObjectAclRequest = null;
if (versionId != null) {
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
} else {
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
if (isACL) {
GetObjectAclRequest getObjectAclRequest = null;
if (versionId != null) {
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
} else {
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
}
return client.getObjectAcl(getObjectAclRequest);
}
return client.getObjectAcl(getObjectAclRequest);
return null;
}
}

0 comments on commit c747c2a

Please sign in to comment.