Skip to content

Commit

Permalink
acl 정보 전송 및 isLatest 버그 수정
Browse files Browse the repository at this point in the history
  • Loading branch information
@HeoKyungseok
HeoKyungseok committed Feb 10, 2023
1 parent 4c13194 commit 8e611b3
Show file tree
Hide file tree
Showing 7 changed files with 139 additions and 30 deletions.
52 changes: 34 additions & 18 deletions src/main/java/ifs_mover/ObjectMover.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,12 @@
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.GetObjectAclRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.Tag;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;
Expand Down Expand Up @@ -487,6 +491,8 @@ private boolean moveObject(String path, boolean isDelete, boolean isFile, String
Map<String, String> tagMap = null;
List<Tag> tagSet = new ArrayList<Tag>();

logger.debug("path:{}, versionId:{}, size:{}", path, versionId, size);

if (type.equalsIgnoreCase(Repository.SWIFT)) {
sourceBucket = path.split("/", 2)[0];
sourcePath = path.split("/", 2)[1];
Expand Down Expand Up @@ -649,9 +655,9 @@ private boolean moveObject(String path, boolean isDelete, boolean isFile, String
}
} while (data != null);

targetRepository.completeMultipart(targetBucket, targetPath, uploadId, partList);
CompleteMultipartUploadResult completeMultipartUploadResult = targetRepository.completeMultipart(targetBucket, targetPath, uploadId, partList);
if (tagSet.size() > 0) {
targetRepository.setTagging(targetBucket, targetPath, tagSet);
targetRepository.setTagging(targetBucket, targetPath, completeMultipartUploadResult.getVersionId(), tagSet);
}
logger.info("move success : {}", path);
} else if ((!type.equalsIgnoreCase(Repository.IFS_FILE) && size > MAX_MULTIPART_SIZE)
Expand All @@ -670,6 +676,7 @@ private boolean moveObject(String path, boolean isDelete, boolean isFile, String
}

ObjectMetadata objectMetadata = sourceRepository.getMetadata(sourceBucket, sourcePath, versionId);
AccessControlList objectAcl = sourceRepository.getAcl(sourceBucket, sourcePath, versionId);
String uploadId = targetRepository.startMultipart(targetBucket, targetPath, objectMetadata);
List<PartETag> partList = new ArrayList<PartETag>();
int partNumber = 1;
Expand All @@ -691,10 +698,11 @@ private boolean moveObject(String path, boolean isDelete, boolean isFile, String
}
}

targetRepository.completeMultipart(targetBucket, targetPath, uploadId, partList);
CompleteMultipartUploadResult completeMultipartUploadResult = targetRepository.completeMultipart(targetBucket, targetPath, uploadId, partList);
targetRepository.setAcl(targetBucket, targetPath, completeMultipartUploadResult.getVersionId(), objectAcl);

if (tagSet.size() > 0) {
targetRepository.setTagging(targetBucket, targetPath, tagSet);
targetRepository.setTagging(targetBucket, targetPath, completeMultipartUploadResult.getVersionId(), tagSet);
}

if (versionId != null && !versionId.isEmpty()) {
Expand All @@ -705,9 +713,11 @@ private boolean moveObject(String path, boolean isDelete, boolean isFile, String
} else {
ObjectData data = null;
String s3ETag = null;
PutObjectResult putObjectResult = null;
if (type.equalsIgnoreCase(Repository.IFS_FILE)) {
data = sourceRepository.getObject(sourcePath);
s3ETag = targetRepository.putObject(isFile, targetBucket, targetPath, data, data.getSize());
putObjectResult = targetRepository.putObject(isFile, targetBucket, targetPath, data, data.getSize());
s3ETag = putObjectResult.getETag();
data.close();
} else {
if (size > 0) {
Expand All @@ -716,17 +726,26 @@ private boolean moveObject(String path, boolean isDelete, boolean isFile, String
logger.warn("not found : {} {}", sourceBucket, sourcePath);
return false;
}
s3ETag = targetRepository.putObject(isFile, targetBucket, targetPath, data, data.getSize());
putObjectResult = targetRepository.putObject(isFile, targetBucket, targetPath, data, data.getSize());
s3ETag = putObjectResult.getETag();
data.close();
} else {
ObjectMetadata meta = sourceRepository.getMetadata(sourceBucket, sourcePath, versionId);
AccessControlList acl = sourceRepository.getAcl(sourceBucket, sourcePath, versionId);
InputStream is = new ByteArrayInputStream(new byte[0]);
targetRepository.putObject(targetBucket, targetPath, is, meta);
logger.debug("size : {}, meta size : {}", size, meta.getContentLength());
putObjectResult = targetRepository.putObject(targetBucket, targetPath, is, meta);
is.close();
targetRepository.setAcl(targetBucket, targetPath, putObjectResult.getVersionId(), acl);
}
}

if (tagSet.size() > 0) {
targetRepository.setTagging(targetBucket, targetPath, tagSet);
targetRepository.setTagging(targetBucket, targetPath, putObjectResult.getVersionId(), tagSet);
}

if (data != null && data.getAcl() != null) {
targetRepository.setAcl(targetBucket, targetPath, putObjectResult.getVersionId(), data.getAcl());
}

if (!type.equalsIgnoreCase(Repository.IFS_FILE) && isFile && size > 0) {
Expand Down Expand Up @@ -775,19 +794,14 @@ private boolean moveObject(String path, boolean isDelete, boolean isFile, String
} else {
if (versionId != null && versionId.compareToIgnoreCase("null") == 0) {
ObjectMetadata meta = sourceRepository.getMetadata(sourceBucket, sourcePath, versionId);
AccessControlList acl = sourceRepository.getAcl(sourceBucket, sourcePath, versionId);
InputStream is = new ByteArrayInputStream(new byte[0]);
// data = sourceRepository.getObject(sourceBucket, sourcePath, versionId);
// data = sourceRepository.getObject(sourceBucket, sourcePath, null);
// if (data == null) {
// logger.warn("not found : {} {}", sourceBucket, sourcePath);
// return false;
// }
// targetRepository.putObject(isFile, targetBucket, targetPath, data, 0L);
targetRepository.putObject(targetBucket, targetPath, is, meta);
PutObjectResult putObjectResult = targetRepository.putObject(targetBucket, targetPath, is, meta);
is.close();
if (tagSet.size() > 0) {
targetRepository.setTagging(targetBucket, targetPath, tagSet);
targetRepository.setTagging(targetBucket, targetPath, putObjectResult.getVersionId(), tagSet);
}
targetRepository.setAcl(targetBucket, targetPath, putObjectResult.getVersionId(), acl);
} else {
logger.info("Ignore this object because Because it is a directory with versionid, path : {}, versionId : {}", path, versionId);
}
Expand All @@ -811,6 +825,7 @@ private boolean moveObject(String path, boolean isDelete, boolean isFile, String
}
return false;
} catch (AmazonClientException ace) {
Utils.logging(logger, ace);
logger.warn("{} {}", path, ace.getMessage());
// Ignore 'etag' even if it's different after gettingObject in S3
// try {
Expand Down Expand Up @@ -944,6 +959,7 @@ public void run() {
}

isTargetSkip = false;
logger.debug("path:{}, versionId:{}, size:{}", path, versionId, size);
retryMoveObject(path, isDelete, isFile, versionId, etag, multipartInfo, tag, size, skipCheck);
if (isTargetSkip) {
continue;
Expand All @@ -967,7 +983,7 @@ public void run() {
if (!isDoneLatest) {
// Latest should be performed at the end.
isTargetSkip = false;
retryMoveObject(latestPath, latestIsDelete, latestIsFile, latestVersionId, latestETag, latestMultipartInfo, latestTag, size, latestSkipCheck);
retryMoveObject(latestPath, latestIsDelete, latestIsFile, latestVersionId, latestETag, latestMultipartInfo, latestTag, latestSize, latestSkipCheck);

if (!isTargetSkip && !(isRerun && !skipCheck)) {
if (isFault) {
Expand Down
7 changes: 7 additions & 0 deletions src/main/java/ifs_mover/repository/IfsFile.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.ObjectMetadata;

import ifs_mover.Config;
Expand Down Expand Up @@ -309,4 +310,10 @@ public SyncMode getTargetSyncMode() {
// TODO Auto-generated method stub
return SyncMode.UNKNOWN;
}

@Override
public AccessControlList getAcl(String bucket, String key, String versionId) {
// TODO Auto-generated method stub
return null;
}
}
73 changes: 65 additions & 8 deletions src/main/java/ifs_mover/repository/IfsS3.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,12 @@
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.BucketVersioningConfiguration;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.GetObjectAclRequest;
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.GetObjectTaggingRequest;
Expand All @@ -46,10 +50,12 @@
import com.amazonaws.services.s3.model.ObjectTagging;
import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.model.S3VersionSummary;
import com.amazonaws.services.s3.model.SetBucketVersioningConfigurationRequest;
import com.amazonaws.services.s3.model.SetObjectAclRequest;
import com.amazonaws.services.s3.model.SetObjectTaggingRequest;
import com.amazonaws.services.s3.model.Tag;
import com.amazonaws.services.s3.model.UploadPartRequest;
Expand Down Expand Up @@ -890,19 +896,23 @@ public ObjectData getObject(String path) {
public ObjectData getObject(String bucket, String key, String versionId) {
ObjectData data = new ObjectData();
GetObjectRequest getObjectRequest = null;
GetObjectAclRequest getObjectAclRequest = null;
S3Object s3Object = null;

// if (versionId == null || versionId.equalsIgnoreCase("null")) {
if (versionId == null) {
getObjectRequest = new GetObjectRequest(bucket, key);
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
} else {
getObjectRequest = new GetObjectRequest(bucket, key).withVersionId(versionId);
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
}
s3Object = client.getObject(getObjectRequest);
data.setS3Object(s3Object);
data.setMetadata(s3Object.getObjectMetadata());
data.setInputStream(s3Object.getObjectContent());
data.setSize(s3Object.getObjectMetadata().getContentLength());
data.setAcl(client.getObjectAcl(getObjectAclRequest));

return data;
}
Expand All @@ -911,35 +921,44 @@ public ObjectData getObject(String bucket, String key, String versionId) {
public ObjectData getObject(String bucket, String key, String versionId, long start) {
ObjectData data = new ObjectData();
GetObjectRequest getObjectRequest = null;
GetObjectAclRequest getObjectAclRequest = null;
S3Object s3Object = null;

if (versionId == null) {
getObjectRequest = new GetObjectRequest(bucket, key).withRange(start);
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
} else {
getObjectRequest = new GetObjectRequest(bucket, key).withVersionId(versionId).withRange(start);
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
}
s3Object = client.getObject(getObjectRequest);
data.setS3Object(s3Object);
data.setMetadata(s3Object.getObjectMetadata());
data.setInputStream(s3Object.getObjectContent());
data.setSize(s3Object.getObjectMetadata().getContentLength());
data.setAcl(client.getObjectAcl(getObjectAclRequest));
return data;
}

@Override
public ObjectData getObject(String bucket, String key, String versionId, long start, long end) {
ObjectData data = new ObjectData();
GetObjectRequest getObjectRequest = null;
GetObjectAclRequest getObjectAclRequest = null;
S3Object s3Object = null;

if (versionId == null) {
getObjectRequest = new GetObjectRequest(bucket, key).withRange(start, end);
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
} else {
getObjectRequest = new GetObjectRequest(bucket, key).withVersionId(versionId).withRange(start, end);
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
}
s3Object = client.getObject(getObjectRequest);
data.setS3Object(s3Object);
data.setInputStream(s3Object.getObjectContent());
data.setSize(s3Object.getObjectMetadata().getContentLength());
data.setAcl(client.getObjectAcl(getObjectAclRequest));
return data;
}

Expand All @@ -957,17 +976,23 @@ public String uploadPart(String bucket, String key, String uploadId, InputStream
}

@Override
public String completeMultipart(String bucket, String key, String uploadId, List<PartETag> list) {
return client.completeMultipartUpload(new CompleteMultipartUploadRequest(bucket, key, uploadId, list)).getETag();
public CompleteMultipartUploadResult completeMultipart(String bucket, String key, String uploadId, List<PartETag> list) {
return client.completeMultipartUpload(new CompleteMultipartUploadRequest(bucket, key, uploadId, list));
}

@Override
public void setTagging(String bucket, String key, List<Tag> tagSet) {
client.setObjectTagging(new SetObjectTaggingRequest(bucket, key, new ObjectTagging(tagSet)));
public void setTagging(String bucket, String key, String versionId, List<Tag> tagSet) {
SetObjectTaggingRequest setObjectTaggingRequest = null;
if (versionId != null) {
setObjectTaggingRequest = new SetObjectTaggingRequest(bucket, key, versionId, new ObjectTagging(tagSet));
} else {
setObjectTaggingRequest = new SetObjectTaggingRequest(bucket, key, new ObjectTagging(tagSet));
}
client.setObjectTagging(setObjectTaggingRequest);
}

@Override
public String putObject(boolean isFile, String bucket, String key, ObjectData data, long size) {
public PutObjectResult putObject(boolean isFile, String bucket, String key, ObjectData data, long size) {
PutObjectRequest putObjectRequest = null;
if (data.getFile() != null) {
putObjectRequest = new PutObjectRequest(bucket, key, data.getFile());
Expand All @@ -979,7 +1004,8 @@ public String putObject(boolean isFile, String bucket, String key, ObjectData da
}
putObjectRequest = new PutObjectRequest(bucket, key, data.getInputStream(), data.getMetadata());
}
return client.putObject(putObjectRequest).getETag();

return client.putObject(putObjectRequest);
}

@Override
Expand Down Expand Up @@ -1155,9 +1181,9 @@ public void run() {
}

@Override
public String putObject(String bucketName, String key, InputStream input, ObjectMetadata metadata) {
public PutObjectResult putObject(String bucketName, String key, InputStream input, ObjectMetadata metadata) {
PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, key, input, metadata);
return client.putObject(putObjectRequest).getETag();
return client.putObject(putObjectRequest);
}

@Override
Expand Down Expand Up @@ -1229,4 +1255,35 @@ public boolean isTargetSync() {
public SyncMode getTargetSyncMode() {
return targetSyncMode;
}

@Override
public void setAcl(String bucket, String key, String versionId, AccessControlList acl) {
// TODO Auto-generated method stub
SetObjectAclRequest setObjectAclRequest = null;
if (acl.getGrantsAsList().size() == 0) {
if (versionId != null) {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, versionId, CannedAccessControlList.Private);
} else {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, CannedAccessControlList.Private);
}
} else {
if (versionId != null) {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, versionId, acl);
} else {
setObjectAclRequest = new SetObjectAclRequest(bucket, key, acl);
}
}
client.setObjectAcl(setObjectAclRequest);
}

@Override
public AccessControlList getAcl(String bucket, String key, String versionId) {
GetObjectAclRequest getObjectAclRequest = null;
if (versionId != null) {
getObjectAclRequest = new GetObjectAclRequest(bucket, key).withVersionId(versionId);
} else {
getObjectAclRequest = new GetObjectAclRequest(bucket, key);
}
return client.getObjectAcl(getObjectAclRequest);
}
}
11 changes: 11 additions & 0 deletions src/main/java/ifs_mover/repository/ObjectData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,22 @@
import java.io.IOException;
import java.io.InputStream;

import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;

public class ObjectData {
private S3Object s3Object;
private ObjectMetadata meta;
private AccessControlList acl;
private InputStream is;
private File file;
private long size;

public ObjectData() {
s3Object = null;
meta = null;
acl = null;
is = null;
file = null;
}
Expand All @@ -40,6 +43,10 @@ public void setMetadata(ObjectMetadata metadata) {
this.meta = metadata;
}

public void setAcl(AccessControlList acl) {
this.acl = acl;
}

public void setInputStream(InputStream is) {
this.is = is;
}
Expand All @@ -56,6 +63,10 @@ public ObjectMetadata getMetadata() {
return meta;
}

public AccessControlList getAcl() {
return acl;
}

public InputStream getInputStream() {
return is;
}
Expand Down
Loading

0 comments on commit 8e611b3

Please sign in to comment.