Skip to content

Commit

Permalink
aws s3 버전 업데이트, DeleteMarker 테스트 추가
Browse files Browse the repository at this point in the history
  • Loading branch information
@pspace-jwkwak
pspace-jwkwak committed Nov 10, 2022
1 parent 05b2a9b commit 3b58f2f
Show file tree
Hide file tree
Showing 2 changed files with 78 additions and 14 deletions.
83 changes: 73 additions & 10 deletions C#/ReplicationTest/ReplicationTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Threading;
Expand Down Expand Up @@ -114,16 +115,35 @@ public void Start(BucketData MainBucket, bool SSL)
log.Info("Bucket Replication!");

// 파일 업로드
var ObjectList = new List<string>() { "aaa", "bbb", "ccc", "1/ddd", "2/eee" };
var ObjectList = new List<string>() { "aaa", "bbb", "ccc", "1/ddd", "1/eee", "2/fff" };

// Put
foreach (var ObjectName in ObjectList) PutObject(MainClient, MainBucket.BucketName, ObjectName);

// Delete
DeleteObject(MainClient, MainBucket.BucketName, "1/eee");

// Copy
CopyObject(MainClient, MainBucket.BucketName, ObjectList[0], MainBucket.BucketName, "copy");

// another copy
var AntherBucket = MainBucket.BucketName + "-anther";
var CopyKey = "source-copy";
CreateBucket(MainClient, AntherBucket);
PutObject(MainClient, AntherBucket, CopyKey);
CopyObject(MainClient, AntherBucket, CopyKey, MainBucket.BucketName, "another-copy");

// multipart
MultipartUpload(MainClient, MainBucket.BucketName, "multi");
log.Info("Upload End!");

Thread.Sleep(Config.Delay * 1000);

// 버저닝 정보를 포함한 비교
if (Config.TestOption != TEST_OPTION_ANOTHER_ONLY) Compare2Target(MainClient, MainBucket, MainClient, MainTargetList);
if (Config.TestOption != TEST_OPTION_LOCAL_ONLY) Compare2Target(MainClient, MainBucket, AltClient, AltTargetList);
log.Info("Replication Check End!");
BucketClear(MainClient, AntherBucket);
}
else
log.Info("Bucket Replication Failed!");
Expand Down Expand Up @@ -155,6 +175,7 @@ static List<ReplicationRule> CreateRelicationRules(List<BucketData> Buckets, int
Status = ReplicationRuleStatus.Enabled,
Destination = new ReplicationDestination() { BucketArn = $"{BucketArnPrefix}{Bucket.BucketName}" },
Filter = new ReplicationRuleFilter() { Prefix = Bucket.Prefix, },
DeleteMarkerReplication = new DeleteMarkerReplication() { Status = Bucket.DeleteMarker ? DeleteMarkerReplicationStatus.Enabled : DeleteMarkerReplicationStatus.Disabled }
};

Rules.Add(Rule);
Expand Down Expand Up @@ -188,8 +209,10 @@ void Compare2Target(AmazonS3Client Main, BucketData MainBucket, AmazonS3Client A
foreach (var Bucket in Buckets)
{
log.Info($"[{Bucket.BucketName}] Compare Check!");
var SourceData = GetListVersions(Main, MainBucket.BucketName, Bucket.Filtering ? Bucket.Prefix : null);
var TargetData = GetListVersions(Alt, Bucket.BucketName, Bucket.Filtering ? Bucket.Prefix : null);
var SourceData = GetListVersions(Main, MainBucket.BucketName, Bucket.Filtering ? Bucket.Prefix : null).OrderBy(x=> x.Key).ThenByDescending(x=> x.VersionId).ToList();
var TargetData = GetListVersions(Alt, Bucket.BucketName, Bucket.Filtering ? Bucket.Prefix : null).OrderBy(x=> x.Key).ThenByDescending(x=> x.VersionId).ToList();

if (!Bucket.DeleteMarker) SourceData.RemoveAll(i => i.IsDeleteMarker);

var Message = CompareMain2Alt(SourceData, TargetData);
var Result = "";
Expand All @@ -200,6 +223,11 @@ void Compare2Target(AmazonS3Client Main, BucketData MainBucket, AmazonS3Client A
}
else
{
Console.WriteLine("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
foreach (var Version in SourceData) Console.WriteLine($"[{Version.Key:10}\t{Version.Size}\t{Version.ETag}\t{Version.IsDeleteMarker}\t{Version.VersionId}]");
Console.WriteLine("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
foreach (var Version in TargetData) Console.WriteLine($"[{Version.Key:10}\t{Version.Size}\t{Version.ETag}\t{Version.IsDeleteMarker}\t{Version.VersionId}]");
Console.WriteLine("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
log.Error($"[{Bucket.BucketName}] is not match! -> {Message}");
Result = "Failed";
}
Expand All @@ -210,14 +238,13 @@ void Compare2Target(AmazonS3Client Main, BucketData MainBucket, AmazonS3Client A
string CompareMain2Alt(List<S3ObjectVersion> Main, List<S3ObjectVersion> Alt)
{
if (Main.Count != Alt.Count) return $"{Main.Count} != {Alt.Count}";

for (int i = 0; i < Main.Count; i++)
for (int Index = 0; Index < Main.Count; Index++)
{
if (Main[i].Key != Alt[i].Key) return $"{Main[i].Key} != {Alt[i].Key}";
if (Main[i].Size != Alt[i].Size) return $"{Main[i].Key} Size does not match! {Main[i].Size} != {Alt[i].Size}";
if (Config.CheckEtag && Main[i].ETag != Alt[i].ETag) return $"{Main[i].Key} ETag does not match! {Main[i].ETag} != {Alt[i].ETag}";
if (Main[i].IsDeleteMarker != Alt[i].IsDeleteMarker) return $"{Main[i].Key} DeleteMarker does not match! {Main[i].IsDeleteMarker} != {Alt[i].IsDeleteMarker}";
if (Config.CheckVersionId && Main[i].VersionId != Alt[i].VersionId) return $"{Main[i].Key} VersionId does not match! {Main[i].VersionId} != {Alt[i].VersionId}";
if (Main[Index].Key != Alt[Index].Key) return $"{Main[Index].Key} != {Alt[Index].Key}";
if (Main[Index].Size != Alt[Index].Size) return $"{Main[Index].Key} Size does not match! {Main[Index].Size} != {Alt[Index].Size}";
if (Config.CheckEtag && Main[Index].ETag != Alt[Index].ETag) return $"{Main[Index].Key} ETag does not match! {Main[Index].ETag} != {Alt[Index].ETag}";
if (Main[Index].IsDeleteMarker != Alt[Index].IsDeleteMarker) return $"{Main[Index].Key} DeleteMarker does not match! {Main[Index].IsDeleteMarker} != {Alt[Index].IsDeleteMarker}";
if (!Main[Index].IsDeleteMarker && Config.CheckVersionId && Main[Index].VersionId != Alt[Index].VersionId) return $"{Main[Index].Key} VersionId does not match! {Main[Index].VersionId} != {Alt[Index].VersionId}";
}
return "";
}
Expand Down Expand Up @@ -395,6 +422,42 @@ static void PutObject(AmazonS3Client Client, string BucketName, string ObjectNam
catch (AggregateException e) { log.Error($"StatusCode : {Utility.GetStatus(e)}, ErrorCode : {Utility.GetErrorCode(e)}", e); }
catch (Exception e) { log.Error(e); }
}
static void DeleteObject(AmazonS3Client Client, string BucketName, string ObjectName)
{
try
{
var Request = new DeleteObjectRequest
{
BucketName = BucketName,
Key = ObjectName,
};

var TaskResponse = Client.DeleteObjectAsync(Request);
TaskResponse.Wait();
var Response = TaskResponse.Result;
}
catch (AggregateException e) { log.Error($"StatusCode : {Utility.GetStatus(e)}, ErrorCode : {Utility.GetErrorCode(e)}", e); }
catch (Exception e) { log.Error(e); }
}
static void CopyObject(AmazonS3Client Client, string SourceBucket, string SourceKey, string DestinationBucket, string DestinationKey)
{
try
{
var Request = new CopyObjectRequest
{
SourceBucket = SourceBucket,
SourceKey = SourceKey,
DestinationBucket = DestinationBucket,
DestinationKey = DestinationKey,
};

var TaskResponse = Client.CopyObjectAsync(Request);
TaskResponse.Wait();
var Response = TaskResponse.Result;
}
catch (AggregateException e) { log.Error($"StatusCode : {Utility.GetStatus(e)}, ErrorCode : {Utility.GetErrorCode(e)}", e); }
catch (Exception e) { log.Error(e); }
}
static bool MultipartUpload(AmazonS3Client Client, string BucketName, string ObjectName)
{
try
Expand Down
9 changes: 5 additions & 4 deletions C#/ReplicationTest/ReplicationTest.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
<AssemblyVersion>1.1.0.0</AssemblyVersion>
</PropertyGroup>

<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
Expand All @@ -14,10 +15,10 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="AWSSDK.Core" Version="3.5.3.9" />
<PackageReference Include="AWSSDK.S3" Version="3.5.10.2" />
<PackageReference Include="log4net" Version="2.0.14" />
<PackageReference Include="MySql.Data" Version="8.0.28" />
<PackageReference Include="AWSSDK.Core" Version="3.7.100.13" />
<PackageReference Include="AWSSDK.S3" Version="3.7.101.13" />
<PackageReference Include="log4net" Version="2.0.15" />
<PackageReference Include="MySql.Data" Version="8.0.31" />
</ItemGroup>

<ItemGroup>
Expand Down

0 comments on commit 3b58f2f

Please sign in to comment.