-
Notifications
You must be signed in to change notification settings - Fork 2
/
s3sync-compare-boto3.py
74 lines (61 loc) · 2.47 KB
/
s3sync-compare-boto3.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#!/usr/bin/python3
#
# iterates over buckets and compares the md5 checksum across objects on primary and secondary connection
#
import sys
import re
import boto3
access_key = 'XXXXXXXXXXXXXXXXXXXX'
secret_key = 'YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY'
rgw_endpoint_primary = 'http://primary:80'
rgw_endpoint_secondary = 'http://secondary:80'
compare_dicts = True
conn_primary = boto3.client('s3',
endpoint_url=rgw_endpoint_primary,
aws_access_key_id = access_key,
aws_secret_access_key = secret_key,
)
conn_secondary = boto3.client('s3',
endpoint_url=rgw_endpoint_secondary,
aws_access_key_id = access_key,
aws_secret_access_key = secret_key,
)
def bucket2list(s3, bucket_name):
result = {}
response = s3.list_objects_v2(Bucket=bucket_name)
while True:
for item in response['Contents']:
result[item['Key']]=item['ETag']
if not response['IsTruncated']:
break;
print("Paging results: ", response['NextContinuationToken'])
response = s3.list_objects_v2(Bucket=bucket_name,ContinuationToken=response['NextContinuationToken'])
return result
if len(sys.argv) == 2:
bucketlist = {'Buckets':[{'Name':sys.argv[1]}]}
else:
bucketlist = conn_primary.list_buckets()
for bucket in bucketlist['Buckets']:
print("Bucket: ", bucket['Name'])
bucketlist_primary = bucket2list(conn_primary, bucket['Name'])
try:
# bucketlist_secondary = bucket2list(conn_secondary, 'test3')
bucketlist_secondary = bucket2list(conn_secondary, bucket['Name'])
for item in bucketlist_primary:
print ("P",item, bucketlist_primary[item])
if item in bucketlist_secondary:
print ("S", item, bucketlist_secondary[item])
if bucketlist_primary[item] != bucketlist_secondary[item]:
print ("ERROR: MD5 mismatch", item, bucketlist_primary[item], bucketlist_secondary[item])
else:
print ("ERROR: no such object on secondary: ", item)
if compare_dicts:
if bucketlist_primary == bucketlist_secondary:
print("Buckets do match")
else:
print("Buckets do not match")
except Exception as e:
if re.search("NoSuchBucket", str(e)):
print ("ERROR: bucket does not exist on secondary: ", bucket['Name'])
else:
print ("ERROR: unable to validate object on secondary, Exception: ", e)