Skip to content

Instantly share code, notes, and snippets.

@akhan4u
Last active May 25, 2021 08:15
Show Gist options
  • Save akhan4u/44e19f657739281d491d10e09fe87f82 to your computer and use it in GitHub Desktop.
Save akhan4u/44e19f657739281d491d10e09fe87f82 to your computer and use it in GitHub Desktop.
s3 replication cross region within same AWS account
import boto3, os, botocore, traceback
from botocore.exceptions import ClientError
s3 = boto3.client('s3')
# READ VALUES FROM ENVIRONMENT VARIABLES
source_s3_bucket = os.environ['source_s3_bucket']
#kms_keyid = os.environ['Kms_Keyid']
# APPEND REGION NAME FOR DESTINATION BUCKET
dest_s3_bucket = source_s3_bucket + '-rep-us-east-2'
# CREATE S3 BUCKET LIST
buckets = [source_s3_bucket, dest_s3_bucket]
# VERIFY IF BUCKET EXISTS
def check_bucket(bucket_name):
try:
s3.head_bucket(Bucket=bucket_name)
print("Bucket Exists!")
return True
except botocore.exceptions.ClientError as e:
error_code = int(e.response['Error']['Code']) # IF A CLIENT ERROR IS THROWN, THEN CHECK THAT IT WAS A 404 ERROR.
if error_code == 403: # YOU DONT HAVE ACCESS TO BUCKET
print("Private Bucket. Forbidden Access!")
return True
elif error_code == 404: # IF IT WAS A 404 ERROR, THEN THE BUCKET DOES NOT EXIST
print("Bucket Does Not Exist! Creating Bucket.....")
create_bucket(bucket_name)
print("Bucket Created")
return False
# CREATE S3 BUCKET
def create_bucket(bucket_name):
s3.create_bucket(
ACL='private',
Bucket=bucket_name,
CreateBucketConfiguration={
'LocationConstraint': 'us-east-2'
}
)
# GET BUCKET VERSIONING
def check_versioning(bucket_name):
bucket_response = s3.get_bucket_versioning(Bucket=bucket_name)
try:
status = bucket_response['Status']
except KeyError:
print('Versioning not ENABLED. Enabling versioning for ' + bucket_name)
enable_versioning(bucket_name)
else:
if status == 'Enabled':
print('Versioning already ENABLED - ' + bucket_name)
# ENABLE VERSIONING
def enable_versioning(bucket_name):
s3.put_bucket_versioning(
Bucket=bucket_name,
VersioningConfiguration={
'Status': 'Enabled'
}
)
# ENABLE REPLICATION ON SOURCE BUCKET
def enable_replication(source_s3_bucket):
s3.put_bucket_replication(
Bucket=source_s3_bucket,
ReplicationConfiguration={
"Role": "arn:aws:iam::142705040095:role/service-role/s3crr_role_for_vm-managed-bucket-rep-checking_to_vm-managed-buck",
"Rules": [
{
"ID": "vm-managed-s3-bucket-replication",
"Priority": 1,
"Filter": {},
"Status": "Enabled",
"Destination": {
"Bucket": "arn:aws:s3:::vm-managed-bucket-for-rep-rep-us-east-2"
},
"DeleteMarkerReplication": {
"Status": "Disabled"
}
}
]
}
)
for bucket in buckets:
check_bucket(bucket)
check_versioning(bucket)
try:
rep_status = s3.get_bucket_replication(Bucket=source_s3_bucket)
data = str(rep_status['ReplicationConfiguration']['Rules'][0]['Status'])
print(data)
except ClientError as e:
error_code = str(e.response['Error']['Code'])
print(error_code)
if error_code == 'ReplicationConfigurationNotFoundError':
print('Enabling Replication on Bucket ' + source_s3_bucket)
enable_replication(source_s3_bucket)
@akhan4u
Copy link
Author

akhan4u commented Feb 28, 2020

Error Received while handling check_replication method:

Bucket Exists!
Versioning already ENABLED - vm-managed-bucket-for-rep
Traceback (most recent call last):
File "create_and_update_s3_bucket.py", line 106, in
check_replication(bucket)
File "create_and_update_s3_bucket.py", line 63, in check_replication
Bucket=bucket_name
File "/usr/local/lib/python3.6/dist-packages/botocore/client.py", line 276, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/usr/local/lib/python3.6/dist-packages/botocore/client.py", line 586, in _make_api_call
raise error_class(parsed_response, operation_name)
botocore.exceptions.ClientError: An error occurred (ReplicationConfigurationNotFoundError) when calling the GetBucketReplication operation: The replication configuration was not found

@yum-dev
Copy link

yum-dev commented Feb 29, 2020

except ClientError as e:
try this instead of botocore.exceptions.ClientError

@akhan4u
Copy link
Author

akhan4u commented Feb 29, 2020

except ClientError as e:
try this instead of botocore.exceptions.ClientError

I have added the new updated Gist. Check the newest revision.

Actually the method that I was calling get_bucket_replication() was itself generating the exception, which was earlier outside the try block. Not the evaluation condition inside the try. So kept the method get_bucket_replication() itself under try and could catch the exception error code.

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment