Last active
September 26, 2020 22:52
-
-
Save jonathanwcrane/c26ea5c5daf0a156342b to your computer and use it in GitHub Desktop.
Find failed multipart uploads in all buckets in a region
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/usr/bin/env python | |
#inventory of all multipart uploads in a region | |
import boto3 | |
import os | |
#Not sure this applies here but keeping just in case | |
region = "us-east-1" | |
#The profile to use in the credentials or boto.config file, if used | |
profile_nm = 'foo' | |
#The env var to look in for the AWS KEY, if used | |
aws_key_env_var = 'AWS_KEY' | |
#The env var to look in for the secret key, if used | |
aws_secret_key_env_var = 'AWS_SECRET_KEY' | |
#First check for ENV VARs with AWS Credentials | |
#and make the connection | |
aws_key = os.environ.get(aws_key_env_var) | |
aws_secret_key = os.environ.get(aws_secret_key_env_var) | |
if (aws_key and aws_secret_key): | |
print("Signing in using ENV VAR credentials") | |
aws_session = boto3.session.Session(aws_access_key_id=aws_key,aws_secret_access_key=aws_secret_key,region_name=region) | |
#If env vars don't exist, use the profile in the boto.config file | |
#If the env vars and profile both exist, the program will never look for the profile, and use the env vars. | |
else: | |
print("Signing in using boto config credentials") | |
aws_session = boto3.session.Session(region_name=region, profile_name=profile_nm) | |
s3 = aws_session.resource('s3') | |
tot_size = 0 | |
bytes_per_gb = 1073741824 | |
for bucket in s3.buckets.all(): | |
bucket_mpu_gb = 0 | |
#bucket = s3.Bucket(bn) | |
for mpu in bucket.multipart_uploads.all(): | |
#print(mpu.initiated) | |
mpu_size = 0 | |
for part in mpu.parts.all(): | |
mpu_size += part.size | |
mpu_gb = mpu_size / bytes_per_gb | |
print(bucket.name+","+str(mpu.initiated)+','+mpu.object_key+','+str(mpu_gb)) | |
#print("Total size of (presumably failed) multipart upload is",mpu_gb,"GB.") | |
bucket_mpu_gb += mpu_gb | |
print("bucket",bucket.name,bucket_mpu_gb,"GB") | |
tot_size += bucket_mpu_gb | |
print("total GB in failed multipart uploads is",tot_size) |
@jonathanwcrane ahh I see. I believe you can actually have boto-based utilities pick up environment variables automatically if they're named a certain way :). See here: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#environment-variables
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
@nijine you're welcome
actually this was probably running in Jenkins or something so it had to be like that because the ENV VARS are set based on what role you want to use or something (it's been a while so don't quote me on that)
@paulwakeford glad you find it useful and nice addition.