Skip to content

Instantly share code, notes, and snippets.

@freewayz
Created October 6, 2016 15:35
Show Gist options
  • Star 14 You must be signed in to star a gist
  • Fork 3 You must be signed in to fork a gist
  • Save freewayz/38f850a7b0c49f8e31c951c2eaf2c2c1 to your computer and use it in GitHub Desktop.
Save freewayz/38f850a7b0c49f8e31c951c2eaf2c2c1 to your computer and use it in GitHub Desktop.
Download files and folder from amazon s3 using boto and pytho local system
#!/usr/bin/env python
import boto
import sys, os
from boto.s3.key import Key
from boto.exception import S3ResponseError
DOWNLOAD_LOCATION_PATH = os.path.expanduser("~") + "/s3-backup/"
if not os.path.exists(DOWNLOAD_LOCATION_PATH):
print ("Making download directory")
os.mkdir(DOWNLOAD_LOCATION_PATH)
def backup_s3_folder():
BUCKET_NAME = "skoolsresources.com"
AWS_ACCESS_KEY_ID= os.getenv("AWS_KEY_ID") # set your AWS_KEY_ID on your environment path
AWS_ACCESS_SECRET_KEY = os.getenv("AWS_ACCESS_KEY") # set your AWS_ACCESS_KEY on your environment path
conn = boto.connect_s3(AWS_ACCESS_KEY_ID, AWS_ACCESS_SECRET_KEY)
bucket = conn.get_bucket(BUCKET_NAME)
#goto through the list of files
bucket_list = bucket.list()
for l in bucket_list:
key_string = str(l.key)
s3_path = DOWNLOAD_LOCATION_PATH + key_string
try:
print ("Current File is ", s3_path)
l.get_contents_to_filename(s3_path)
except (OSError,S3ResponseError) as e:
pass
# check if the file has been downloaded locally
if not os.path.exists(s3_path):
try:
os.makedirs(s3_path)
except OSError as exc:
# let guard againts race conditions
import errno
if exc.errno != errno.EEXIST:
raise
if __name__ == '__main__':
backup_s3_folder()
@h4bibfigueredo
Copy link

h4bibfigueredo commented Jul 8, 2019

Tks for the code, but I am was trying to use this to download multiple files and seems like my S3Connection isn't working, at least that my perception.

Newbiew here, trying to access first my bucket with a print but isn't working....

import boto
import sys,os
from boto.s3.key import Key
from boto.s3.connection import S3Connection

download_location_path = os.path.expanduser("~") + "/s3-backup/"

if not os.path.exists(download_location_path):
print("Making download dir")
os.mkdir(download_location_path)

def backup_s3_folder():
bucket_name = 'enterprise-2'
access_key = 'enterprise7'
secret_key = 'xxxxx'
host = 'etc.org'
ports = 9021
connect = S3Connection(access_key,secret_key,host,ports)
bucket = connect.get_bucket('ed5-enterprise-data-services-uat-2')

bucket_list = bucket.list('archives/01apr19/yyy')
return print(bucket_list)

"""
for l in bucket_list:
key_string = str(l.key)
s3_path = download_location_path + key_string
try:
print("Current file is ", s3_path)
l.get_contents_to_filename(s3_path)
except (OSError, S3ResponseError) as e:
pass
if not os.path.exists(s3_path):
try:
os.mkdir(s3_path)
except OSError as exc:
import errno
if exc.errno != errno.EEXIST:
raise
"""
if name == 'main':
backup_s3_folder()

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment