Create a gist now

Instantly share code, notes, and snippets.

Object Storage Uploader Script
# ================================================================================
# ObjectStorageUploader.py
# © Copyright IBM Corporation 2014.
# LICENSE: MIT (http://opensource.org/licenses/MIT)
# ================================================================================
import argparse
import os
import math
import http.client
from urllib.parse import urlparse
from urllib.parse import quote
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-f", "--file", help="file to upload to SWIFT")
parser.add_argument("-u", "--username", help="SWIFT username")
parser.add_argument("-p", "--password", help="SWIFT password (SoftLayer API key)")
parser.add_argument("-c", "--cluster", help="SWIFT cluster to use (default: dal05.objectstorage.softlayer.net)")
parser.add_argument("-t", "--target", help="location on SWIFT cluster to store file (container/filename.vhd)")
args = parser.parse_args()
if not args.file:
filename = select_file()
else:
filename = args.file
if not args.username:
swift_user_name, swift_password = get_swift_credentials()
else:
swift_user_name = args.username
swift_password = args.password
if not args.cluster:
storage_url, auth_token = authenticate_swift(
swift_user_name,
swift_password,
"dal05.objectstorage.softlayer.net"
)
else:
storage_url, auth_token = authenticate_swift(
swift_user_name,
swift_password,
args.cluster
)
if not args.target:
container = select_container(storage_url, auth_token)
swift_target_path = "{}/{}".format(container, quote(filename))
else:
swift_target_path = args.target
upload_file(filename, swift_target_path, storage_url, auth_token)
def select_file():
current_path = (os.path.dirname(os.path.realpath(__file__)))
print("Files in {}".format(current_path))
files = get_file_list(current_path)
return prompt_for_choice(files, "Select file for upload:")
def get_file_list(path):
files = []
for file in os.listdir(path):
if os.path.isfile(file):
files.append(file)
return files
def prompt_for_choice(list, prompt_label):
for i, item in enumerate(list):
if len(item):
print("{}) {}".format(i, item))
selected_index = input(prompt_label)
if selected_index.isdigit():
if 0 <= int(selected_index) < len(list):
print()
return list[int(selected_index)]
print("Invalid Input: {}".format(selected_index))
return prompt_for_choice(list, prompt_label)
def get_swift_credentials():
swift_user_name = input("Swift Username:")
swift_password = input("Swift Password:")
return swift_user_name, swift_password
def authenticate_swift(swift_user_name, swift_password, selected_endpoint=""):
if selected_endpoint == "":
swift_endpoints = [
"dal05.objectstorage.softlayer.net",
"sng01.objectstorage.softlayer.net",
"ams01.objectstorage.softlayer.net",
]
selected_endpoint = prompt_for_choice(
swift_endpoints,
"Select Object Storage Endpoint:"
)
print("Authenticating...")
headers = {
"X-Storage-User": swift_user_name,
"X-Storage-Pass": swift_password
}
try:
response = object_storage_request(
selected_endpoint,
"/auth/v1.0/",
headers
)
except Exception:
swift_user_name, swift_password = get_swift_credentials()
return authenticate_swift(swift_user_name, swift_password)
print("Success!")
storage_url = response.getheader("X-Storage-Url")
auth_token = response.getheader("X-Auth-Token")
return storage_url, auth_token
def select_container(storage_url, auth_token):
url_tuple = urlparse(storage_url)
headers = {"X-Auth-Token": auth_token}
try:
response = object_storage_request(
url_tuple.netloc,
url_tuple.path,
headers
)
except Exception:
swift_user_name, swift_password = get_swift_credentials()
storage_url, auth_token = authenticate_swift(swift_user_name, swift_password)
return select_container(storage_url, auth_token)
containers = response.read().decode("utf-8").split("\n")
return prompt_for_choice(containers, "Select Container:")
def object_storage_request(server, path, headers, method="GET", data=""):
connection = http.client.HTTPConnection(server)
connection.request(method, path, data, headers)
response = connection.getresponse()
if 200 <= response.getcode() < 300:
return response
print("Error {}: {}".format(response.status, response.reason))
raise Exception(response.status, response.reason)
def upload_file(filename, swift_target_path, storage_url, auth_token):
url_tuple = urlparse(storage_url)
headers = {"X-Auth-Token": auth_token}
file_size = os.path.getsize(filename)
block_size = 1048576
chunk_size = 5 * block_size
chunks = math.ceil(file_size / chunk_size)
print("Reading in file")
file = open(filename, 'rb')
print("Uploading {} to \"{}\"".format(filename, swift_target_path))
for i in range(0, chunks):
data = file.read(chunk_size)
print("Uploading part {} of {}".format(i + 1, chunks))
chunk_name = "chunk-{0:0>5}".format(i)
object_storage_request(
url_tuple.netloc,
"{}/{}/{}".format(url_tuple.path, swift_target_path, chunk_name),
headers,
"PUT",
data
)
try:
print("Writing manifest file")
headers = {
"X-Auth-Token": auth_token,
"X-Object-Manifest": swift_target_path,
"Content-Length": 0,
}
object_storage_request(
url_tuple.netloc,
"{}/{}".format(url_tuple.path, swift_target_path),
headers,
"PUT"
)
except Exception:
return
print("File Uploaded!")
if __name__ == "__main__":
main()
#!/bin/bash
# ================================================================================
# ObjectStorageUploader.sh
# © Copyright IBM Corporation 2014.
# LICENSE: MIT (http://opensource.org/licenses/MIT)
# ================================================================================
#./objectstorageupload.sh dsl-4.4.10.iso 'myContainer/file.vhd' 'SLOS1234-1:SL1234' 'apikey'
fileToUpload=$1
swiftTargetPath=$2
swiftUsername=$3
swiftPassword=$4
swiftEndpoint='https://dal05.objectstorage.softlayer.net/auth/v1.0/'
apiResponse=$(curl -X GET -H "X-Storage-User: $swiftUsername" -H "X-Storage-Pass: $swiftPassword" -s -i $swiftEndpoint)
swiftAuthToken=$(echo "$apiResponse" | grep "X-Auth-Token:" | sed 's/X-Auth-Token: //g' | tr -d '\r')
swiftStorageUrl=$(echo "$apiResponse" | grep "X-Storage-Url:" | sed 's/X-Storage-Url: //g' | tr -d '\r')
fileSize=$(wc -c $fileToUpload | awk '{print $1}')
blockSize=1048576
let chunkSize=2048 #2GB chunks
let chunks=($fileSize/$blockSize+$chunkSize-1)/$chunkSize;
for ((i=0; i<chunks; i++))
do
printf -v chunkName "chunk-%05d" $i
let skipChunk=$i*chunkSize
dd if=$fileToUpload bs=$blockSize count=$chunkSize skip=$skipChunk | curl -X PUT -H "X-Auth-Token: $swiftAuthToken" --data-binary @- "$swiftStorageUrl/$swiftTargetPath/$chunkName"
done
curl -X PUT -H "X-Auth-Token: $swiftAuthToken" -H "X-Object-Manifest: $swiftTargetPath" -H "Content-Length: 0" $swiftStorageUrl/$swiftTargetPath
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment