Object Storage Uploader Script
# ================================================================================ | |
# ObjectStorageUploader.py | |
# © Copyright IBM Corporation 2014. | |
# LICENSE: MIT (http://opensource.org/licenses/MIT) | |
# ================================================================================ | |
import argparse | |
import os | |
import math | |
import http.client | |
from urllib.parse import urlparse | |
from urllib.parse import quote | |
def main(): | |
parser = argparse.ArgumentParser() | |
parser.add_argument("-f", "--file", help="file to upload to SWIFT") | |
parser.add_argument("-u", "--username", help="SWIFT username") | |
parser.add_argument("-p", "--password", help="SWIFT password (SoftLayer API key)") | |
parser.add_argument("-c", "--cluster", help="SWIFT cluster to use (default: dal05.objectstorage.softlayer.net)") | |
parser.add_argument("-t", "--target", help="location on SWIFT cluster to store file (container/filename.vhd)") | |
args = parser.parse_args() | |
if not args.file: | |
filename = select_file() | |
else: | |
filename = args.file | |
if not args.username: | |
swift_user_name, swift_password = get_swift_credentials() | |
else: | |
swift_user_name = args.username | |
swift_password = args.password | |
if not args.cluster: | |
storage_url, auth_token = authenticate_swift( | |
swift_user_name, | |
swift_password, | |
"dal05.objectstorage.softlayer.net" | |
) | |
else: | |
storage_url, auth_token = authenticate_swift( | |
swift_user_name, | |
swift_password, | |
args.cluster | |
) | |
if not args.target: | |
container = select_container(storage_url, auth_token) | |
swift_target_path = "{}/{}".format(container, quote(filename)) | |
else: | |
swift_target_path = args.target | |
upload_file(filename, swift_target_path, storage_url, auth_token) | |
def select_file(): | |
current_path = (os.path.dirname(os.path.realpath(__file__))) | |
print("Files in {}".format(current_path)) | |
files = get_file_list(current_path) | |
return prompt_for_choice(files, "Select file for upload:") | |
def get_file_list(path): | |
files = [] | |
for file in os.listdir(path): | |
if os.path.isfile(file): | |
files.append(file) | |
return files | |
def prompt_for_choice(list, prompt_label): | |
for i, item in enumerate(list): | |
if len(item): | |
print("{}) {}".format(i, item)) | |
selected_index = input(prompt_label) | |
if selected_index.isdigit(): | |
if 0 <= int(selected_index) < len(list): | |
print() | |
return list[int(selected_index)] | |
print("Invalid Input: {}".format(selected_index)) | |
return prompt_for_choice(list, prompt_label) | |
def get_swift_credentials(): | |
swift_user_name = input("Swift Username:") | |
swift_password = input("Swift Password:") | |
return swift_user_name, swift_password | |
def authenticate_swift(swift_user_name, swift_password, selected_endpoint=""): | |
if selected_endpoint == "": | |
swift_endpoints = [ | |
"dal05.objectstorage.softlayer.net", | |
"sng01.objectstorage.softlayer.net", | |
"ams01.objectstorage.softlayer.net", | |
] | |
selected_endpoint = prompt_for_choice( | |
swift_endpoints, | |
"Select Object Storage Endpoint:" | |
) | |
print("Authenticating...") | |
headers = { | |
"X-Storage-User": swift_user_name, | |
"X-Storage-Pass": swift_password | |
} | |
try: | |
response = object_storage_request( | |
selected_endpoint, | |
"/auth/v1.0/", | |
headers | |
) | |
except Exception: | |
swift_user_name, swift_password = get_swift_credentials() | |
return authenticate_swift(swift_user_name, swift_password) | |
print("Success!") | |
storage_url = response.getheader("X-Storage-Url") | |
auth_token = response.getheader("X-Auth-Token") | |
return storage_url, auth_token | |
def select_container(storage_url, auth_token): | |
url_tuple = urlparse(storage_url) | |
headers = {"X-Auth-Token": auth_token} | |
try: | |
response = object_storage_request( | |
url_tuple.netloc, | |
url_tuple.path, | |
headers | |
) | |
except Exception: | |
swift_user_name, swift_password = get_swift_credentials() | |
storage_url, auth_token = authenticate_swift(swift_user_name, swift_password) | |
return select_container(storage_url, auth_token) | |
containers = response.read().decode("utf-8").split("\n") | |
return prompt_for_choice(containers, "Select Container:") | |
def object_storage_request(server, path, headers, method="GET", data=""): | |
connection = http.client.HTTPConnection(server) | |
connection.request(method, path, data, headers) | |
response = connection.getresponse() | |
if 200 <= response.getcode() < 300: | |
return response | |
print("Error {}: {}".format(response.status, response.reason)) | |
raise Exception(response.status, response.reason) | |
def upload_file(filename, swift_target_path, storage_url, auth_token): | |
url_tuple = urlparse(storage_url) | |
headers = {"X-Auth-Token": auth_token} | |
file_size = os.path.getsize(filename) | |
block_size = 1048576 | |
chunk_size = 5 * block_size | |
chunks = math.ceil(file_size / chunk_size) | |
print("Reading in file") | |
file = open(filename, 'rb') | |
print("Uploading {} to \"{}\"".format(filename, swift_target_path)) | |
for i in range(0, chunks): | |
data = file.read(chunk_size) | |
print("Uploading part {} of {}".format(i + 1, chunks)) | |
chunk_name = "chunk-{0:0>5}".format(i) | |
object_storage_request( | |
url_tuple.netloc, | |
"{}/{}/{}".format(url_tuple.path, swift_target_path, chunk_name), | |
headers, | |
"PUT", | |
data | |
) | |
try: | |
print("Writing manifest file") | |
headers = { | |
"X-Auth-Token": auth_token, | |
"X-Object-Manifest": swift_target_path, | |
"Content-Length": 0, | |
} | |
object_storage_request( | |
url_tuple.netloc, | |
"{}/{}".format(url_tuple.path, swift_target_path), | |
headers, | |
"PUT" | |
) | |
except Exception: | |
return | |
print("File Uploaded!") | |
if __name__ == "__main__": | |
main() |
#!/bin/bash | |
# ================================================================================ | |
# ObjectStorageUploader.sh | |
# © Copyright IBM Corporation 2014. | |
# LICENSE: MIT (http://opensource.org/licenses/MIT) | |
# ================================================================================ | |
#./objectstorageupload.sh dsl-4.4.10.iso 'myContainer/file.vhd' 'SLOS1234-1:SL1234' 'apikey' | |
fileToUpload=$1 | |
swiftTargetPath=$2 | |
swiftUsername=$3 | |
swiftPassword=$4 | |
swiftEndpoint='https://dal05.objectstorage.softlayer.net/auth/v1.0/' | |
apiResponse=$(curl -X GET -H "X-Storage-User: $swiftUsername" -H "X-Storage-Pass: $swiftPassword" -s -i $swiftEndpoint) | |
swiftAuthToken=$(echo "$apiResponse" | grep "X-Auth-Token:" | sed 's/X-Auth-Token: //g' | tr -d '\r') | |
swiftStorageUrl=$(echo "$apiResponse" | grep "X-Storage-Url:" | sed 's/X-Storage-Url: //g' | tr -d '\r') | |
fileSize=$(wc -c $fileToUpload | awk '{print $1}') | |
blockSize=1048576 | |
let chunkSize=2048 #2GB chunks | |
let chunks=($fileSize/$blockSize+$chunkSize-1)/$chunkSize; | |
for ((i=0; i<chunks; i++)) | |
do | |
printf -v chunkName "chunk-%05d" $i | |
let skipChunk=$i*chunkSize | |
dd if=$fileToUpload bs=$blockSize count=$chunkSize skip=$skipChunk | curl -X PUT -H "X-Auth-Token: $swiftAuthToken" --data-binary @- "$swiftStorageUrl/$swiftTargetPath/$chunkName" | |
done | |
curl -X PUT -H "X-Auth-Token: $swiftAuthToken" -H "X-Object-Manifest: $swiftTargetPath" -H "Content-Length: 0" $swiftStorageUrl/$swiftTargetPath |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment