Object Storage Uploader Script
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# ================================================================================ | |
# ObjectStorageUploader.py | |
# © Copyright IBM Corporation 2014. | |
# LICENSE: MIT (http://opensource.org/licenses/MIT) | |
# ================================================================================ | |
import argparse | |
import os | |
import math | |
import http.client | |
from urllib.parse import urlparse | |
from urllib.parse import quote | |
def main(): | |
parser = argparse.ArgumentParser() | |
parser.add_argument("-f", "--file", help="file to upload to SWIFT") | |
parser.add_argument("-u", "--username", help="SWIFT username") | |
parser.add_argument("-p", "--password", help="SWIFT password (SoftLayer API key)") | |
parser.add_argument("-c", "--cluster", help="SWIFT cluster to use (default: dal05.objectstorage.softlayer.net)") | |
parser.add_argument("-t", "--target", help="location on SWIFT cluster to store file (container/filename.vhd)") | |
args = parser.parse_args() | |
if not args.file: | |
filename = select_file() | |
else: | |
filename = args.file | |
if not args.username: | |
swift_user_name, swift_password = get_swift_credentials() | |
else: | |
swift_user_name = args.username | |
swift_password = args.password | |
if not args.cluster: | |
storage_url, auth_token = authenticate_swift( | |
swift_user_name, | |
swift_password, | |
"dal05.objectstorage.softlayer.net" | |
) | |
else: | |
storage_url, auth_token = authenticate_swift( | |
swift_user_name, | |
swift_password, | |
args.cluster | |
) | |
if not args.target: | |
container = select_container(storage_url, auth_token) | |
swift_target_path = "{}/{}".format(container, quote(filename)) | |
else: | |
swift_target_path = args.target | |
upload_file(filename, swift_target_path, storage_url, auth_token) | |
def select_file(): | |
current_path = (os.path.dirname(os.path.realpath(__file__))) | |
print("Files in {}".format(current_path)) | |
files = get_file_list(current_path) | |
return prompt_for_choice(files, "Select file for upload:") | |
def get_file_list(path): | |
files = [] | |
for file in os.listdir(path): | |
if os.path.isfile(file): | |
files.append(file) | |
return files | |
def prompt_for_choice(list, prompt_label): | |
for i, item in enumerate(list): | |
if len(item): | |
print("{}) {}".format(i, item)) | |
selected_index = input(prompt_label) | |
if selected_index.isdigit(): | |
if 0 <= int(selected_index) < len(list): | |
print() | |
return list[int(selected_index)] | |
print("Invalid Input: {}".format(selected_index)) | |
return prompt_for_choice(list, prompt_label) | |
def get_swift_credentials(): | |
swift_user_name = input("Swift Username:") | |
swift_password = input("Swift Password:") | |
return swift_user_name, swift_password | |
def authenticate_swift(swift_user_name, swift_password, selected_endpoint=""): | |
if selected_endpoint == "": | |
swift_endpoints = [ | |
"dal05.objectstorage.softlayer.net", | |
"sng01.objectstorage.softlayer.net", | |
"ams01.objectstorage.softlayer.net", | |
] | |
selected_endpoint = prompt_for_choice( | |
swift_endpoints, | |
"Select Object Storage Endpoint:" | |
) | |
print("Authenticating...") | |
headers = { | |
"X-Storage-User": swift_user_name, | |
"X-Storage-Pass": swift_password | |
} | |
try: | |
response = object_storage_request( | |
selected_endpoint, | |
"/auth/v1.0/", | |
headers | |
) | |
except Exception: | |
swift_user_name, swift_password = get_swift_credentials() | |
return authenticate_swift(swift_user_name, swift_password) | |
print("Success!") | |
storage_url = response.getheader("X-Storage-Url") | |
auth_token = response.getheader("X-Auth-Token") | |
return storage_url, auth_token | |
def select_container(storage_url, auth_token): | |
url_tuple = urlparse(storage_url) | |
headers = {"X-Auth-Token": auth_token} | |
try: | |
response = object_storage_request( | |
url_tuple.netloc, | |
url_tuple.path, | |
headers | |
) | |
except Exception: | |
swift_user_name, swift_password = get_swift_credentials() | |
storage_url, auth_token = authenticate_swift(swift_user_name, swift_password) | |
return select_container(storage_url, auth_token) | |
containers = response.read().decode("utf-8").split("\n") | |
return prompt_for_choice(containers, "Select Container:") | |
def object_storage_request(server, path, headers, method="GET", data=""): | |
connection = http.client.HTTPConnection(server) | |
connection.request(method, path, data, headers) | |
response = connection.getresponse() | |
if 200 <= response.getcode() < 300: | |
return response | |
print("Error {}: {}".format(response.status, response.reason)) | |
raise Exception(response.status, response.reason) | |
def upload_file(filename, swift_target_path, storage_url, auth_token): | |
url_tuple = urlparse(storage_url) | |
headers = {"X-Auth-Token": auth_token} | |
file_size = os.path.getsize(filename) | |
block_size = 1048576 | |
chunk_size = 5 * block_size | |
chunks = math.ceil(file_size / chunk_size) | |
print("Reading in file") | |
file = open(filename, 'rb') | |
print("Uploading {} to \"{}\"".format(filename, swift_target_path)) | |
for i in range(0, chunks): | |
data = file.read(chunk_size) | |
print("Uploading part {} of {}".format(i + 1, chunks)) | |
chunk_name = "chunk-{0:0>5}".format(i) | |
object_storage_request( | |
url_tuple.netloc, | |
"{}/{}/{}".format(url_tuple.path, swift_target_path, chunk_name), | |
headers, | |
"PUT", | |
data | |
) | |
try: | |
print("Writing manifest file") | |
headers = { | |
"X-Auth-Token": auth_token, | |
"X-Object-Manifest": swift_target_path, | |
"Content-Length": 0, | |
} | |
object_storage_request( | |
url_tuple.netloc, | |
"{}/{}".format(url_tuple.path, swift_target_path), | |
headers, | |
"PUT" | |
) | |
except Exception: | |
return | |
print("File Uploaded!") | |
if __name__ == "__main__": | |
main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/bash | |
# ================================================================================ | |
# ObjectStorageUploader.sh | |
# © Copyright IBM Corporation 2014. | |
# LICENSE: MIT (http://opensource.org/licenses/MIT) | |
# ================================================================================ | |
#./objectstorageupload.sh dsl-4.4.10.iso 'myContainer/file.vhd' 'SLOS1234-1:SL1234' 'apikey' | |
fileToUpload=$1 | |
swiftTargetPath=$2 | |
swiftUsername=$3 | |
swiftPassword=$4 | |
swiftEndpoint='https://dal05.objectstorage.softlayer.net/auth/v1.0/' | |
apiResponse=$(curl -X GET -H "X-Storage-User: $swiftUsername" -H "X-Storage-Pass: $swiftPassword" -s -i $swiftEndpoint) | |
swiftAuthToken=$(echo "$apiResponse" | grep "X-Auth-Token:" | sed 's/X-Auth-Token: //g' | tr -d '\r') | |
swiftStorageUrl=$(echo "$apiResponse" | grep "X-Storage-Url:" | sed 's/X-Storage-Url: //g' | tr -d '\r') | |
fileSize=$(wc -c $fileToUpload | awk '{print $1}') | |
blockSize=1048576 | |
let chunkSize=2048 #2GB chunks | |
let chunks=($fileSize/$blockSize+$chunkSize-1)/$chunkSize; | |
for ((i=0; i<chunks; i++)) | |
do | |
printf -v chunkName "chunk-%05d" $i | |
let skipChunk=$i*chunkSize | |
dd if=$fileToUpload bs=$blockSize count=$chunkSize skip=$skipChunk | curl -X PUT -H "X-Auth-Token: $swiftAuthToken" --data-binary @- "$swiftStorageUrl/$swiftTargetPath/$chunkName" | |
done | |
curl -X PUT -H "X-Auth-Token: $swiftAuthToken" -H "X-Object-Manifest: $swiftTargetPath" -H "Content-Length: 0" $swiftStorageUrl/$swiftTargetPath |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment