Object Storage Uploader Script

ObjectStorageUploader.py

# ================================================================================
#     ObjectStorageUploader.py
#     © Copyright IBM Corporation 2014.
#     LICENSE: MIT (http://opensource.org/licenses/MIT)    
# ================================================================================

import argparse
import os
import math
import http.client
from urllib.parse import urlparse
from urllib.parse import quote


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-f", "--file", help="file to upload to SWIFT")
    parser.add_argument("-u", "--username", help="SWIFT username")
    parser.add_argument("-p", "--password", help="SWIFT password (SoftLayer API key)")
    parser.add_argument("-c", "--cluster", help="SWIFT cluster to use (default: dal05.objectstorage.softlayer.net)")
    parser.add_argument("-t", "--target", help="location on SWIFT cluster to store file (container/filename.vhd)")
    args = parser.parse_args()

    if not args.file:
        filename = select_file()
    else:
        filename = args.file

    if not args.username:
        swift_user_name, swift_password = get_swift_credentials()
    else:
        swift_user_name = args.username
        swift_password = args.password

    if not args.cluster:
        storage_url, auth_token = authenticate_swift(
            swift_user_name,
            swift_password,
            "dal05.objectstorage.softlayer.net"
        )
    else:
        storage_url, auth_token = authenticate_swift(
            swift_user_name,
            swift_password,
            args.cluster
        )

    if not args.target:
        container = select_container(storage_url, auth_token)
        swift_target_path = "{}/{}".format(container, quote(filename))
    else:
        swift_target_path = args.target

    upload_file(filename, swift_target_path, storage_url, auth_token)


def select_file():
    current_path = (os.path.dirname(os.path.realpath(__file__)))
    print("Files in {}".format(current_path))
    files = get_file_list(current_path)
    return prompt_for_choice(files, "Select file for upload:")


def get_file_list(path):
    files = []
    for file in os.listdir(path):
        if os.path.isfile(file):
            files.append(file)
    return files


def prompt_for_choice(list, prompt_label):
    for i, item in enumerate(list):
        if len(item):
            print("{}) {}".format(i, item))
    selected_index = input(prompt_label)
    if selected_index.isdigit():
        if 0 <= int(selected_index) < len(list):
            print()
            return list[int(selected_index)]

    print("Invalid Input: {}".format(selected_index))
    return prompt_for_choice(list, prompt_label)


def get_swift_credentials():
    swift_user_name = input("Swift Username:")
    swift_password = input("Swift Password:")
    return swift_user_name, swift_password


def authenticate_swift(swift_user_name, swift_password, selected_endpoint=""):
    if selected_endpoint == "":
        swift_endpoints = [
            "dal05.objectstorage.softlayer.net",
            "sng01.objectstorage.softlayer.net",
            "ams01.objectstorage.softlayer.net",
        ]
        selected_endpoint = prompt_for_choice(
            swift_endpoints,
            "Select Object Storage Endpoint:"
        )

    print("Authenticating...")
    headers = {
        "X-Storage-User": swift_user_name,
        "X-Storage-Pass": swift_password
    }
    try:
        response = object_storage_request(
            selected_endpoint,
            "/auth/v1.0/",
            headers
        )
    except Exception:
        swift_user_name, swift_password = get_swift_credentials()
        return authenticate_swift(swift_user_name, swift_password)
    print("Success!")

    storage_url = response.getheader("X-Storage-Url")
    auth_token = response.getheader("X-Auth-Token")

    return storage_url, auth_token


def select_container(storage_url, auth_token):
    url_tuple = urlparse(storage_url)

    headers = {"X-Auth-Token": auth_token}
    try:
        response = object_storage_request(
            url_tuple.netloc,
            url_tuple.path,
            headers
        )
    except Exception:
        swift_user_name, swift_password = get_swift_credentials()
        storage_url, auth_token = authenticate_swift(swift_user_name, swift_password)
        return select_container(storage_url, auth_token)

    containers = response.read().decode("utf-8").split("\n")
    return prompt_for_choice(containers, "Select Container:")


def object_storage_request(server, path, headers, method="GET", data=""):
    connection = http.client.HTTPConnection(server)
    connection.request(method, path, data, headers)
    response = connection.getresponse()

    if 200 <= response.getcode() < 300:
        return response

    print("Error {}: {}".format(response.status, response.reason))
    raise Exception(response.status, response.reason)


def upload_file(filename, swift_target_path, storage_url, auth_token):
    url_tuple = urlparse(storage_url)
    headers = {"X-Auth-Token": auth_token}

    file_size = os.path.getsize(filename)
    block_size = 1048576
    chunk_size = 5 * block_size
    chunks = math.ceil(file_size / chunk_size)

    print("Reading in file")
    file = open(filename, 'rb')
    print("Uploading {} to \"{}\"".format(filename, swift_target_path))
    for i in range(0, chunks):
        data = file.read(chunk_size)
        print("Uploading part {} of {}".format(i + 1, chunks))
        chunk_name = "chunk-{0:0>5}".format(i)
        object_storage_request(
            url_tuple.netloc,
            "{}/{}/{}".format(url_tuple.path, swift_target_path, chunk_name),
            headers,
            "PUT",
            data
        )

    try:
        print("Writing manifest file")
        headers = {
            "X-Auth-Token": auth_token,
            "X-Object-Manifest": swift_target_path,
            "Content-Length": 0,
        }
        object_storage_request(
            url_tuple.netloc,
            "{}/{}".format(url_tuple.path, swift_target_path),
            headers,
            "PUT"
        )
    except Exception:
        return

    print("File Uploaded!")

if __name__ == "__main__":
    main()

ObjectStorageUploader.sh

#!/bin/bash
# ================================================================================
#     ObjectStorageUploader.sh
#     © Copyright IBM Corporation 2014.
#     LICENSE: MIT (http://opensource.org/licenses/MIT)    
# ================================================================================

#./objectstorageupload.sh dsl-4.4.10.iso 'myContainer/file.vhd' 'SLOS1234-1:SL1234' 'apikey'

fileToUpload=$1
swiftTargetPath=$2
swiftUsername=$3
swiftPassword=$4

swiftEndpoint='https://dal05.objectstorage.softlayer.net/auth/v1.0/'

apiResponse=$(curl -X GET -H "X-Storage-User: $swiftUsername" -H "X-Storage-Pass: $swiftPassword" -s -i $swiftEndpoint)
swiftAuthToken=$(echo "$apiResponse" | grep "X-Auth-Token:" | sed 's/X-Auth-Token: //g' | tr -d '\r')
swiftStorageUrl=$(echo "$apiResponse" | grep "X-Storage-Url:" | sed 's/X-Storage-Url: //g' | tr -d '\r')

fileSize=$(wc -c $fileToUpload | awk '{print $1}')
blockSize=1048576
let chunkSize=2048 #2GB chunks
let chunks=($fileSize/$blockSize+$chunkSize-1)/$chunkSize;

for ((i=0; i<chunks; i++))
do
   printf -v chunkName "chunk-%05d" $i
   let skipChunk=$i*chunkSize

   dd if=$fileToUpload bs=$blockSize count=$chunkSize skip=$skipChunk | curl -X PUT -H "X-Auth-Token: $swiftAuthToken" --data-binary @- "$swiftStorageUrl/$swiftTargetPath/$chunkName"
done

curl -X PUT -H "X-Auth-Token: $swiftAuthToken" -H "X-Object-Manifest: $swiftTargetPath" -H "Content-Length: 0" $swiftStorageUrl/$swiftTargetPath