gistfile1.txt

$ cat read_list_1.csv
0-104857599

$ cat read_list_2.csv
0-52428799
52428800-104857599

$ cat read_list_10.csv
0-10485759
10485760-20971519
20971520-31457279
31457280-41943039
41943040-52428799
52428800-62914559
62914560-73400319
73400320-83886079
83886080-94371839
94371840-104857599

gistfile2.txt

$ python range-read-object.py
0-104857599
{u'content-length': u'104857600', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 0-104857599/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'tx24837f8789844924b69e7-005b4ce038', u'date': u'Mon, 16 Jul 2018 18:13:12 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'tx24837f8789844924b69e7-005b4ce038'}
Exiting Main Thread, Time Cost: 32.0516409874

$ cp read_list_2.csv read_list.csv
$ python range-read-object.py
52428800-1048575990-52428799

{u'content-length': u'52428800', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 52428800-104857599/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'tx6c785c7ae7a74aa5a44dd-005b4ce069', u'date': u'Mon, 16 Jul 2018 18:14:01 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'tx6c785c7ae7a74aa5a44dd-005b4ce069'}
{u'content-length': u'52428800', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 0-52428799/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'tx3e15598bc60040f3839ad-005b4ce069', u'date': u'Mon, 16 Jul 2018 18:14:01 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'tx3e15598bc60040f3839ad-005b4ce069'}
Exiting Main Thread, Time Cost: 29.2925350666

$ cp read_list_10.csv read_list.csv
$ python range-read-object.py
20971520-31457279
10485760-20971519
31457280-41943039
0-10485759
 94371840-10485759952428800-62914559

41943040-52428799
 62914560-73400319
73400320-83886079
83886080-94371839
{u'content-length': u'10485760', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 73400320-83886079/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'txb90c7b05c05d4c35968ad-005b4ce096', u'date': u'Mon, 16 Jul 2018 18:14:46 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'txb90c7b05c05d4c35968ad-005b4ce096'}
{u'content-length': u'10485760', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 52428800-62914559/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'tx3f50c746f3dd4e24ac188-005b4ce096', u'date': u'Mon, 16 Jul 2018 18:14:46 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'tx3f50c746f3dd4e24ac188-005b4ce096'}
{u'content-length': u'10485760', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 20971520-31457279/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'txa4be225d86504007aea2f-005b4ce096', u'date': u'Mon, 16 Jul 2018 18:14:46 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'txa4be225d86504007aea2f-005b4ce096'}
{u'content-length': u'10485760', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 10485760-20971519/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'tx3d6251b6fddf49d08c5dd-005b4ce096', u'date': u'Mon, 16 Jul 2018 18:14:46 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'tx3d6251b6fddf49d08c5dd-005b4ce096'}
{u'content-length': u'10485760', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 31457280-41943039/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'tx82e274c9e22b4db995e74-005b4ce096', u'date': u'Mon, 16 Jul 2018 18:14:46 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'tx82e274c9e22b4db995e74-005b4ce096'}
{u'content-length': u'10485760', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 94371840-104857599/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'tx4e593db9c74c4ab59b2b5-005b4ce096', u'date': u'Mon, 16 Jul 2018 18:14:46 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'tx4e593db9c74c4ab59b2b5-005b4ce096'}
{u'content-length': u'10485760', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 62914560-73400319/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'txa78723e4dd164ae6a313e-005b4ce096', u'date': u'Mon, 16 Jul 2018 18:14:46 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'txa78723e4dd164ae6a313e-005b4ce096'}
{u'content-length': u'10485760', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 0-10485759/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'tx23eb49a951e7402ba4758-005b4ce096', u'date': u'Mon, 16 Jul 2018 18:14:46 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'tx23eb49a951e7402ba4758-005b4ce096'}
{u'content-length': u'10485760', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 83886080-94371839/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'tx4d2b0538927840fe9e53f-005b4ce096', u'date': u'Mon, 16 Jul 2018 18:14:46 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'tx4d2b0538927840fe9e53f-005b4ce096'}
{u'content-length': u'10485760', u'x-object-meta-mtime': u'1527050074.452', u'accept-ranges': u'bytes', u'last-modified': u'Mon, 16 Jul 2018 16:30:52 GMT', u'content-range': u'bytes 41943040-52428799/104857600', u'etag': u'978fe39760e4d83e05c41579c35c3596', u'x-timestamp': u'1531758651.12702', u'x-trans-id': u'txdc3d0a30fc8847508f0e7-005b4ce096', u'date': u'Mon, 16 Jul 2018 18:14:46 GMT', u'content-type': u'application/octet-stream', u'x-openstack-request-id': u'txdc3d0a30fc8847508f0e7-005b4ce096'}
Exiting Main Thread, Time Cost: 26.2570338249

range-read-object.conf

[global]
# admin_user, password, auth url and prefix
admin_user=test
admin_pass=xxxx
source_auth_url=https://xxx.swiftstack.com/auth/v2.0
source_swift_base=https://xxx.swiftstack.com/v1/

# number of theard with account, e.g 10 accounts then 10 threads
n_read_threads=10

# object read ist e.g 100MB
read_list=read_list.csv

range-read-object.py

#!/usr/bin/python

import swiftclient
import pprint
import Queue
import threading
import time
import timeit
import ConfigParser
import sys

# global variant from config file
admin_user = ''
admin_pass = ''
source_auth_url = ''
source_swift_base = ''
n_read_threads = 1

# local variant use for code only
pp = pprint.PrettyPrinter(indent=4)
exitFlag = 0


def get_Config():
    global admin_user
    global admin_pass
    global source_auth_url
    global n_read_threads
    global read_list
    global source_swift_base

    config = ConfigParser.ConfigParser()
    config.read(r'range-read-object.conf')
    admin_user = config.get('global', 'admin_user')
    admin_pass = config.get('global', 'admin_pass')
    source_auth_url = config.get('global', 'source_auth_url')
    n_read_threads = config.getint('global', 'n_read_threads')
    read_list = config.get('global', 'read_list')
    source_swift_base = config.get('global', 'source_swift_base')

class readThread(threading.Thread):
    def __init__(self, threadID, name, q, l,):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.q = q
        self.L = l

    def run(self):
        process_read_thread(self.name, self.q, self.L)


def process_read_thread(threadName, q, l):
    while not exitFlag:
        l.acquire()
        if not q.empty():
            data = q.get()
            l.release()
            process_read(data)
        else:
            l.release()
        time.sleep(1)


def make_read_threadlist(int_obj_threads):
    threadList = []
    for i in range(1, int_obj_threads + 1):
        threadList.append('read Thread-' + str(i))

    return threadList


def process_read(data):
    global admin_user
    global admin_pass
    global source_auth_url
    global read_list
    global source_swift_base

    try:
        swift_s = swiftclient.client.Connection(
            authurl=source_auth_url,
            user=admin_user,
            key=admin_pass,
            tenant_name=admin_user,
            auth_version='2.0',
            os_options={'object_storage_url': source_swift_base+"AUTH_test"})
        resp_headers, containers = swift_s.get_account()
        for c in containers:
            print data[1]
            resp_headers, object_content = get_object_range_response(swift_s, "test",
                                                               "100MB", data[1])
            print resp_headers
        swift_s.close()
    except Exception, err:
        print data[1] + ": " + str(err)


def get_object_range_response(swift_x, container_name, object_name,
                              range_bounary):
    try:
        resp_headers, object_content = swift_x.get_object(container_name,
                                                       object_name,
                                                          headers={"range":"bytes="+range_bounary})
    except Exception:
        sys.exc_clear()
    return resp_headers, object_content


def get_read_list(read_list):
    with open(read_list) as f:
        lines = f.readlines()
    read_list = [x.strip() for x in lines]
    return read_list


def main():
    global n_read_threads
    global admin_user
    global admin_pass
    global source_auth_url
    global destination_auth_url
    global read_list

    start = timeit.default_timer()

    # Get Config
    get_Config()

    # Get object list
    read_list = get_read_list(read_list)

    # make a read thread list
    readThreadList = make_read_threadlist(n_read_threads)

    # Generate read dictionary by read list
    namedict = {}
    namecount = 1
    for obj in read_list:
        namedict[namecount] = obj
        namecount += 1

    # Queue # = thread list
    queueLock = threading.Lock()
    workQueue = Queue.Queue(0)  # queue size is infinite
    threads = []
    threadID = 1

    # Create new threads
    for tName in readThreadList:
        thread = readThread(threadID, tName, workQueue, queueLock)
        thread.start()
        threads.append(thread)
        threadID += 1

    # Fill the queue
    queueLock.acquire()

    # for word in nameList:
    for word in namedict.iteritems():
        workQueue.put(word)
    queueLock.release()

    # Wait for queue to empty
    while not workQueue.empty():
        pass

    # Notify threads it's time to exit
    global exitFlag
    exitFlag = 1

    # Wait for all threads to complete
    for t in threads:
        t.join()

    print 'Exiting Main Thread, Time Cost: %s' \
        % (timeit.default_timer() - start)


if __name__ == '__main__':
    try:
        main()
    except (KeyboardInterrupt):
        print "Abort, Got Keyboard Interrupt !"
        sys.exit(1)

• Preparation: update credential in range-read-object.py
• Run
  • $ python range-read-object.py
• Quick Test Result
  • 1 vs 2 vs 10 threads against with 100MB file
  • 32 vs 29 vs 26