exercise_swift

exercise_swift.py

#!/opt/ss/bin/python

import swiftclient
import logging
import signal
import time
from optparse import OptionParser
import json

METHODS = [
    'head_account',
    'get_account',
    'post_account',
    'head_container',
    'get_container',
    'put_container',
    'post_container',
    'delete_container',
    'head_object',
    'get_object',
    'put_object',
    'post_object',
    'delete_object',
]



class SwiftExcerciserClient(object):

    def __init__(self, *args, **kwargs):
        self.client = swiftclient.Connection(*args, **kwargs)
        self.results = {}
        for method in METHODS:
            self.results[method] = { 'ok': 0, 'error': 0}

    def do_method(self, method, *args, **kwargs):
        func = getattr(self.client, method)
        try:
            output = func(*args, **kwargs)
            self.results[method]['ok'] += 1
            return output
        except swiftclient.ClientException:
            self.results[method]['error'] += 1
            if method in ['get_account', 'get_container']:
                return [], []  # empty header and thing lists


class SourceFile(object):
    """Stolen from swift/common/bench.py:
Iterable, file-like object to lazily emit a bunch of zeros in
reasonable-size chunks.

swift.common.direct_client wants iterables, but swiftclient wants
file-like objects where hasattr(thing, 'read') is true. Therefore,
this class can do both.
"""

    def __init__(self, size, chunk_size=1024 * 64):
        self.pos = 0
        self.size = size
        self.chunk_size = chunk_size

    def __iter__(self):
        return self

    def __len__(self):
        return self.size

    def next(self):
        if self.pos >= self.size:
            raise StopIteration
        chunk_size = min(self.size - self.pos, self.chunk_size)
        yield '0' * chunk_size
        self.pos += chunk_size

    def read(self, desired_size):
        chunk_size = min(self.size - self.pos, desired_size)
        self.pos += chunk_size
        return '0' * chunk_size


class SwiftExerciser(object):

    def __init__(self, logger, options):
        self.logger = logger
        self.options = options
        self.client = SwiftExcerciserClient(
            authurl=self.options.auth,
            user=self.options.user,
            key=self.options.key,
            retries=2,
            insecure=True)
        # self.accounts = []

    # don't forget to figure out how to accept a CTRL-C and terminate gracefully

    # def create_account(self):
    #     account_name = time.time()
    #     self.accounts.append(account_name)
    #     self.a += 1
    #
    #     self.c = 0
    #     while self.c < self.options.num_containers:
    #         self.create_container(account_name)

    def create_container(self):
        self.client.do_method('put_container', self.c)
        headers, containers = self.client.do_method('get_account')
        print json.dumps(containers, sort_keys = False, indent = 4)
        for container in containers:
            # post an update to all of the containers
            pass
        self.c += 1

        self.f = 0
        while self.f < self.options.num_objects:
            self.create_file()

    def create_file(self):
        file = SourceFile(self.options.object_size)
        self.client.do_method('put_object', self.c, self.f, file)
        headers, objects = self.client.do_method('get_container', self.c)
        print json.dumps(objects, sort_keys = False, indent = 4)
        for obj in objects:
            self.client.do_method('get_object', self.c, obj['name'])
            # post to file

        self.f += 1

        # if self.options.num_workers > 1 and self.f == 0:
        #     # only spawn a worker in the first loop
        #     self.spawn_worker()

    # def delete_account(self):
    #     while self.c:
    #         self.delete_container()
    #     # find the account in the list and delete it
    #     self.a -= 1

    def delete_container(self):
        while self.f:
            self.delete_file()
        headers, containers = self.client.do_method('get_account')
        print json.dumps(containers, sort_keys = False, indent = 4)
        for container in containers:
            # post update to containers
            pass

        self.client.do_method('delete_container', self.c)
        self.c -= 1

    def delete_file(self):
        headers, objects = self.client.do_method('get_container', self.c)
        print json.dumps(objects, sort_keys = False, indent = 4)
        for obj in objects:
            self.client.do_method('get_object', self.c, obj['name'])
            # post update to file
        self.client.do_method('delete_object', self.c, self.f)
        self.f -= 1

    # def spawn_worker(self):
    #     # the real work
    #     pass

    def run(self):

        # self.a = 0
        # while self.a < self.options.num_accounts:
        #     try:
        #         self.create_account()
        #         self.is_superuser = True
        #     except ClientException:
        #         self.is_superuser = False
        self.c = 0
        while self.c < self.options.num_containers:
            self.create_container()

        # while self.a:
        #     if self.is_superuser:
        #         self.delete_account()
        #     else:
        #         while self.c:
        #             self.delete_container()


        # return:
        results = self.client.results
        for method in METHODS:
            ok = results[method]['ok']
            total = ok + results[method]['error']
            if total == 0:
                results.pop(method)
            else:
                results[method]['total'] = total
                if ok and total:
                    results[method]['percent'] = (ok / total) * 100
                else:
                    results[method]['percent'] = 0


        return results

if __name__ == '__main__':
    # command line options
    usage = """usage: %prog [OPTIONS]

Total number of operations will be
--num-workers * --num-accounts * --num-containers * --num-objects * ???
"""

    parser = OptionParser(usage=usage)
    parser.add_option('-A', '--auth', dest='auth',
                      help='URL for obtaining an auth token')
    parser.add_option('-U', '--user', dest='user',
                      help='User name for obtaining an auth token')
    parser.add_option('-K', '--key', dest='key',
                      help='Key for obtaining an auth token')
    parser.add_option('-s', '--object-size', dest='object_size', default=2000,
                      help='Size of objects to PUT (in bytes). Default 2000 Bytes.')
    # parser.add_option('-a', '--num-accounts', dest='num_accounts', default=1,
    #                   help='Number of accounts to distribute containers among. Default 1.'
    #                        'Will only work properly if the user provided is a superuser.')
    parser.add_option('-o', '--num-objects', dest='num_objects', default=10,
                      help='Number of objects to PUT. Default: 10.')
    parser.add_option('-c', '--num-containers', dest='num_containers', default=5,
                      help='Number of containers to distribute objects among. Default 5.')
    # parser.add_option('-w', '-num-workers', dest='num_workers', default=1,
    #                   help='Number of semi-concurrent process to run. Default 1.')
    parser.add_option('-x', '--no-delete', dest='delete', action='store_false',
                      help='If set, will not delete the objects created. Default: False.')
    parser.add_option('-L', '--log-level', dest='log_level', default="INFO",
                      help='Log level. Default: INFO.')

    options, args = parser.parse_args()

    # sigterm
    def sigterm(signum, frame):
        sys.exit('Termination signal received.')
    signal.signal(signal.SIGTERM, sigterm)

    # logging
    logger = logging.getLogger(__name__)
    logger.propagate = False
    logger.setLevel({
        'debug': logging.DEBUG,
        'info': logging.INFO,
        'warning': logging.WARNING,
        'error': logging.ERROR,
        'critical': logging.CRITICAL}.get(
            options.log_level.lower(), logging.INFO))
    loghandler = logging.StreamHandler()
    logger.addHandler(loghandler)
    # logger = LogAdapter(logger, 'exercise-swift')
    logformat = logging.Formatter('%(server)s %(asctime)s %(levelname)s '
                                  '%(message)s')
    loghandler.setFormatter(logformat)

    # The run
    controller = SwiftExerciser(logger, options)
    output = controller.run()
    print json.dumps(output, sort_keys = False, indent = 4)