bench_object_put.py

import sys
import time
import random

from eventlet import Timeout, GreenPool, queue, debug
from swift.common.bufferedhttp import http_connect
from swift.common.http import is_success
from swift.common.ring import Ring

debug.hub_exceptions(True)

CONTAINERS = 5
CLIENTS = 10

conn_timeout = 10
response_timeout = 30


def direct_put_container_object(node, part, account, container, name):
    path = '/%s/%s/%s' % (account, container, name)
    headers = {
        'x-timestamp': 0,
        'x-content-type': 'text/plain',
        'x-size': 0,
        'x-etag': 'x',
        'x-storage-policy-index': random.choice([0, 1]),
    }
    with Timeout(conn_timeout):
        conn = http_connect(node['ip'], node['port'], node['device'], part,
                            'PUT', path, headers=headers)

    with Timeout(response_timeout):
        resp = conn.getresponse()
        body = resp.read()

    conn.close()

    if is_success(resp.status):
        return True
    else:
        # print resp.status, body
        return False


def make_request(q, node, container):
    name = str(time.time() // 1000)
    try:
        resp = direct_put_container_object(node, 0, '.a', container, name)
    except Timeout as t:
        print 'TIMEOUT %ss!' % t.seconds
        resp = False
    q.put(resp)


def main():
    ring = Ring('/etc/swift/container.ring.gz')
    node = ring._devs[1]
    p = GreenPool(CLIENTS)
    q = queue.LightQueue()
    resps = []
    last_report = time.time()
    containers = ['container-%d' % i for i in range(CONTAINERS)]
    while True:
        container = random.choice(containers)
        p.spawn_n(make_request, q, node, container)
        while True:
            try:
                resps.append(q.get(block=False))
            except queue.Empty:
                break
            now = time.time()
            delta_t = now - last_report 
            if delta_t > 3:
                print '%f/s (%d failures)' % (
                    len(resps) / delta_t,
                    len([r for r in resps if not r]),
                )
                resps = []
                last_report = now


if __name__ == "__main__":
    sys.exit(main())

get_info_database.py

import sys
import time
import itertools
import os

from swift.container.backend import ContainerBroker


def main():
    try:
        db_path, num_gets = sys.argv[1:3]
    except (IndexError, ValueError):
        return 'ERROR: %s <db-file-name> <num-gets> [get_info]' % sys.argv[0]

    db_file = os.path.abspath(db_path)
    if not os.path.exists(db_file):
        return 'ERROR: %s does not exist!' % db_file

    broker = ContainerBroker(db_file, account='a', container='c')

    try:
        num_gets = int(num_gets)
    except ValueError:
        return 'ERROR: %s is not an integer!'

    if len(sys.argv) > 3:
        method_name = sys.argv[3]
    else:
        method_name = 'get_info'
    try:
        get = getattr(broker, method_name)
    except AttributeError:
        return 'ERROR: unknown method %s, try ' \
            '(get_info, get_replication_info)'

    start = time.time()
    last_report = start
    for reported in range(num_gets):
        get()
        now = time.time()
        if last_report + 1 < now:
            rate = reported / (now - start)
            print '%s (%s/s)' % (reported, rate)
            last_report = now
    now = time.time()
    rate = reported / (now - start)
    print '%s (%s/s)' % (reported, rate)


if __name__ == "__main__":
    sys.exit(main())

make_full_database.py

import sys
import time
import itertools
import os

from swift.container.backend import ContainerBroker
# from test.unit.container.test_backend import TestContainerBrokerBeforeIndex 
# ContainerBroker = TestContainerBrokerBeforeIndex.get_broker_class()

def main():
    try:
        db_path, num_items = sys.argv[1:3]
    except (IndexError, ValueError):
        return 'ERROR: %s <db-file-name> <num_items> [policy_index] [deleted]' % sys.argv[0]

    if len(sys.argv) > 3:
        policy_index = int(sys.argv[3])
    else:
        policy_index = 0

    if len(sys.argv) > 4:
        deleted = int(sys.argv[4])
    else:
        deleted = 0

    db_file = os.path.abspath(db_path)

    broker = ContainerBroker(db_file, account='a', container='c')
    start = time.time()
    ts = itertools.count(int(start))
    kwargs = {'storage_policy_index': policy_index}
    if not os.path.exists(db_file):
        try:
            broker.initialize(ts.next(), policy_index)
        except TypeError:
            broker.initialize(ts.next())
            kwargs = {}

    last_report = start
    for i in range(int(num_items)):
        obj_name = 'o%020d' % i
        broker.put_object(obj_name, ts.next(), 0, 'c', 'x', deleted=deleted,
                          **kwargs)
        if not i % 1000:
            now = time.time()
            if last_report + 1 < now:
                rate = i / (now - start)
                print '%s (%s/s)' % (obj_name, rate)
                last_report = now

    broker._commit_puts()


if __name__ == "__main__":
    sys.exit(main())

scan_full_database.py

import sys
import time
import itertools
import os

from swift.container.backend import ContainerBroker

def find_misplaced_objects(broker):
    start = time.time()
    count = sum(1 for x in broker.iter_misplaced_objects())
    print '%s (%s)' % (count, time.time() - start)


def main():
    try:
        db_path, num_items = sys.argv[1:3]
    except (IndexError, ValueError):
        return 'ERROR: %s <db-file-name> <num_items>|<misplaced> [marker]' % sys.argv[0]

    db_file = os.path.abspath(db_path)
    if not os.path.exists(db_file):
        return 'ERROR: %s does not exist!' % db_file

    broker = ContainerBroker(db_file, account='a', container='c')

    try:
        num_items = int(num_items)
    except ValueError:
        return find_misplaced_objects(broker)

    limit = min(num_items, 10000)

    if len(sys.argv) > 3:
        marker = sys.argv[3]
    else:
        marker = ''

    reported = 0
    start = time.time()
    last_report = start
    while True:
        items = broker.list_objects_iter(limit, marker, None, None, None)
        reported += len(items)
        if items and reported < num_items:
            marker = items[-1][0]
        else:
            break
        now = time.time()
        if last_report + 1 < now:
            rate = reported / (now - start)
            print '%r (%s/s)' % (marker, rate)
            last_report = now
    now = time.time()
    rate = reported / (now - start)
    print '%r (%s/s)' % (marker, rate)
    last_report = now


if __name__ == "__main__":
    sys.exit(main())