Driver script for another pass at Swift ring balance optimization.

profile_ring_rebalance.py

import time
import cProfile
from itertools import product, imap
from contextlib import contextmanager

from swift.common import ring

PROFILE = True

@contextmanager
def profiling(filename):
    if PROFILE:
        print 'Profiling to %r' % filename
        prof = cProfile.Profile()
        prof.enable()
    else:
        print 'NOT Profiling to %r' % filename
    try:
        start = time.time()
        yield()
    finally:
        if PROFILE:
            prof.disable()
            prof.dump_stats(filename)
        print '  wall-time delta: %.2fs' % (time.time() - start,)


ip_3 = xrange(1, 254)
ip_4 = xrange(1, 254)
ip_iter = imap('.'.join, imap(str, product([10], [1], ip_3, ip_4)))
dev_id_iter = iter(xrange(10000))

part_power = 18
dev_count = 600
dev_count2 = 300
del_count = 100

print "Using part-power = %d, adding %d devices, removing %d, then adding %d more..." % (
    part_power, dev_count, del_count, dev_count2)

rb = ring.RingBuilder(part_power, 3, 1)
for zone in [0, 1, 2, 3, 4]:
    for _ in xrange(dev_count / 5):
        rb.add_dev({'id': dev_id_iter.next(), 'zone': zone, 'weight': 1, 'ip':
                    ip_iter.next(), 'port': 10000, 'device': 'sda1'})

# Now rebalance (will be an initial balancing)
with profiling('initial_balance.prof'):
    rb.rebalance()

# Remove the last del_count of dev_count...
for i in xrange(dev_count - del_count, dev_count):
    rb.remove_dev(i)
rb.pretend_min_part_hours_passed()

with profiling('deleting_200_rebalance.prof'):
    rb.rebalance()

# Now add another 1k devices
for zone in [0, 1, 2, 3, 4]:
    for _ in xrange(dev_count2 / 5):
        rb.add_dev({'id': dev_id_iter.next(), 'zone': zone, 'weight': 1, 'ip':
                    ip_iter.next(), 'port': 10000, 'device': 'sda1'})
rb.pretend_min_part_hours_passed()

# Now rebalance again
with profiling('first_rebalance.prof'):
    rb.rebalance()