time_yield_objects

gistfile1.txt

(swift-3.8.15) (acoles) ~/0dev/openstack/swift{p-misplaced-faster} % git show -q|cat
commit eef67f5319da794b7786ed0613cdbc77889551af
(swift-3.8.15) (acoles) ~/0dev/openstack/swift{p-misplaced-faster} % PYTHONPATH=. python ./time_yield_objects.py 1000000 10
Unable to read test config /etc/swift/test.conf - file not found
yielded 1000000 rows in 1.718s
yielded 1000000 rows in 1.350s
yielded 1000000 rows in 1.317s
yielded 1000000 rows in 1.356s
yielded 1000000 rows in 1.317s
yielded 1000000 rows in 1.535s
yielded 1000000 rows in 1.319s
yielded 1000000 rows in 1.373s
yielded 1000000 rows in 1.321s
yielded 1000000 rows in 1.365s
average rows/sec = 7327785s

(swift-3.8.15) (acoles) ~/0dev/openstack/swift{master,!} % git show -q|cat
commit 5de745c2bc3aa82049424b9c3e8744cda53a4c74
(swift-3.8.15) (acoles) ~/0dev/openstack/swift{master,+} % PYTHONPATH=. python ./time_yield_objects.py 1000000 10
Unable to read test config /etc/swift/test.conf - file not found
yielded 1000000 rows in 2.488s
yielded 1000000 rows in 2.230s
yielded 1000000 rows in 2.203s
yielded 1000000 rows in 2.423s
yielded 1000000 rows in 2.536s
yielded 1000000 rows in 2.182s
yielded 1000000 rows in 2.129s
yielded 1000000 rows in 2.254s
yielded 1000000 rows in 2.403s
yielded 1000000 rows in 2.388s
average rows/sec = 4186907s

time_yield_objects.py

import logging
import random
import sys
import time

from swift.common.utils import Timestamp, ShardRange
from test.unit.container.test_sharder import TestSharder


class Test(TestSharder):
    def time_yield_objects(self, num_objs):
        self.setUp()
        broker = self._make_broker()
        objects = [
            ('o%02d' % i, self.ts_encoded(), 10, 'text/plain', 'etag_a',
             round(random.random()), 0) for i in range(num_objs)]
        for obj in objects:
            broker.put_object(*obj)

        src_range = ShardRange('dont/care', Timestamp.now())
        with self._mock_sharder(conf={}) as sharder:
            sharder.logger = logging.getLogger()
            start = time.time()
            batches = [b for b, _ in
                       sharder.yield_objects(broker, src_range)]
        return time.time() - start


if __name__ == '__main__':
    num_objs = int(sys.argv[1])
    elapsed = Test().time_yield_objects(num_objs)
    print('yielded %d rows in %.3fs' % (num_objs, elapsed))

sql diff applied to proposed change

(swift-3.8.15) (acoles) ~/0dev/openstack/swift{p-misplaced-faster} % PYTHONPATH=. python ./time_yield_objects.py 1000000 10
Unable to read test config /etc/swift/test.conf - file not found
yielded 1000000 rows in 1.401s
yielded 1000000 rows in 1.330s
yielded 1000000 rows in 1.400s
yielded 1000000 rows in 1.679s
yielded 1000000 rows in 1.311s
yielded 1000000 rows in 1.707s
yielded 1000000 rows in 1.966s
yielded 1000000 rows in 1.791s
yielded 1000000 rows in 1.576s
yielded 1000000 rows in 1.822s
average rows/sec = 5488679s