queue_vs_list.py

import json
import timeit
from copy import deepcopy
from queue import Queue


q = Queue()
l = list()
for x in range(100000):
    x = json.dumps({
        'asctime': '2019-10-10 15:03:36,862',
        'created': 1570737816.8622458,
        'exc_info': 'null',
        'filename': 'test.py',
        'funcName': 'main',
        'levelname': 'INFO',
        'levelno': 20,
        'lineno': 50,
        'message': f'message {x}',
        'module': 'test',
        'pathname': 'test.py',
        'process': 73016,
        'processName': 'MainProcess',
        'relativeCreated': 491.5478229522705,
        'thread': 4532229568,
        'threadName': 'MainThread'
	})

    q.put_nowait(deepcopy(x))
    l.append(deepcopy(x))

def qmain():
    r = ''
    count = 0
    while not q.empty():
        count += 1
        r += q.get(block=False)
        if count % 100 == 0:
            r = ''

def lmain():
    r = ''
    while len(l):
        r += ''.join(l[:100])
        del l[:100]
        r = ''


if __name__ == "__main__":
    print(f'iterating over queue: {timeit.timeit(qmain, number=1000)}')
    print(f'slicing list by 100:  {timeit.timeit(lmain, number=1000)}')