Last active
March 22, 2018 21:47
-
-
Save Sekenre/9de0654d314581d2fcbfe2b289c3a836 to your computer and use it in GitHub Desktop.
Profiling ways to concatenate chunked bytes
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": 1, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [ | |
"%load_ext memory_profiler" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"import random\n", | |
"import struct\n", | |
"def gen(size, count):\n", | |
" return ( struct.pack('B', random.randrange(255)) * size for b in range(count) )" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 3, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"[b'\\xb6\\xb6\\xb6\\xb6\\xb6', b'\\x9a\\x9a\\x9a\\x9a\\x9a', b'DDDDD', b'\\xc3\\xc3\\xc3\\xc3\\xc3', b'\\xd0\\xd0\\xd0\\xd0\\xd0']\n" | |
] | |
} | |
], | |
"source": [ | |
"print(list(gen(5,5)))" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 4, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [ | |
"def concat_join(items):\n", | |
" buf = []\n", | |
" for item in items:\n", | |
" buf.append(item)\n", | |
" return b''.join(buf)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 5, | |
"metadata": {}, | |
"outputs": [], | |
"source": [ | |
"def concat_bytes(items):\n", | |
" buf = bytearray()\n", | |
" for item in items:\n", | |
" buf.extend(item)\n", | |
" return buf" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 6, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"100000 loops, best of 10: 2.47 µs per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%%timeit -r 10 -n 100000 data = gen(10000,5000)\n", | |
"concat_bytes(data)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 7, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"100000 loops, best of 10: 890 ns per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%%timeit -r 10 -n 100000 data = gen(10000,5000)\n", | |
"concat_join(data)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 8, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"peak memory: 96.49 MiB, increment: 43.43 MiB\n" | |
] | |
} | |
], | |
"source": [ | |
"%%memit data = gen(10000,5000)\n", | |
"concat_bytes(data)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 9, | |
"metadata": {}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"peak memory: 52.93 MiB, increment: -2.66 MiB\n" | |
] | |
} | |
], | |
"source": [ | |
"%%memit data = gen(10000,5000)\n", | |
"concat_join(data)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 3", | |
"language": "python", | |
"name": "python3" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.6.1" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 2 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment