Created
February 17, 2017 14:14
-
-
Save aavanian/51a031d13bdf2b3a6d7d1abd8e67aae9 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": 1, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [ | |
"import pickle\n", | |
"import numpy as np\n", | |
"from levy_data import cdf" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"### Numpy Dump / Load to file (calls pickle)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"1 loop, best of 3: 611 ms per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%timeit cdf.dump('test')" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 3, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"The slowest run took 18.17 times longer than the fastest. This could mean that an intermediate result is being cached.\n", | |
"1 loop, best of 3: 32.9 ms per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%timeit test = np.load('test')" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"*Note: we should memoize the data everywhere it's loaded to improve performance*" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"### Numpy Dumps / Loads and write to file" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 4, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"1 loop, best of 3: 751 ms per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%%timeit\n", | |
"a = cdf.dumps()\n", | |
"with open('test', 'bw') as f:\n", | |
" f.write(a)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 5, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"The slowest run took 16.75 times longer than the fastest. This could mean that an intermediate result is being cached.\n", | |
"1 loop, best of 3: 34.6 ms per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%%timeit\n", | |
"with open('test', 'rb') as f:\n", | |
" test = np.loads(f.read())" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"### Direct to pickle file" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 6, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"1 loop, best of 3: 394 ms per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%%timeit\n", | |
"with open('test', 'bw') as f:\n", | |
" pickle.dump(cdf, f)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 7, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"The slowest run took 195.77 times longer than the fastest. This could mean that an intermediate result is being cached.\n", | |
"1 loop, best of 3: 5.38 ms per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%%timeit\n", | |
"with open('test', 'br') as f:\n", | |
" pickle.load(f)" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"### Numpy savez / load" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 8, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"1 loop, best of 3: 872 ms per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%timeit np.savez('test', cdf)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 9, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"The slowest run took 78.33 times longer than the fastest. This could mean that an intermediate result is being cached.\n", | |
"1 loop, best of 3: 4.77 ms per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%timeit test = np.load('test')" | |
] | |
}, | |
{ | |
"cell_type": "markdown", | |
"metadata": {}, | |
"source": [ | |
"### Numpy savez_compressed / load" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"1 loop, best of 3: 908 ms per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%timeit np.savez_compressed('test', cdf)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 4, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"100 loops, best of 3: 6.17 ms per loop\n" | |
] | |
} | |
], | |
"source": [ | |
"%timeit test = np.load('test')" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [] | |
} | |
], | |
"metadata": { | |
"anaconda-cloud": {}, | |
"kernelspec": { | |
"display_name": "Python [conda env:vol_tools3.5]", | |
"language": "python", | |
"name": "conda-env-vol_tools3.5-py" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 3 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython3", | |
"version": "3.5.2" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 1 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment