parallel (2).ipynb

{"metadata": {"language_info": {"file_extension": ".py", "pygments_lexer": "ipython3", "codemirror_mode": {"name": "ipython", "version": 3}, "name": "python", "version": "3.4.3", "nbconvert_exporter": "python", "mimetype": "text/x-python"}, "kernelspec": {"display_name": "Python 3", "name": "python3", "language": "python"}}, "cells": [{"metadata": {"collapsed": true, "trusted": true}, "source": "from IPython.parallel import Client", "outputs": [], "execution_count": 1, "cell_type": "code"}, {"metadata": {"collapsed": true, "trusted": true}, "source": "c = Client()", "outputs": [], "execution_count": 2, "cell_type": "code"}, {"metadata": {"collapsed": false, "trusted": true}, "source": "len(c.ids)", "outputs": [{"metadata": {}, "data": {"text/plain": "128"}, "execution_count": 3, "output_type": "execute_result"}], "execution_count": 3, "cell_type": "code"}, {"metadata": {}, "source": "Print all 128 nodes hostnames", "cell_type": "markdown"}, {"metadata": {"scrolled": true, "collapsed": false, "trusted": true}, "source": "%px import socket; print(socket.gethostname())", "outputs": [{"name": "stdout", "text": "[stdout:0] gcn-4-51.sdsc.edu\n[stdout:1] gcn-4-51.sdsc.edu\n[stdout:2] gcn-4-51.sdsc.edu\n[stdout:3] gcn-4-57.sdsc.edu\n[stdout:4] gcn-4-57.sdsc.edu\n[stdout:5] gcn-4-57.sdsc.edu\n[stdout:6] gcn-4-57.sdsc.edu\n[stdout:7] gcn-4-65.sdsc.edu\n[stdout:8] gcn-4-65.sdsc.edu\n[stdout:9] gcn-4-65.sdsc.edu\n[stdout:10] gcn-4-55.sdsc.edu\n[stdout:11] gcn-4-55.sdsc.edu\n[stdout:12] gcn-4-55.sdsc.edu\n[stdout:13] gcn-4-55.sdsc.edu\n[stdout:14] gcn-4-55.sdsc.edu\n[stdout:15] gcn-4-55.sdsc.edu\n[stdout:16] gcn-4-55.sdsc.edu\n[stdout:17] gcn-4-55.sdsc.edu\n[stdout:18] gcn-4-64.sdsc.edu\n[stdout:19] gcn-4-64.sdsc.edu\n[stdout:20] gcn-4-64.sdsc.edu\n[stdout:21] gcn-4-64.sdsc.edu\n[stdout:22] gcn-4-64.sdsc.edu\n[stdout:23] gcn-4-64.sdsc.edu\n[stdout:24] gcn-4-64.sdsc.edu\n[stdout:25] gcn-4-64.sdsc.edu\n[stdout:26] gcn-4-58.sdsc.edu\n[stdout:27] gcn-4-58.sdsc.edu\n[stdout:28] gcn-4-58.sdsc.edu\n[stdout:29] gcn-4-58.sdsc.edu\n[stdout:30] gcn-4-58.sdsc.edu\n[stdout:31] gcn-4-58.sdsc.edu\n[stdout:32] gcn-4-58.sdsc.edu\n[stdout:33] gcn-4-58.sdsc.edu\n[stdout:34] gcn-4-58.sdsc.edu\n[stdout:35] gcn-4-58.sdsc.edu\n[stdout:36] gcn-4-58.sdsc.edu\n[stdout:37] gcn-4-58.sdsc.edu\n[stdout:38] gcn-4-58.sdsc.edu\n[stdout:39] gcn-4-58.sdsc.edu\n[stdout:40] gcn-4-58.sdsc.edu\n[stdout:41] gcn-4-58.sdsc.edu\n[stdout:42] gcn-4-53.sdsc.edu\n[stdout:43] gcn-4-53.sdsc.edu\n[stdout:44] gcn-4-53.sdsc.edu\n[stdout:45] gcn-4-54.sdsc.edu\n[stdout:46] gcn-4-54.sdsc.edu\n[stdout:47] gcn-4-54.sdsc.edu\n[stdout:48] gcn-4-51.sdsc.edu\n[stdout:49] gcn-4-51.sdsc.edu\n[stdout:50] gcn-4-51.sdsc.edu\n[stdout:51] gcn-4-51.sdsc.edu\n[stdout:52] gcn-4-51.sdsc.edu\n[stdout:53] gcn-4-51.sdsc.edu\n[stdout:54] gcn-4-51.sdsc.edu\n[stdout:55] gcn-4-51.sdsc.edu\n[stdout:56] gcn-4-51.sdsc.edu\n[stdout:57] gcn-4-51.sdsc.edu\n[stdout:58] gcn-4-51.sdsc.edu\n[stdout:59] gcn-4-51.sdsc.edu\n[stdout:60] gcn-4-51.sdsc.edu\n[stdout:61] gcn-4-57.sdsc.edu\n[stdout:62] gcn-4-57.sdsc.edu\n[stdout:63] gcn-4-57.sdsc.edu\n[stdout:64] gcn-4-57.sdsc.edu\n[stdout:65] gcn-4-57.sdsc.edu\n[stdout:66] gcn-4-57.sdsc.edu\n[stdout:67] gcn-4-57.sdsc.edu\n[stdout:68] gcn-4-57.sdsc.edu\n[stdout:69] gcn-4-57.sdsc.edu\n[stdout:70] gcn-4-57.sdsc.edu\n[stdout:71] gcn-4-57.sdsc.edu\n[stdout:72] gcn-4-57.sdsc.edu\n[stdout:73] gcn-4-65.sdsc.edu\n[stdout:74] gcn-4-65.sdsc.edu\n[stdout:75] gcn-4-65.sdsc.edu\n[stdout:76] gcn-4-65.sdsc.edu\n[stdout:77] gcn-4-65.sdsc.edu\n[stdout:78] gcn-4-65.sdsc.edu\n[stdout:79] gcn-4-65.sdsc.edu\n[stdout:80] gcn-4-65.sdsc.edu\n[stdout:81] gcn-4-65.sdsc.edu\n[stdout:82] gcn-4-65.sdsc.edu\n[stdout:83] gcn-4-65.sdsc.edu\n[stdout:84] gcn-4-65.sdsc.edu\n[stdout:85] gcn-4-65.sdsc.edu\n[stdout:86] gcn-4-55.sdsc.edu\n[stdout:87] gcn-4-55.sdsc.edu\n[stdout:88] gcn-4-55.sdsc.edu\n[stdout:89] gcn-4-55.sdsc.edu\n[stdout:90] gcn-4-55.sdsc.edu\n[stdout:91] gcn-4-55.sdsc.edu\n[stdout:92] gcn-4-55.sdsc.edu\n[stdout:93] gcn-4-55.sdsc.edu\n[stdout:94] gcn-4-64.sdsc.edu\n[stdout:95] gcn-4-64.sdsc.edu\n[stdout:96] gcn-4-64.sdsc.edu\n[stdout:97] gcn-4-64.sdsc.edu\n[stdout:98] gcn-4-64.sdsc.edu\n[stdout:99] gcn-4-64.sdsc.edu\n[stdout:100] gcn-4-64.sdsc.edu\n[stdout:101] gcn-4-64.sdsc.edu\n[stdout:102] gcn-4-53.sdsc.edu\n[stdout:103] gcn-4-53.sdsc.edu\n[stdout:104] gcn-4-53.sdsc.edu\n[stdout:105] gcn-4-53.sdsc.edu\n[stdout:106] gcn-4-53.sdsc.edu\n[stdout:107] gcn-4-53.sdsc.edu\n[stdout:108] gcn-4-53.sdsc.edu\n[stdout:109] gcn-4-53.sdsc.edu\n[stdout:110] gcn-4-53.sdsc.edu\n[stdout:111] gcn-4-53.sdsc.edu\n[stdout:112] gcn-4-53.sdsc.edu\n[stdout:113] gcn-4-53.sdsc.edu\n[stdout:114] gcn-4-53.sdsc.edu\n[stdout:115] gcn-4-54.sdsc.edu\n[stdout:116] gcn-4-54.sdsc.edu\n[stdout:117] gcn-4-54.sdsc.edu\n[stdout:118] gcn-4-54.sdsc.edu\n[stdout:119] gcn-4-54.sdsc.edu\n[stdout:120] gcn-4-54.sdsc.edu\n[stdout:121] gcn-4-54.sdsc.edu\n[stdout:122] gcn-4-54.sdsc.edu\n[stdout:123] gcn-4-54.sdsc.edu\n[stdout:124] gcn-4-54.sdsc.edu\n[stdout:125] gcn-4-54.sdsc.edu\n[stdout:126] gcn-4-54.sdsc.edu\n[stdout:127] gcn-4-54.sdsc.edu\n", "output_type": "stream"}, {"name": "stderr", "text": "\n", "output_type": "stream"}], "execution_count": 4, "cell_type": "code"}, {"metadata": {"collapsed": true, "trusted": true}, "source": "view = c.load_balanced_view()", "outputs": [], "execution_count": 5, "cell_type": "code"}, {"metadata": {"collapsed": true, "trusted": true}, "source": "def process_file(filename):\n    import time\n    time.sleep(10)", "outputs": [], "execution_count": 6, "cell_type": "code"}, {"metadata": {"collapsed": true, "trusted": true}, "source": "from glob import glob", "outputs": [], "execution_count": 7, "cell_type": "code"}, {"metadata": {"collapsed": true, "trusted": true}, "source": "filenames = sorted(glob(\"data/*.csv\"))", "outputs": [], "execution_count": 8, "cell_type": "code"}, {"metadata": {"collapsed": false, "trusted": true}, "source": "filenames[:5]", "outputs": [{"metadata": {}, "data": {"text/plain": "['data/data_0000.csv',\n 'data/data_0001.csv',\n 'data/data_0002.csv',\n 'data/data_0003.csv',\n 'data/data_0004.csv']"}, "execution_count": 9, "output_type": "execute_result"}], "execution_count": 9, "cell_type": "code"}, {"metadata": {"collapsed": false, "trusted": true}, "source": "len(filenames)", "outputs": [{"metadata": {}, "data": {"text/plain": "1000"}, "execution_count": 10, "output_type": "execute_result"}], "execution_count": 10, "cell_type": "code"}, {"metadata": {}, "source": "`view.map` gets 2 arguments: a function that accepts 1 argument and a list of arguments.\n\nThe function will be sent to the available engines in order to process all the arguments in the list. Jupyter will take care of load balancing the resources available", "cell_type": "markdown"}, {"metadata": {"collapsed": false, "trusted": true}, "source": "%time optional_return_values = view.map(process_file, filenames, block=True)", "outputs": [{"name": "stdout", "text": "CPU times: user 28.9 s, sys: 2.51 s, total: 31.4 s\nWall time: 1min 20s\n", "output_type": "stream"}], "execution_count": 11, "cell_type": "code"}, {"metadata": {"collapsed": true, "trusted": true}, "source": "", "outputs": [], "execution_count": null, "cell_type": "code"}], "nbformat_minor": 0, "nbformat": 4}