Skip to content

Instantly share code, notes, and snippets.

@piroyoung

piroyoung/parallel_ipython

Created February 22, 2015 09:13
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save piroyoung/ce0cde05097211a1236d to your computer and use it in GitHub Desktop.
Save piroyoung/ce0cde05097211a1236d to your computer and use it in GitHub Desktop.
Download ZIP
A small demo for parallel processing using python
Raw
parallel_ipython
{
"metadata": {
"name": "",
"signature": "sha256:c3ec7130fab908e700b4e6f99bf9bd04e658422652e8a59a20e187beae1af590"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# \u304a\u524d\u8ab0\u3084\n",
"\n",
"* \u307f\u305a\u304b\u307f\u3000\u3072\u308d\u304d\u3000\uff08\u30c7\u30fc\u30bf\u30b5\u30a4\u30a8\u30f3\u30c6\u30a3\u30b9\u30c8\uff09\n",
"* @piroyoung\u3000\uff08\u30c7\u30fc\u30bf\u30b5\u30a4\u30a8\u30f3\u30c6\u30a3\u30b9\u30c8\uff09\n",
"* \u30a2\u30c9\u30c6\u30af\u306e\u30c7\u30fc\u30bf\u5206\u6790\u5c4b\u3055\u3093\u30fb\u6570\u7406\u30e2\u30c7\u30eb\u5c4b\u3055\u3093\u3000\uff08\u30c7\u30fc\u30bf\u30b5\u30a4\u30a8\u30f3\u30c6\u30a3\u30b9\u30c8\uff09\n",
"* python / R / SQL / AWS / Tableau / Scala / Spark\n",
"* \u305d\u3046\u3001\u50d5\u304c\u30c7\u30fc\u30bf\u30b5\u30a4\u30a8\u30f3\u30c6\u30a3\u30b9\u30c8\u3055( \u00b4 \u25bd ` )\uff89\n",
"\n",
"\n",
"#\u3000\u4f55\u3059\u308b\u3093\u3084\n",
"\n",
"* \u5f79\u306b\u7acb\u3064\u8a71\u3092\u3057\u307e\u3059\u3002\n",
"* \u81ea\u7136\u8a00\u8a9e\u306e\u524d\u51e6\u7406\u3063\u3066\u91cd\u3044\u3002\n",
"* \u4e26\u5217\u3067\u51e6\u7406\u3057\u305f\u3044\u306d\n",
"* iPython\u3067\u306fmultiprocessing.Pool\u304c\u4f7f\u3048\u306a\u3044\n",
"* \u3067\u3082iPython\u3067\u3082\u4e26\u5217\u51e6\u7406\u304c\u3067\u304d\u308b\u3053\u3068\u306b\u6c17\u3065\u3044\u305f\n",
"* \u306a\u306e\u3067\u4e26\u5217\u5316\u5206\u304b\u3061\u66f8\u304d\u306e\u30c7\u30e2\u3057\u307e\u3059\u3002"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# ipython\u3067\u4e26\u5217\u51e6\u7406\u3092\u3059\u308b\u8a71\u3000@piroyoung\n",
"\n",
"## ipython\u3063\u3066\u306a\u3093\u3084\n",
"* \u3053\u308c\n",
"* \u5206\u6790\u8005\u5fa1\u7528\u305f\u3061\u306epython\u306e\u5bfe\u8a71\u74b0\u5883\n",
"* \u30d6\u30e9\u30a6\u30b6\u3067\u52d5\u3044\u3066\u304b\u306a\u308a\u30af\u30fc\u30eb\n",
"* EC2\u3067\u52d5\u304b\u3059\u4e8b\u304c\u3067\u304d\u3066\u3001\u304a\u4f7f\u3044\u306e\u30d6\u30e9\u30a6\u30b6\u304c\u958b\u767a\u74b0\u5883\u306b\uff01\n",
"* R3\u30a4\u30f3\u30b9\u30bf\u30f3\u30b9\u3000\uff1e\u3000\u30b9\u30dd\u30c3\u30c8\u3067\u3055\u305b\u3070 \u30e1\u30e2\u30ea122G \u304c0.2$ / hour"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import MeCab\n",
"import time"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 17
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"t = MeCab.Tagger(\"-Owakati\")"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 2
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## \u308f\u304b\u3061\u304c\u304d\u3057\u305f\u7d50\u679c\u3092\u8fd4\u3059"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"wakati = lambda x: t.parse(x).rstrip()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 3
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"wakati('\u3059\u3082\u3082\u3082\u3082\u3082\u3054\u6ce8\u6587\u306f\u3046\u3055\u304e\u3067\u3059\u304b\u3082\u3082\u3082\u3082\u3093\u3082\u3082\u3093\u3093\u3082\u3082\u3082\u3068\u3057\u307e\u308a\u3093\u3082\u3082\u3082\u3082\u3082\u3082\u3082\u3082\u3082\u3082\u3082\u3082\u306e\u3046\u3061\u30b9\u30e4\u30a1')"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 22,
"text": [
"'\u3059\u3082\u3082 \u3082\u3082 \u3082 \u3054 \u6ce8\u6587 \u306f \u3046\u3055\u304e \u3067\u3059 \u304b \u3082\u3082 \u3082 \u3082\u3093 \u3082 \u3082\u3093 \u3093 \u3082 \u3082\u3082 \u3068 \u3057\u307e \u308a\u3093\u3082 \u3082\u3082 \u3082 \u3082\u3082 \u3082 \u3082\u3082 \u3082 \u3082\u3082 \u306e \u3046\u3061 \u30b9\u30e4\u30a1'"
]
}
],
"prompt_number": 22
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## \u9752\u7a7a\u6587\u5eab\u306e\u659c\u967d\u3092\u8aad\u307f\u8fbc\u3080"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"with open('shayo.txt') as fp:\n",
" text = [line.replace('\\u3000','').rstrip() for line in fp.readlines()]"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 5
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"text[:30]"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 6,
"text": [
"['\u671d\u3001\u98df\u5802\u3067\u30b9\u30a6\u30d7\u3092\u4e00\u3055\u3058\u3001\u3059\u3063\u3068\u5438\u3063\u3066\u304a\u6bcd\u3055\u307e\u304c\u3001',\n",
" '\u300c\u3042\u300d',\n",
" '\u3068\u5e7d\u304b\u3059\u304b\u306a\u53eb\u3073\u58f0\u3092\u304a\u6319\u3052\u306b\u306a\u3063\u305f\u3002',\n",
" '\u300c\u9aea\u306e\u6bdb\uff1f\u300d',\n",
" '\u30b9\u30a6\u30d7\u306b\u4f55\u304b\u3001\u30a4\u30e4\u306a\u3082\u306e\u3067\u3082\u5165\u3063\u3066\u3044\u305f\u306e\u304b\u3057\u3089\u3001\u3068\u601d\u3063\u305f\u3002',\n",
" '\u300c\u3044\u3044\u3048\u300d',\n",
" '\u304a\u6bcd\u3055\u307e\u306f\u3001\u4f55\u4e8b\u3082\u7121\u304b\u3063\u305f\u3088\u3046\u306b\u3001\u307e\u305f\u3072\u3089\u308a\u3068\u4e00\u3055\u3058\u3001\u30b9\u30a6\u30d7\u3092\u304a\u53e3\u306b\u6d41\u3057\u8fbc\u307f\u3001\u3059\u307e\u3057\u3066\u304a\u9854\u3092\u6a2a\u306b\u5411\u3051\u3001\u304a\u52dd\u624b\u306e\u7a93\u306e\u3001\u6e80\u958b\u306e\u5c71\u685c\u306b\u8996\u7dda\u3092\u9001\u308a\u3001\u305d\u3046\u3057\u3066\u304a\u9854\u3092\u6a2a\u306b\u5411\u3051\u305f\u307e\u307e\u3001\u307e\u305f\u3072\u3089\u308a\u3068\u4e00\u3055\u3058\u3001\u30b9\u30a6\u30d7\u3092\u5c0f\u3055\u306a\u304a\u5507\u306e\u3042\u3044\u3060\u306b\u6ed1\u308a\u8fbc\u307e\u305b\u305f\u3002\u30d2\u30e9\u30ea\u3001\u3068\u3044\u3046\u5f62\u5bb9\u306f\u3001\u304a\u6bcd\u3055\u307e\u306e\u5834\u5408\u3001\u6c7a\u3057\u3066\u8a87\u5f35\u3067\u306f\u7121\u3044\u3002\u5a66\u4eba\u96d1\u8a8c\u306a\u3069\u306b\u51fa\u3066\u3044\u308b\u304a\u98df\u4e8b\u306e\u3044\u305f\u3060\u304d\u65b9\u306a\u3069\u3068\u306f\u3001\u3066\u3093\u3067\u307e\u308b\u3067\u3001\u9055\u3063\u3066\u3044\u3089\u3063\u3057\u3083\u308b\u3002\u5f1f\u306e\u76f4\u6cbb\u306a\u304a\u3058\u304c\u3044\u3064\u304b\u3001\u304a\u9152\u3092\u98f2\u307f\u306a\u304c\u3089\u3001\u59c9\u306e\u79c1\u306b\u5411\u3063\u3066\u3053\u3046\u8a00\u3063\u305f\u4e8b\u304c\u3042\u308b\u3002',\n",
" '\u300c\u7235\u4f4d\u3057\u3083\u304f\u3044\u304c\u3042\u308b\u304b\u3089\u3001\u8cb4\u65cf\u3060\u3068\u3044\u3046\u308f\u3051\u306b\u306f\u3044\u304b\u306a\u3044\u3093\u3060\u305c\u3002\u7235\u4f4d\u304c\u7121\u304f\u3066\u3082\u3001\u5929\u7235\u3068\u3044\u3046\u3082\u306e\u3092\u6301\u3063\u3066\u3044\u308b\u7acb\u6d3e\u306a\u8cb4\u65cf\u306e\u3072\u3068\u3082\u3042\u308b\u3057\u3001\u304a\u308c\u305f\u3061\u306e\u3088\u3046\u306b\u7235\u4f4d\u3060\u3051\u306f\u6301\u3063\u3066\u3044\u3066\u3082\u3001\u8cb4\u65cf\u3069\u3053\u308d\u304b\u3001\u8ce4\u6c11\u305b\u3093\u307f\u3093\u306b\u3061\u304b\u3044\u306e\u3082\u3044\u308b\u3002\u5ca9\u5cf6\u306a\u3093\u3066\u306e\u306f\uff08\u3068\u76f4\u6cbb\u306e\u5b66\u53cb\u306e\u4f2f\u7235\u306e\u304a\u540d\u524d\u3092\u6319\u3052\u3066\uff09\u3042\u3093\u306a\u306e\u306f\u3001\u307e\u3063\u305f\u304f\u3001\u65b0\u5bbf\u306e\u904a\u5ed3\u3086\u3046\u304b\u304f\u306e\u5ba2\u5f15\u304d\u756a\u982d\u3088\u308a\u3082\u3001\u3082\u3063\u3068\u3052\u3073\u3066\u308b\u611f\u3058\u3058\u3083\u306d\u3048\u304b\u3002\u3053\u306a\u3044\u3060\u3082\u3001\u67f3\u4e95\u3084\u306a\u3044\uff08\u3068\u3001\u3084\u306f\u308a\u5f1f\u306e\u5b66\u53cb\u3067\u3001\u5b50\u7235\u306e\u5fa1\u6b21\u7537\u306e\u304b\u305f\u306e\u304a\u540d\u524d\u3092\u6319\u3052\u3066\uff09\u306e\u5144\u8cb4\u306e\u7d50\u5a5a\u5f0f\u306b\u3001\u3042\u3093\u3061\u304d\u3057\u3087\u3046\u3001\u30bf\u30ad\u30b7\u30a4\u30c9\u306a\u3093\u304b\u7740\u3066\u3001\u306a\u3093\u3060\u3063\u3066\u307e\u305f\u3001\u30bf\u30ad\u30b7\u30a4\u30c9\u306a\u3093\u304b\u3092\u7740\u3066\u6765\u308b\u5fc5\u8981\u304c\u3042\u308b\u3093\u3060\u3001\u305d\u308c\u306f\u307e\u3042\u3044\u3044\u3068\u3057\u3066\u3001\u30c6\u30fc\u30d6\u30eb\u30b9\u30d4\u30fc\u30c1\u306e\u6642\u306b\u3001\u3042\u306e\u91ce\u90ce\u3001\u30b4\u30b6\u30a4\u30de\u30b9\u30eb\u3068\u3044\u3046\u4e0d\u53ef\u601d\u8b70\u306a\u8a00\u8449\u3092\u3064\u304b\u3063\u305f\u306e\u306b\u306f\u3001\u3052\u3063\u3068\u306a\u3063\u305f\u3002\u6c17\u53d6\u308b\u3068\u3044\u3046\u4e8b\u306f\u3001\u4e0a\u54c1\u3068\u3044\u3046\u4e8b\u3068\u3001\u305c\u3093\u305c\u3093\u7121\u95a2\u4fc2\u306a\u3042\u3055\u307e\u3057\u3044\u865a\u52e2\u3060\u3002\u9ad8\u7b49\u5fa1\u304a\u3093\u4e0b\u5bbf\u3068\u66f8\u3044\u3066\u3042\u308b\u770b\u677f\u304c\u672c\u90f7\u3042\u305f\u308a\u306b\u3088\u304f\u3042\u3063\u305f\u3082\u306e\u3060\u3051\u308c\u3069\u3082\u3001\u3058\u3063\u3055\u3044\u83ef\u65cf\u306a\u3093\u3066\u3082\u306e\u306e\u5927\u90e8\u5206\u306f\u3001\u9ad8\u7b49\u5fa1\u4e5e\u98df\u304a\u3093\u3053\u3058\u304d\u3068\u3067\u3082\u3044\u3063\u305f\u3088\u3046\u306a\u3082\u306e\u306a\u3093\u3060\u3002\u3057\u3093\u306e\u8cb4\u65cf\u306f\u3001\u3042\u3093\u306a\u5ca9\u5cf6\u307f\u305f\u3044\u306a\u4e0b\u624b\u306a\u6c17\u53d6\u308a\u304b\u305f\u306a\u3093\u304b\u3001\u3057\u3084\u3057\u306a\u3044\u3088\u3002\u304a\u308c\u305f\u3061\u306e\u4e00\u65cf\u3067\u3082\u3001\u307b\u3093\u3082\u306e\u306e\u8cb4\u65cf\u306f\u3001\u307e\u3042\u3001\u30de\u30de\u304f\u3089\u3044\u306e\u3082\u306e\u3060\u308d\u3046\u3002\u3042\u308c\u306f\u3001\u307b\u3093\u3082\u306e\u3060\u3088\u3002\u304b\u306a\u308f\u306d\u3048\u3068\u3053\u308d\u304c\u3042\u308b\u300d',\n",
" '\u30b9\u30a6\u30d7\u306e\u3044\u305f\u3060\u304d\u304b\u305f\u306b\u3057\u3066\u3082\u3001\u79c1\u305f\u3061\u306a\u3089\u3001\u304a\u76bf\u3055\u3089\u306e\u4e0a\u306b\u3059\u3053\u3057\u3046\u3064\u3080\u304d\u3001\u305d\u3046\u3057\u3066\u30b9\u30d7\u30a6\u30f3\u3092\u6a2a\u306b\u6301\u3063\u3066\u30b9\u30a6\u30d7\u3092\u63ac\u3059\u304f\u3044\u3001\u30b9\u30d7\u30a6\u30f3\u3092\u6a2a\u306b\u3057\u305f\u307e\u307e\u53e3\u5143\u306b\u904b\u3093\u3067\u3044\u305f\u3060\u304f\u306e\u3060\u3051\u308c\u3069\u3082\u3001\u304a\u6bcd\u3055\u307e\u306f\u5de6\u624b\u306e\u304a\u6307\u3092\u8efd\u304f\u30c6\u30fc\u30d6\u30eb\u306e\u7e01\u3075\u3061\u306b\u304b\u3051\u3066\u3001\u4e0a\u4f53\u3092\u304b\u304c\u3081\u308b\u4e8b\u3082\u7121\u304f\u3001\u304a\u9854\u3092\u3057\u3083\u3093\u3068\u6319\u3052\u3066\u3001\u304a\u76bf\u3092\u308d\u304f\u306b\u898b\u3082\u305b\u305a\u30b9\u30d7\u30a6\u30f3\u3092\u6a2a\u306b\u3057\u3066\u3055\u3063\u3068\u63ac\u3063\u3066\u3001\u305d\u308c\u304b\u3089\u3001\u71d5\u3064\u3070\u3081\u306e\u3088\u3046\u306b\u3001\u3068\u3067\u3082\u5f62\u5bb9\u3057\u305f\u3044\u304f\u3089\u3044\u306b\u8efd\u304f\u9bae\u3084\u304b\u306b\u30b9\u30d7\u30a6\u30f3\u3092\u304a\u53e3\u3068\u76f4\u89d2\u306b\u306a\u308b\u3088\u3046\u306b\u6301\u3061\u904b\u3093\u3067\u3001\u30b9\u30d7\u30a6\u30f3\u306e\u5c16\u7aef\u305b\u3093\u305f\u3093\u304b\u3089\u3001\u30b9\u30a6\u30d7\u3092\u304a\u5507\u306e\u3042\u3044\u3060\u306b\u6d41\u3057\u8fbc\u3080\u306e\u3067\u3042\u308b\u3002\u305d\u3046\u3057\u3066\u3001\u7121\u5fc3\u305d\u3046\u306b\u3042\u3061\u3053\u3061\u508d\u898b\u308f\u304d\u307f\u306a\u3069\u306a\u3055\u308a\u306a\u304c\u3089\u3001\u3072\u3089\u308a\u3072\u3089\u308a\u3068\u3001\u307e\u308b\u3067\u5c0f\u3055\u306a\u7ffc\u306e\u3088\u3046\u306b\u30b9\u30d7\u30a6\u30f3\u3092\u3042\u3064\u304b\u3044\u3001\u30b9\u30a6\u30d7\u3092\u4e00\u6ef4\u3082\u304a\u3053\u307c\u3057\u306b\u306a\u308b\u4e8b\u3082\u7121\u3044\u3057\u3001\u5438\u3046\u97f3\u3082\u304a\u76bf\u306e\u97f3\u3082\u3001\u3061\u3063\u3068\u3082\u304a\u7acb\u3066\u306b\u306a\u3089\u306c\u306e\u3060\u3002\u305d\u308c\u306f\u6240\u8b02\u3044\u308f\u3086\u308b\u6b63\u5f0f\u793c\u6cd5\u306b\u304b\u306a\u3063\u305f\u3044\u305f\u3060\u304d\u65b9\u3067\u306f\u7121\u3044\u304b\u3082\u77e5\u308c\u306a\u3044\u3051\u308c\u3069\u3082\u3001\u79c1\u306e\u76ee\u306b\u306f\u3001\u3068\u3066\u3082\u53ef\u611b\u304b\u308f\u3044\u3089\u3057\u304f\u3001\u305d\u308c\u3053\u305d\u307b\u3093\u3082\u306e\u307f\u305f\u3044\u306b\u898b\u3048\u308b\u3002\u307e\u305f\u3001\u4e8b\u5b9f\u3001\u304a\u98f2\u7269\u306f\u3001\u53e3\u306b\u6d41\u3057\u8fbc\u3080\u3088\u3046\u306b\u3057\u3066\u3044\u305f\u3060\u3044\u305f\u307b\u3046\u304c\u3001\u4e0d\u601d\u8b70\u306a\u304f\u3089\u3044\u306b\u304a\u3044\u3057\u3044\u3082\u306e\u3060\u3002\u3051\u308c\u3069\u3082\u3001\u79c1\u306f\u76f4\u6cbb\u306e\u8a00\u3046\u3088\u3046\u306a\u9ad8\u7b49\u5fa1\u4e5e\u98df\u306a\u306e\u3060\u304b\u3089\u3001\u304a\u6bcd\u3055\u307e\u306e\u3088\u3046\u306b\u3042\u3093\u306a\u306b\u8efd\u304f\u7121\u96d1\u4f5c\u3080\u305e\u3046\u3055\u306b\u30b9\u30d7\u30a6\u30f3\u3092\u3042\u3084\u3064\u308b\u4e8b\u304c\u51fa\u6765\u305a\u3001\u4ed5\u65b9\u306a\u304f\u3001\u3042\u304d\u3089\u3081\u3066\u3001\u304a\u76bf\u306e\u4e0a\u306b\u3046\u3064\u3080\u304d\u3001\u6240\u8b02\u6b63\u5f0f\u793c\u6cd5\u3069\u304a\u308a\u306e\u9670\u6c17\u306a\u3044\u305f\u3060\u304d\u65b9\u3092\u3057\u3066\u3044\u308b\u306e\u3067\u3042\u308b\u3002',\n",
" '\u30b9\u30a6\u30d7\u306b\u9650\u3089\u305a\u3001\u304a\u6bcd\u3055\u307e\u306e\u98df\u4e8b\u306e\u3044\u305f\u3060\u304d\u65b9\u306f\u3001\u9817\u3059\u3053\u3076\u308b\u793c\u6cd5\u306b\u306f\u305a\u308c\u3066\u3044\u308b\u3002\u304a\u8089\u304c\u51fa\u308b\u3068\u3001\u30ca\u30a4\u30d5\u3068\u30d5\u30aa\u30af\u3067\u3001\u3055\u3063\u3055\u3068\u5168\u90e8\u5c0f\u3055\u304f\u5207\u308a\u308f\u3051\u3066\u3057\u307e\u3063\u3066\u3001\u305d\u308c\u304b\u3089\u30ca\u30a4\u30d5\u3092\u6368\u3066\u3001\u30d5\u30aa\u30af\u3092\u53f3\u624b\u306b\u6301\u3061\u304b\u3048\u3001\u305d\u306e\u4e00\u304d\u308c\u4e00\u304d\u308c\u3092\u30d5\u30aa\u30af\u306b\u523a\u3057\u3066\u3086\u3063\u304f\u308a\u697d\u3057\u305d\u3046\u306b\u53ec\u3057\u4e0a\u304c\u3063\u3066\u3044\u3089\u3063\u3057\u3083\u308b\u3002\u307e\u305f\u3001\u9aa8\u3064\u304d\u306e\u30c1\u30ad\u30f3\u306a\u3069\u3001\u79c1\u305f\u3061\u304c\u304a\u76bf\u3092\u9cf4\u3089\u3055\u305a\u306b\u9aa8\u304b\u3089\u8089\u3092\u5207\u308a\u306f\u306a\u3059\u306e\u306b\u82e6\u5fc3\u3057\u3066\u3044\u308b\u6642\u3001\u304a\u6bcd\u3055\u307e\u306f\u3001\u5e73\u6c17\u3067\u3072\u3087\u3044\u3068\u6307\u5148\u3067\u9aa8\u306e\u3068\u3053\u308d\u3092\u3064\u307e\u3093\u3067\u6301\u3061\u4e0a\u3052\u3001\u304a\u53e3\u3067\u9aa8\u3068\u8089\u3092\u306f\u306a\u3057\u3066\u6f84\u307e\u3057\u3066\u3044\u3089\u3063\u3057\u3083\u308b\u3002\u305d\u3093\u306a\u91ce\u86ee\u306a\u4ed5\u8349\u3082\u3001\u304a\u6bcd\u3055\u307e\u304c\u306a\u3055\u308b\u3068\u3001\u53ef\u611b\u3089\u3057\u3044\u3070\u304b\u308a\u304b\u3001\u3078\u3093\u306b\u30a8\u30ed\u30c1\u30c3\u30af\u306b\u3055\u3048\u898b\u3048\u308b\u306e\u3060\u304b\u3089\u3001\u3055\u3059\u304c\u306b\u307b\u3093\u3082\u306e\u306f\u9055\u3063\u305f\u3082\u306e\u3067\u3042\u308b\u3002\u9aa8\u3064\u304d\u306e\u30c1\u30ad\u30f3\u306e\u5834\u5408\u3060\u3051\u3067\u306a\u304f\u3001\u304a\u6bcd\u3055\u307e\u306f\u3001\u30e9\u30f3\u30c1\u306e\u304a\u83dc\u3055\u3044\u306e\u30cf\u30e0\u3084\u30bd\u30bb\u30fc\u30b8\u306a\u3069\u3082\u3001\u3072\u3087\u3044\u3068\u6307\u5148\u3067\u3064\u307e\u3093\u3067\u53ec\u3057\u4e0a\u308b\u4e8b\u3055\u3048\u6642\u305f\u307e\u3042\u308b\u3002',\n",
" '\u300c\u304a\u3080\u3059\u3073\u304c\u3001\u3069\u3046\u3057\u3066\u304a\u3044\u3057\u3044\u306e\u3060\u304b\u3001\u77e5\u3063\u3066\u3044\u307e\u3059\u304b\u3002\u3042\u308c\u306f\u306d\u3001\u4eba\u9593\u306e\u6307\u3067\u63e1\u308a\u3057\u3081\u3066\u4f5c\u308b\u304b\u3089\u3067\u3059\u3088\u300d',\n",
" '\u3068\u304a\u3063\u3057\u3083\u3063\u305f\u4e8b\u3082\u3042\u308b\u3002',\n",
" '\u672c\u5f53\u306b\u3001\u624b\u3067\u305f\u3079\u305f\u3089\u3001\u304a\u3044\u3057\u3044\u3060\u308d\u3046\u306a\u3001\u3068\u79c1\u3082\u601d\u3046\u4e8b\u304c\u3042\u308b\u3051\u308c\u3069\u3001\u79c1\u306e\u3088\u3046\u306a\u9ad8\u7b49\u5fa1\u4e5e\u98df\u304c\u3001\u4e0b\u624b\u306b\u771f\u4f3c\u307e\u306d\u3057\u3066\u305d\u308c\u3092\u3084\u3063\u305f\u3089\u3001\u305d\u308c\u3053\u305d\u307b\u3093\u3082\u306e\u306e\u4e5e\u98df\u306e\u56f3\u306b\u306a\u3063\u3066\u3057\u307e\u3044\u305d\u3046\u306a\u6c17\u3082\u3059\u308b\u306e\u3067\u6211\u6162\u3057\u3066\u3044\u308b\u3002',\n",
" '\u5f1f\u306e\u76f4\u6cbb\u3067\u3055\u3048\u3001\u30de\u30de\u306b\u306f\u304b\u306a\u308f\u306d\u3048\u3001\u3068\u8a00\u3063\u3066\u3044\u308b\u304c\u3001\u3064\u304f\u3065\u304f\u79c1\u3082\u3001\u304a\u6bcd\u3055\u307e\u306e\u771f\u4f3c\u306f\u56f0\u96e3\u3067\u3001\u7d76\u671b\u307f\u305f\u3044\u306a\u3082\u306e\u3092\u3055\u3048\u611f\u3058\u308b\u4e8b\u304c\u3042\u308b\u3002\u3044\u3064\u304b\u3001\u897f\u7247\u753a\u306e\u304a\u3046\u3061\u306e\u5965\u5ead\u3067\u3001\u79cb\u306e\u306f\u3058\u3081\u306e\u6708\u306e\u3044\u3044\u591c\u3067\u3042\u3063\u305f\u304c\u3001\u79c1\u306f\u304a\u6bcd\u3055\u307e\u3068\u4e8c\u4eba\u3067\u304a\u6c60\u306e\u7aef\u306e\u3042\u305a\u307e\u3084\u3067\u3001\u304a\u6708\u898b\u3092\u3057\u3066\u3001\u72d0\u304d\u3064\u306d\u306e\u5ac1\u5165\u308a\u3068\u9f20\u306d\u305a\u307f\u306e\u5ac1\u5165\u308a\u3068\u306f\u3001\u304a\u5ac1\u306e\u304a\u652f\u5ea6\u304c\u3069\u3046\u3061\u304c\u3046\u304b\u3001\u306a\u3069\u7b11\u3044\u306a\u304c\u3089\u8a71\u5408\u3063\u3066\u3044\u308b\u3046\u3061\u306b\u3001\u304a\u6bcd\u3055\u307e\u306f\u3001\u3064\u3068\u304a\u7acb\u3061\u306b\u306a\u3063\u3066\u3001\u3042\u305a\u307e\u3084\u306e\u508d\u305d\u3070\u306e\u8429\u306f\u304e\u306e\u3057\u3052\u307f\u306e\u5965\u3078\u304a\u306f\u3044\u308a\u306b\u306a\u308a\u3001\u305d\u308c\u304b\u3089\u3001\u8429\u306e\u767d\u3044\u82b1\u306e\u3042\u3044\u3060\u304b\u3089\u3001\u3082\u3063\u3068\u3042\u3056\u3084\u304b\u306b\u767d\u3044\u304a\u9854\u3092\u304a\u51fa\u3057\u306b\u306a\u3063\u3066\u3001\u5c11\u3057\u7b11\u3063\u3066\u3001',\n",
" '\u300c\u304b\u305a\u5b50\u3084\u3001\u304a\u6bcd\u3055\u307e\u304c\u3044\u307e\u4f55\u3092\u306a\u3055\u3063\u3066\u3044\u308b\u304b\u3001\u3042\u3066\u3066\u3054\u3089\u3093\u300d',\n",
" '\u3068\u304a\u3063\u3057\u3083\u3063\u305f\u3002',\n",
" '\u300c\u304a\u82b1\u3092\u6298\u3063\u3066\u3044\u3089\u3063\u3057\u3083\u308b\u300d',\n",
" '\u3068\u7533\u3057\u4e0a\u3052\u305f\u3089\u3001\u5c0f\u3055\u3044\u58f0\u3092\u6319\u3052\u3066\u304a\u7b11\u3044\u306b\u306a\u308a\u3001',\n",
" '\u300c\u304a\u3057\u3063\u3053\u3088\u300d',\n",
" '\u3068\u304a\u3063\u3057\u3083\u3063\u305f\u3002',\n",
" '\u3061\u3063\u3068\u3082\u3057\u3083\u304c\u3093\u3067\u3044\u3089\u3063\u3057\u3083\u3089\u306a\u3044\u306e\u306b\u306f\u9a5a\u3044\u305f\u304c\u3001\u3051\u308c\u3069\u3082\u3001\u79c1\u306a\u3069\u306b\u306f\u3068\u3066\u3082\u771f\u4f3c\u3089\u308c\u306a\u3044\u3001\u3057\u3093\u304b\u3089\u53ef\u611b\u3089\u3057\u3044\u611f\u3058\u304c\u3042\u3063\u305f\u3002',\n",
" '\u3051\u3055\u306e\u30b9\u30a6\u30d7\u306e\u4e8b\u304b\u3089\u3001\u305a\u3044\u3076\u3093\u8131\u7dda\u3057\u3061\u3083\u3063\u305f\u3051\u308c\u3069\u3001\u3053\u306a\u3044\u3060\u6216\u3042\u308b\u672c\u3067\u8aad\u3093\u3067\u3001\u30eb\u30a4\u738b\u671d\u306e\u9803\u306e\u8cb4\u5a66\u4eba\u305f\u3061\u306f\u3001\u5bae\u6bbf\u306e\u304a\u5ead\u3084\u3001\u305d\u308c\u304b\u3089\u5eca\u4e0b\u306e\u9685\u3059\u307f\u306a\u3069\u3067\u3001\u5e73\u6c17\u3067\u304a\u3057\u3063\u3053\u3092\u3057\u3066\u3044\u305f\u3068\u3044\u3046\u4e8b\u3092\u77e5\u308a\u3001\u305d\u306e\u7121\u5fc3\u3055\u304c\u3001\u672c\u5f53\u306b\u53ef\u611b\u3089\u3057\u304f\u3001\u79c1\u306e\u304a\u6bcd\u3055\u307e\u306a\u3069\u3082\u3001\u305d\u306e\u3088\u3046\u306a\u307b\u3093\u3082\u306e\u306e\u8cb4\u5a66\u4eba\u306e\u6700\u5f8c\u306e\u3072\u3068\u308a\u306a\u306e\u3067\u306f\u306a\u304b\u308d\u3046\u304b\u3068\u8003\u3048\u305f\u3002',\n",
" '\u3055\u3066\u3001\u3051\u3055\u306f\u3001\u30b9\u30a6\u30d7\u3092\u4e00\u3055\u3058\u304a\u5438\u3044\u306b\u306a\u3063\u3066\u3001\u3042\u3001\u3068\u5c0f\u3055\u3044\u58f0\u3092\u304a\u6319\u3052\u306b\u306a\u3063\u305f\u306e\u3067\u3001\u9aea\u306e\u6bdb\uff1f\u3068\u304a\u305f\u305a\u306d\u3059\u308b\u3068\u3001\u3044\u3044\u3048\u3001\u3068\u304a\u7b54\u3048\u306b\u306a\u308b\u3002',\n",
" '\u300c\u5869\u8f9b\u304b\u3063\u305f\u304b\u3057\u3089\u300d',\n",
" '\u3051\u3055\u306e\u30b9\u30a6\u30d7\u306f\u3001\u3053\u306a\u3044\u3060\u30a2\u30e1\u30ea\u30ab\u304b\u3089\u914d\u7d66\u306b\u306a\u3063\u305f\u7f50\u8a70\u304b\u3093\u3065\u3081\u306e\u30b0\u30ea\u30f3\u30d4\u30a4\u30b9\u3092\u88cf\u3054\u3057\u3057\u3066\u3001\u79c1\u304c\u30dd\u30bf\u30fc\u30b8\u30e5\u307f\u305f\u3044\u306b\u4f5c\u3063\u305f\u3082\u306e\u3067\u3001\u3082\u3068\u3082\u3068\u304a\u6599\u7406\u306b\u306f\u81ea\u4fe1\u304c\u7121\u3044\u306e\u3067\u3001\u304a\u6bcd\u3055\u307e\u306b\u3001\u3044\u3044\u3048\u3001\u3068\u8a00\u308f\u308c\u3066\u3082\u3001\u306a\u304a\u3082\u3001\u306f\u3089\u306f\u3089\u3057\u3066\u305d\u3046\u305f\u305a\u306d\u305f\u3002',\n",
" '\u300c\u304a\u4e0a\u624b\u306b\u51fa\u6765\u307e\u3057\u305f\u300d',\n",
" '\u304a\u6bcd\u3055\u307e\u306f\u3001\u307e\u3058\u3081\u306b\u305d\u3046\u8a00\u3044\u3001\u30b9\u30a6\u30d7\u3092\u3059\u307e\u3057\u3066\u3001\u305d\u308c\u304b\u3089\u304a\u6d77\u82d4\u306e\u308a\u3067\u5305\u3093\u3060\u304a\u3080\u3059\u3073\u3092\u624b\u3067\u3064\u307e\u3093\u3067\u304a\u3042\u304c\u308a\u306b\u306a\u3063\u305f\u3002',\n",
" '\u79c1\u306f\u5c0f\u3055\u3044\u6642\u304b\u3089\u3001\u671d\u3054\u306f\u3093\u304c\u304a\u3044\u3057\u304f\u306a\u304f\u3001\u5341\u6642\u9803\u306b\u306a\u3089\u306a\u3051\u308c\u3070\u3001\u304a\u306a\u304b\u304c\u3059\u304b\u306a\u3044\u306e\u3067\u3001\u305d\u306e\u6642\u3082\u3001\u30b9\u30a6\u30d7\u3060\u3051\u306f\u3069\u3046\u3084\u3089\u3059\u307e\u3057\u305f\u3051\u308c\u3069\u3082\u3001\u98df\u3079\u308b\u306e\u304c\u305f\u3044\u304e\u3067\u3001\u304a\u3080\u3059\u3073\u3092\u304a\u76bf\u306b\u8f09\u305b\u3066\u3001\u305d\u308c\u306b\u304a\u7bb8\u306f\u3057\u3092\u7a81\u8fbc\u307f\u3001\u3050\u3057\u3083\u3050\u3057\u3083\u306b\u3053\u308f\u3057\u3066\u3001\u305d\u308c\u304b\u3089\u3001\u305d\u306e\u4e00\u304b\u3051\u3089\u3092\u304a\u7bb8\u3067\u3064\u307e\u307f\u4e0a\u3052\u3001\u304a\u6bcd\u3055\u307e\u304c\u30b9\u30a6\u30d7\u3092\u53ec\u3057\u4e0a\u308b\u6642\u306e\u30b9\u30d7\u30a6\u30f3\u307f\u305f\u3044\u306b\u3001\u304a\u7bb8\u3092\u304a\u53e3\u3068\u76f4\u89d2\u306b\u3057\u3066\u3001\u307e\u308b\u3067\u5c0f\u9ce5\u306b\u990c\u3048\u3055\u3092\u3084\u308b\u3088\u3046\u306a\u5de5\u5408\u3050\u3042\u3044\u306b\u304a\u53e3\u306b\u62bc\u3057\u8fbc\u307f\u3001\u306e\u308d\u306e\u308d\u3068\u3044\u305f\u3060\u3044\u3066\u3044\u308b\u3046\u3061\u306b\u3001\u304a\u6bcd\u3055\u307e\u306f\u3082\u3046\u304a\u98df\u4e8b\u3092\u5168\u90e8\u3059\u307e\u3057\u3066\u3057\u307e\u3063\u3066\u3001\u305d\u3063\u3068\u304a\u7acb\u3061\u306b\u306a\u308a\u3001\u671d\u65e5\u306e\u5f53\u3063\u3066\u3044\u308b\u58c1\u306b\u304a\u80cc\u4e2d\u3092\u3082\u305f\u305b\u304b\u3051\u3001\u3057\u3070\u3089\u304f\u9ed9\u3063\u3066\u79c1\u306e\u304a\u98df\u4e8b\u306e\u4ed5\u65b9\u3092\u898b\u3066\u3044\u3089\u3057\u3066\u3001',\n",
" '\u300c\u304b\u305a\u5b50\u306f\u3001\u307e\u3060\u3001\u99c4\u76ee\u306a\u306e\u306d\u3002\u671d\u5fa1\u98ef\u304c\u4e00\u756a\u304a\u3044\u3057\u304f\u306a\u308b\u3088\u3046\u306b\u306a\u3089\u306a\u3051\u308c\u3070\u300d',\n",
" '\u3068\u304a\u3063\u3057\u3083\u3063\u305f\u3002']"
]
}
],
"prompt_number": 6
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# \u3053\u3053\u304b\u3089\u304c\u4e26\u5217\u5316\u306e\u8a71\uff08ipython.parallel.Client\uff09\n",
"\n",
"* \u307e\u305aipython\u306e\u30db\u30fc\u30e0\u304b\u3089\u30af\u30e9\u30b9\u30bf\u30fc\u3092\u8d77\u52d5"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"from IPython.parallel import Client\n",
"\n",
"c = Client() #\u30af\u30e9\u30a4\u30a2\u30f3\u30c8\u30aa\u30d6\u30b8\u30a7\u30af\u30c8\u306e\u30a4\u30f3\u30b9\u30bf\u30f3\u30b9\u3092\u4f5c\u308b\n",
"cluster = c[:] #\u5168\u306e\u30fc\u3069\u3092\u4f7f\u7528\u3057\u3066\n",
"len(cluster)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 7,
"text": [
"8"
]
}
],
"prompt_number": 7
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"cluster.execute('''\n",
"import time\n",
"''')"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 8,
"text": [
"<AsyncResult: finished>"
]
}
],
"prompt_number": 8
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## mapping\n",
"\n",
"* \u914d\u5217\u30fb\u30ea\u30b9\u30c8\u3000\u306a\u3069\u306e\u5168\u3066\u306e\u8981\u7d20\u306b\u304a\u306a\u3058\u51e6\u7406\u3092\u9069\u7528\u3055\u305b\u308b\u3053\u3068\n",
"* \u4e26\u5217\u5316\u3068\u306e\u76f8\u6027\u304c\u975e\u5e38\u306b\u3044\u3044\n",
"* \u3060\u304b\u3089\u5e30\u7d0d\u7684\u306a\u51e6\u7406\u4ee5\u5916\u306ffor\u6587\u3088\u308amap(or apply)\u3092\u4f7f\u3063\u305f\u307b\u3046\u304c\u3044\u3044\u4e8b\u304c\u304a\u304a\u3044"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"res = cluster.map(lambda x: x**2 ,range(10)) #\u30de\u30c3\u30d4\u30f3\u30b0\uff01"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 9
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"res.get()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 10,
"text": [
"[0, 1, 4, 9, 16, 25, 36, 49, 64, 81]"
]
}
],
"prompt_number": 10
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## [\u691c\u8a3c\uff11]\u901f\u5ea6\u6bd4\u8f03 for\u6587\u3000vs\u3000iPython.parallel"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"start = time.time()\n",
"\n",
"for i in range(8):\n",
" time.sleep(1)\n",
"\n",
"elapsed_time = time.time() - start\n",
"print(elapsed_time)\n",
"\n"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"8.02717399597168\n"
]
}
],
"prompt_number": 11
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"start = time.time()\n",
"\n",
"res = cluster.map(lambda x: time.sleep(1), range(8)).get()\n",
"\n",
"elapsed_time = time.time() - start\n",
"print(elapsed_time)\n"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"1.0132977962493896\n"
]
}
],
"prompt_number": 12
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## \u308f\u304b\u3061\u304c\u304d\n",
"## [\u691c\u8a3c\uff12]\u901f\u5ea6\u6bd4\u8f03 \u30b3\u30f3\u30d7\u30ea\u30d8\u30f3\u30b7\u30e7\u30f3 vs iPython.parallel"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"\n",
"start = time.time()\n",
"\n",
"parsed = [wakati(line) for line in text]\n",
"\n",
"elapsed_time = time.time() - start\n",
"print(elapsed_time)\n",
"\n"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"1.438910961151123\n"
]
}
],
"prompt_number": 18
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"cluster.execute('''\n",
"import MeCab\n",
"t = MeCab.Tagger(\"-Owakati\")\n",
"''')"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 19,
"text": [
"<AsyncResult: execute>"
]
}
],
"prompt_number": 19
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"\n",
"start = time.time()\n",
"\n",
"res = cluster.map(wakati, text)\n",
"parsed = res.get()\n",
"\n",
"elapsed_time = time.time() - start\n",
"print(elapsed_time)\n"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"0.40604114532470703\n"
]
}
],
"prompt_number": 20
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"parsed[:5]"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 16,
"text": [
"['\u671d \u3001 \u98df\u5802 \u3067 \u30b9\u30a6\u30d7 \u3092 \u4e00 \u3055\u3058 \u3001 \u3059\u3063\u3068 \u5438\u3063 \u3066 \u304a\u6bcd\u3055\u307e \u304c \u3001',\n",
" '\u300c \u3042 \u300d',\n",
" '\u3068 \u5e7d \u304b\u3059\u304b \u306a \u53eb\u3073\u58f0 \u3092 \u304a \u6319\u3052 \u306b \u306a\u3063 \u305f \u3002',\n",
" '\u300c \u9aea\u306e\u6bdb \uff1f \u300d',\n",
" '\u30b9\u30a6\u30d7 \u306b \u4f55 \u304b \u3001 \u30a4\u30e4 \u306a \u3082\u306e \u3067\u3082 \u5165\u3063 \u3066 \u3044 \u305f \u306e \u304b\u3057\u3089 \u3001 \u3068 \u601d\u3063 \u305f \u3002']"
]
}
],
"prompt_number": 16
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# \u4e26\u5217\u5316\u3068\u5909\u6570\u306e\u30b9\u30b3\u30fc\u30d7"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"a = 1\n",
"b = 2\n",
"\n",
"cluster.map(lambda x :(a + b), range(10)).get()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"ename": "CompositeError",
"evalue": "one or more exceptions from call to method: <lambda>\n[0:apply]: NameError: name 'a' is not defined\n[1:apply]: NameError: name 'a' is not defined\n[2:apply]: NameError: name 'a' is not defined\n[3:apply]: NameError: name 'a' is not defined\n.... 4 more exceptions ...",
"output_type": "pyerr",
"traceback": [
"[0:apply]: ",
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)\u001b[0;32m<string>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m",
"\u001b[0;32m/usr/local/python3/lib/python3.4/site-packages/IPython/parallel/client/remotefunction.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *sequences)\u001b[0m",
"\u001b[1;32m 232\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mapping\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[1;32m 233\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mversion_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0;32m--> 234\u001b[0;31m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0msequences\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0msequences\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0m\u001b[1;32m 235\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[1;32m 236\u001b[0m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0;32m<ipython-input-33-38d136022dd7>\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m",
"\u001b[0;31mNameError\u001b[0m: name 'a' is not defined",
"",
"[1:apply]: ",
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)\u001b[0;32m<string>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m",
"\u001b[0;32m/usr/local/python3/lib/python3.4/site-packages/IPython/parallel/client/remotefunction.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *sequences)\u001b[0m",
"\u001b[1;32m 232\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mapping\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[1;32m 233\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mversion_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0;32m--> 234\u001b[0;31m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0msequences\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0msequences\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0m\u001b[1;32m 235\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[1;32m 236\u001b[0m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0;32m<ipython-input-33-38d136022dd7>\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m",
"\u001b[0;31mNameError\u001b[0m: name 'a' is not defined",
"",
"[2:apply]: ",
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)\u001b[0;32m<string>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m",
"\u001b[0;32m/usr/local/python3/lib/python3.4/site-packages/IPython/parallel/client/remotefunction.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *sequences)\u001b[0m",
"\u001b[1;32m 232\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mapping\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[1;32m 233\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mversion_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0;32m--> 234\u001b[0;31m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0msequences\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0msequences\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0m\u001b[1;32m 235\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[1;32m 236\u001b[0m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0;32m<ipython-input-33-38d136022dd7>\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m",
"\u001b[0;31mNameError\u001b[0m: name 'a' is not defined",
"",
"[3:apply]: ",
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)\u001b[0;32m<string>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m",
"\u001b[0;32m/usr/local/python3/lib/python3.4/site-packages/IPython/parallel/client/remotefunction.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *sequences)\u001b[0m",
"\u001b[1;32m 232\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_mapping\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[1;32m 233\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mversion_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0;32m--> 234\u001b[0;31m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0msequences\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0msequences\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0m\u001b[1;32m 235\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[1;32m 236\u001b[0m \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m",
"\u001b[0;32m<ipython-input-33-38d136022dd7>\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(x)\u001b[0m",
"\u001b[0;31mNameError\u001b[0m: name 'a' is not defined",
"",
"... 4 more exceptions ..."
]
}
],
"prompt_number": 33
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"a = 1\n",
"b = 2\n",
"\n",
"cluster[\"a\"] = a\n",
"cluster[\"b\"] = b\n",
"\n",
"cluster.map(lambda x :(a + b), range(10)).get()"
],
"language": "python",
"metadata": {},
"outputs": [
{
"metadata": {},
"output_type": "pyout",
"prompt_number": 35,
"text": [
"[3, 3, 3, 3, 3, 3, 3, 3, 3, 3]"
]
}
],
"prompt_number": 35
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"http://zokupata.connpass.com/event/11948/"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# \u3069\u3046\u4f7f\u3046\u306e\u304b\u306f\u8cb4\u65b9\u6b21\u7b2c"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment