Created
April 23, 2018 21:47
-
-
Save danielfrg/e63a2bc6ba5b872d56b71e1dbd9c9524 to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
{ | |
"cells": [ | |
{ | |
"cell_type": "code", | |
"execution_count": 1, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"data": { | |
"text/html": [ | |
"Current session configs: <tt>{'conf': {'spark.yarn.appMasterEnv.PYSPARK_PYTHON': '/opt/cloudera/parcels/bzrspark-0.1.0/bin/python', 'spark.yarn.appMasterEnv.PYSPARK_DRIVER_PYTHON': '/opt/cloudera/parcels/bzrspark-0.1.0/bin/python', 'spark.yarn.executorEnv.PYSPARK_PYTHON': '/opt/cloudera/parcels/bzrspark-0.1.0/bin/python', 'spark.pyspark.python': '/opt/cloudera/parcels/bzrspark-0.1.0/bin/python', 'spark.pyspark.driver.python': '/opt/cloudera/parcels/bzrspark-0.1.0/bin/python', 'spark.r.command': '/opt/cloudera/parcels/bzrspark-0.1.0/bin/R', 'spark.r.driver.command': '/opt/cloudera/parcels/bzrspark-0.1.0/bin/Rscript', 'spark.r.shell.command': '/opt/cloudera/parcels/bzrspark-0.1.0/bin/R'}, 'kind': 'sparkr'}</tt><br>" | |
], | |
"text/plain": [ | |
"<IPython.core.display.HTML object>" | |
] | |
}, | |
"metadata": {}, | |
"output_type": "display_data" | |
}, | |
{ | |
"data": { | |
"text/html": [ | |
"No active sessions." | |
], | |
"text/plain": [ | |
"<IPython.core.display.HTML object>" | |
] | |
}, | |
"metadata": {}, | |
"output_type": "display_data" | |
} | |
], | |
"source": [ | |
"%%configure -f\n", | |
"{\"conf\": {\"spark.yarn.appMasterEnv.PYSPARK_PYTHON\": \"/opt/cloudera/parcels/bzrspark-0.1.0/bin/python\",\n", | |
" \"spark.yarn.appMasterEnv.PYSPARK_DRIVER_PYTHON\": \"/opt/cloudera/parcels/bzrspark-0.1.0/bin/python\",\n", | |
" \"spark.yarn.executorEnv.PYSPARK_PYTHON\": \"/opt/cloudera/parcels/bzrspark-0.1.0/bin/python\",\n", | |
" \"spark.pyspark.python\": \"/opt/cloudera/parcels/bzrspark-0.1.0/bin/python\",\n", | |
" \"spark.pyspark.driver.python\": \"/opt/cloudera/parcels/bzrspark-0.1.0/bin/python\",\n", | |
" \"spark.r.command\": \"/opt/cloudera/parcels/bzrspark-0.1.0/bin/R\",\n", | |
" \"spark.r.driver.command\": \"/opt/cloudera/parcels/bzrspark-0.1.0/bin/Rscript\",\n", | |
" \"spark.r.shell.command\" : \"/opt/cloudera/parcels/bzrspark-0.1.0/bin/R\"\n", | |
" }\n", | |
"}" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 2, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"Starting Spark application\n" | |
] | |
}, | |
{ | |
"data": { | |
"text/html": [ | |
"<table>\n", | |
"<tr><th>ID</th><th>YARN Application ID</th><th>Kind</th><th>State</th><th>Spark UI</th><th>Driver log</th><th>Current session?</th></tr><tr><td>2</td><td>application_1519680446611_0031</td><td>sparkr</td><td>idle</td><td><a target=\"_blank\" href=\"http://ip-172-31-10-169.ec2.internal:8088/proxy/application_1519680446611_0031/\">Link</a></td><td><a target=\"_blank\" href=\"http://ip-172-31-7-137.ec2.internal:8042/node/containerlogs/container_1519680446611_0031_01_000001/centos\">Link</a></td><td>✔</td></tr></table>" | |
], | |
"text/plain": [ | |
"<IPython.core.display.HTML object>" | |
] | |
}, | |
"metadata": {}, | |
"output_type": "display_data" | |
}, | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"SparkContext available as 'sc'.\n", | |
"SqlContext available as 'sqlContext'.\n", | |
"Java ref type org.apache.spark.api.java.JavaSparkContext id 0 " | |
] | |
} | |
], | |
"source": [ | |
"sc" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 3, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"[1] 2" | |
] | |
} | |
], | |
"source": [ | |
"1+1" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 4, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"[1] \"ip-172-31-7-137.ec2.internal, /yarn/nm/usercache/centos/filecache/76/sparkr.zip, /opt/cloudera/parcels/bzrspark-0.1.0/lib/R/library\"" | |
] | |
} | |
], | |
"source": [ | |
"noop <- function(x) {\n", | |
" path <- toString(.libPaths())\n", | |
" host <- toString(Sys.info()['nodename'])\n", | |
" host_path <- toString(cbind(host,path))\n", | |
" host_path\n", | |
"}\n", | |
"noop(1)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 5, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"[1] \"/yarn/nm/usercache/centos/filecache/76/sparkr.zip, /opt/cloudera/parcels/bzrspark-0.1.0/lib/R/library\"" | |
] | |
} | |
], | |
"source": [ | |
"toString(.libPaths())" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 6, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [ | |
"library(sparklyr)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 7, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"\n", | |
"Attaching package: ‘dplyr’\n", | |
"\n", | |
"The following objects are masked from ‘package:SparkR’:\n", | |
"\n", | |
" arrange, between, collect, contains, count, cume_dist, dense_rank,\n", | |
" desc, distinct, explain, filter, first, group_by, intersect, lag,\n", | |
" last, lead, mutate, n, n_distinct, ntile, percent_rank, rename,\n", | |
" row_number, sample_frac, select, sql, summarize\n", | |
"\n", | |
"The following objects are masked from ‘package:stats’:\n", | |
"\n", | |
" filter, lag\n", | |
"\n", | |
"The following objects are masked from ‘package:base’:\n", | |
"\n", | |
" intersect, setdiff, setequal, union\n" | |
] | |
} | |
], | |
"source": [ | |
"library(dplyr)\n" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": 10, | |
"metadata": { | |
"collapsed": false | |
}, | |
"outputs": [ | |
{ | |
"name": "stdout", | |
"output_type": "stream", | |
"text": [ | |
"[1] \"Error in library(ggplot2): there is no package called ‘ggplot2’\\n----LIVY_END_OF_ERROR----\"\n", | |
"Warning messages:\n", | |
"1: In png(\"/yarn/nm/usercache/centos/appcache/application_1519680446611_0031/container_1519680446611_0031_01_000001/tmp/5603796457509368919.png\") :\n", | |
" unable to load shared object '/opt/cloudera/parcels/bzrspark-0.1.0/lib/R/library/grDevices/libs//cairo.so':\n", | |
" libXrender.so.1: cannot open shared object file: No such file or directory\n", | |
"2: In png(\"/yarn/nm/usercache/centos/appcache/application_1519680446611_0031/container_1519680446611_0031_01_000001/tmp/5603796457509368919.png\") :\n", | |
" failed to load cairo DLL" | |
] | |
} | |
], | |
"source": [ | |
"library(ggplot2)" | |
] | |
}, | |
{ | |
"cell_type": "code", | |
"execution_count": null, | |
"metadata": { | |
"collapsed": true | |
}, | |
"outputs": [], | |
"source": [] | |
} | |
], | |
"metadata": { | |
"kernelspec": { | |
"display_name": "Python 2", | |
"language": "python", | |
"name": "python2" | |
}, | |
"language_info": { | |
"codemirror_mode": { | |
"name": "ipython", | |
"version": 2 | |
}, | |
"file_extension": ".py", | |
"mimetype": "text/x-python", | |
"name": "python", | |
"nbconvert_exporter": "python", | |
"pygments_lexer": "ipython2", | |
"version": "2.7.11" | |
} | |
}, | |
"nbformat": 4, | |
"nbformat_minor": 2 | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment