-
-
Save huitseeker/a868af0dd8064cfe9806f4974a955386 to your computer and use it in GitHub Desktop.
Note book associated to the presentation from http://www.meetup.com/Big-Data-Romandie/events/230345605/ How to test normality assumptions from RandomRDD in Spark
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "metadata" : { | |
| "name" : "UpdateStateBykey", | |
| "user_save_timestamp" : "1970-01-01T01:00:00.000Z", | |
| "auto_save_timestamp" : "1970-01-01T01:00:00.000Z", | |
| "language_info" : { | |
| "name" : "scala", | |
| "file_extension" : "scala", | |
| "codemirror_mode" : "text/x-scala" | |
| }, | |
| "trusted" : true, | |
| "customLocalRepo" : null, | |
| "customRepos" : null, | |
| "customDeps" : null, | |
| "customImports" : null, | |
| "customArgs" : null, | |
| "customSparkConf" : { | |
| "spark.app.name" : "Notebook", | |
| "spark.master" : "local[*]", | |
| "spark.executor.memory" : "1G" | |
| } | |
| }, | |
| "cells" : [ { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "presentation" : { | |
| "tabs_state" : "{\n \"tab_id\": \"#tab1296412684-0\"\n}", | |
| "pivot_chart_state" : "{\n \"hiddenAttributes\": [],\n \"menuLimit\": 200,\n \"cols\": [],\n \"rows\": [],\n \"vals\": [],\n \"exclusions\": {},\n \"inclusions\": {},\n \"unusedAttrsVertical\": 85,\n \"autoSortUnusedAttrs\": false,\n \"inclusionsInfo\": {},\n \"aggregatorName\": \"Count\",\n \"rendererName\": \"Table\"\n}" | |
| }, | |
| "id" : "E57BD84023EA48BE9A765C22A9C95DD8" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "List(1,2,3).scanLeft(0){ case (accumulator, y) => accumulator + y }", | |
| "outputs" : [ { | |
| "name" : "stdout", | |
| "output_type" : "stream", | |
| "text" : "res8: List[Int] = List(0, 1, 3, 6)\n" | |
| }, { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "<div>\n <script data-this=\"{"dataId":"anon09f6c054288977875b24fc0b291ac9d9","dataInit":[],"genId":"1296412684"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/tabs'], \n function(playground, _magictabs) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magictabs,\n \"o\": {}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <div>\n <ul class=\"nav nav-tabs\" id=\"ul1296412684\"><li>\n <a href=\"#tab1296412684-0\"><i class=\"fa fa-table\"/></a>\n </li><li>\n <a href=\"#tab1296412684-1\"><i class=\"fa fa-dot-circle-o\"/></a>\n </li><li>\n <a href=\"#tab1296412684-2\"><i class=\"fa fa-line-chart\"/></a>\n </li><li>\n <a href=\"#tab1296412684-3\"><i class=\"fa fa-bar-chart\"/></a>\n </li><li>\n <a href=\"#tab1296412684-4\"><i class=\"fa fa-cubes\"/></a>\n </li></ul>\n\n <div class=\"tab-content\" id=\"tab1296412684\"><div class=\"tab-pane\" id=\"tab1296412684-0\">\n <div>\n <script data-this=\"{"dataId":"anon4c6b22c9e3a39aaecf0d0ef560d2ad32","dataInit":[{"_1":0,"_2":0},{"_1":1,"_2":1},{"_1":2,"_2":3},{"_1":3,"_2":6}],"genId":"1286233958"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/tableChart'], \n function(playground, _magictableChart) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magictableChart,\n \"o\": {\"headers\":[\"_1\",\"_2\"],\"width\":600,\"height\":400}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <span class=\"chart-total-item-count\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anona306252400f1ceca5ca8511542f07da8","initialValue":"4"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p> entries total</span>\n <span class=\"chart-sampling-warning\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anon9d088266f33f2f90840f2e68414341f0","initialValue":""}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p></span>\n <div>\n </div>\n </div></div>\n </div><div class=\"tab-pane\" id=\"tab1296412684-1\">\n <div>\n <script data-this=\"{"dataId":"anon9ee993d706bd90230ab2e129e2b64fb9","dataInit":[{"_1":0,"_2":0},{"_1":1,"_2":1},{"_1":2,"_2":3},{"_1":3,"_2":6}],"genId":"758796434"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/scatterChart'], \n function(playground, _magicscatterChart) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magicscatterChart,\n \"o\": {\"x\":\"_1\",\"y\":\"_2\",\"width\":600,\"height\":400}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <span class=\"chart-total-item-count\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anon922d37664f8d197bae7fcb2cc26fad88","initialValue":"4"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p> entries total</span>\n <span class=\"chart-sampling-warning\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anone600d6c22b2b034b16d9a0b48fdceea7","initialValue":""}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p></span>\n <div>\n </div>\n </div></div>\n </div><div class=\"tab-pane\" id=\"tab1296412684-2\">\n <div>\n <script data-this=\"{"dataId":"anoneaf45a03c0d0bc618ab05707e9b5e1c1","dataInit":[{"_1":0,"_2":0},{"_1":1,"_2":1},{"_1":2,"_2":3},{"_1":3,"_2":6}],"genId":"1572278797"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/lineChart'], \n function(playground, _magiclineChart) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magiclineChart,\n \"o\": {\"x\":\"_1\",\"y\":\"_2\",\"width\":600,\"height\":400}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <span class=\"chart-total-item-count\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anonfda7b8f77d23b2f6a7f7441f3acd71e3","initialValue":"4"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p> entries total</span>\n <span class=\"chart-sampling-warning\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anonfd66476235636ebc710370a703d7ba53","initialValue":""}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p></span>\n <div>\n </div>\n </div></div>\n </div><div class=\"tab-pane\" id=\"tab1296412684-3\">\n <div>\n <script data-this=\"{"dataId":"anonc52d5d1cf38f3fbda77728ff2ec06685","dataInit":[{"_1":0,"_2":0},{"_1":1,"_2":1},{"_1":2,"_2":3},{"_1":3,"_2":6}],"genId":"763633313"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/barChart'], \n function(playground, _magicbarChart) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magicbarChart,\n \"o\": {\"x\":\"_1\",\"y\":\"_2\",\"width\":600,\"height\":400}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <span class=\"chart-total-item-count\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anon6b806be305f05a365304af95d3b7d085","initialValue":"4"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p> entries total</span>\n <span class=\"chart-sampling-warning\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anon8885e8a2382cd3206fdc8dee78406970","initialValue":""}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p></span>\n <div>\n </div>\n </div></div>\n </div><div class=\"tab-pane\" id=\"tab1296412684-4\">\n <div>\n <script data-this=\"{"dataId":"anon1b424a5eea6608daa9572d3521288a98","dataInit":[{"_1":0,"_2":0},{"_1":1,"_2":1},{"_1":2,"_2":3},{"_1":3,"_2":6}],"genId":"240902220"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/pivotChart'], \n function(playground, _magicpivotChart) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magicpivotChart,\n \"o\": {\"width\":600,\"height\":400,\"derivedAttributes\":{},\"extraOptions\":{}}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <span class=\"chart-total-item-count\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anonfb9bfbd3d601045061ac92dc9769d5f7","initialValue":"4"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p> entries total</span>\n <span class=\"chart-sampling-warning\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anond22c8f2d1a310f0c4681fe39c4e305d1","initialValue":""}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p></span>\n <div>\n </div>\n </div></div>\n </div></div>\n </div>\n </div></div>" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 4, | |
| "time" : "Took: 1 second 53 milliseconds, at 2016-4-28 19:22" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "4D4434CB4119463F9A64575609BB4112" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "import org.apache.spark.streaming.{StreamingContext, Seconds}\n\n\n@transient val ssc = new StreamingContext(sparkContext, Seconds(5))\n//sparkContext.getConf.set(\"spark.streaming.checkpoint.directory\", \"file:///tmp/myTestNotebook\")\nssc.checkpoint(\"file:///tmp/myTestNotebook\")", | |
| "outputs" : [ { | |
| "name" : "stdout", | |
| "output_type" : "stream", | |
| "text" : "import org.apache.spark.streaming.{StreamingContext, Seconds}\nssc: org.apache.spark.streaming.StreamingContext = org.apache.spark.streaming.StreamingContext@465ea398\n" | |
| }, { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 5, | |
| "time" : "Took: 1 second 124 milliseconds, at 2016-4-28 19:45" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "A94EE68DA02B4D6E8B1CBA8338D96629" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "", | |
| "outputs" : [ { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 6, | |
| "time" : "Took: 529 milliseconds, at 2016-4-28 19:45" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "B81B1DB5D0DA4DF2A8DEE78BCB0A5BEA" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "import org.apache.spark.mllib.random.RandomRDDs._\ndef u(): RDD[Double] = normalRDD(ssc.sparkContext, 100000L, 10)", | |
| "outputs" : [ { | |
| "name" : "stdout", | |
| "output_type" : "stream", | |
| "text" : "import org.apache.spark.mllib.random.RandomRDDs._\nu: ()org.apache.spark.rdd.RDD[Double]\n" | |
| }, { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 7, | |
| "time" : "Took: 784 milliseconds, at 2016-4-28 19:46" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "presentation" : { | |
| "tabs_state" : "{\n \"tab_id\": \"#tab1471824747-0\"\n}", | |
| "pivot_chart_state" : "{\n \"hiddenAttributes\": [],\n \"menuLimit\": 200,\n \"cols\": [],\n \"rows\": [],\n \"vals\": [],\n \"exclusions\": {},\n \"inclusions\": {},\n \"unusedAttrsVertical\": 85,\n \"autoSortUnusedAttrs\": false,\n \"inclusionsInfo\": {},\n \"aggregatorName\": \"Count\",\n \"rendererName\": \"Table\"\n}" | |
| }, | |
| "id" : "533774C60CFF438982FB634A2989460C" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "u().take(5)", | |
| "outputs" : [ { | |
| "name" : "stdout", | |
| "output_type" : "stream", | |
| "text" : "res15: Array[Double] = Array(0.049278734204755884, 0.7943494262237427, -0.19549539852199105, -0.854604947280623, -0.4116935595133253)\n" | |
| }, { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "<div>\n <script data-this=\"{"dataId":"anoned9602520436e92c597a796b5e02b8ac","dataInit":[],"genId":"1471824747"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/tabs'], \n function(playground, _magictabs) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magictabs,\n \"o\": {}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <div>\n <ul class=\"nav nav-tabs\" id=\"ul1471824747\"><li>\n <a href=\"#tab1471824747-0\"><i class=\"fa fa-table\"/></a>\n </li><li>\n <a href=\"#tab1471824747-1\"><i class=\"fa fa-dot-circle-o\"/></a>\n </li><li>\n <a href=\"#tab1471824747-2\"><i class=\"fa fa-line-chart\"/></a>\n </li><li>\n <a href=\"#tab1471824747-3\"><i class=\"fa fa-bar-chart\"/></a>\n </li><li>\n <a href=\"#tab1471824747-4\"><i class=\"fa fa-cubes\"/></a>\n </li></ul>\n\n <div class=\"tab-content\" id=\"tab1471824747\"><div class=\"tab-pane\" id=\"tab1471824747-0\">\n <div>\n <script data-this=\"{"dataId":"anon8ce651d7de408658d2626c1603e2e97b","dataInit":[{"_1":0,"_2":0.049278734204755884},{"_1":1,"_2":0.7943494262237427},{"_1":2,"_2":-0.19549539852199105},{"_1":3,"_2":-0.854604947280623},{"_1":4,"_2":-0.4116935595133253}],"genId":"1725829341"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/tableChart'], \n function(playground, _magictableChart) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magictableChart,\n \"o\": {\"headers\":[\"_1\",\"_2\"],\"width\":600,\"height\":400}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <span class=\"chart-total-item-count\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anon6cdc7b0aaf01a49be7a3c8c671351a3a","initialValue":"5"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p> entries total</span>\n <span class=\"chart-sampling-warning\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anon47f2ccd00f42d81478c52c9e8b0634f7","initialValue":""}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p></span>\n <div>\n </div>\n </div></div>\n </div><div class=\"tab-pane\" id=\"tab1471824747-1\">\n <div>\n <script data-this=\"{"dataId":"anonae4437ae246491a92af7f500ca9b4e32","dataInit":[{"_1":0,"_2":0.049278734204755884},{"_1":1,"_2":0.7943494262237427},{"_1":2,"_2":-0.19549539852199105},{"_1":3,"_2":-0.854604947280623},{"_1":4,"_2":-0.4116935595133253}],"genId":"110853079"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/scatterChart'], \n function(playground, _magicscatterChart) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magicscatterChart,\n \"o\": {\"x\":\"_1\",\"y\":\"_2\",\"width\":600,\"height\":400}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <span class=\"chart-total-item-count\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anoned52e8e169f6e5ada5f59a5d1c78710d","initialValue":"5"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p> entries total</span>\n <span class=\"chart-sampling-warning\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anon8ea48a921296e0e98a723042d3fe6ef5","initialValue":""}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p></span>\n <div>\n </div>\n </div></div>\n </div><div class=\"tab-pane\" id=\"tab1471824747-2\">\n <div>\n <script data-this=\"{"dataId":"anone31a8dde622c986dec27161459fea197","dataInit":[{"_1":0,"_2":0.049278734204755884},{"_1":1,"_2":0.7943494262237427},{"_1":2,"_2":-0.19549539852199105},{"_1":3,"_2":-0.854604947280623},{"_1":4,"_2":-0.4116935595133253}],"genId":"1337569119"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/lineChart'], \n function(playground, _magiclineChart) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magiclineChart,\n \"o\": {\"x\":\"_1\",\"y\":\"_2\",\"width\":600,\"height\":400}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <span class=\"chart-total-item-count\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anon34e250bc3f1132802bb49dd1509f860a","initialValue":"5"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p> entries total</span>\n <span class=\"chart-sampling-warning\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anonddeb1fcda0e82c74a4ed3779473349fb","initialValue":""}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p></span>\n <div>\n </div>\n </div></div>\n </div><div class=\"tab-pane\" id=\"tab1471824747-3\">\n <div>\n <script data-this=\"{"dataId":"anone58aa1a5afd93db7a7d4cbaf65d95733","dataInit":[{"_1":0,"_2":0.049278734204755884},{"_1":1,"_2":0.7943494262237427},{"_1":2,"_2":-0.19549539852199105},{"_1":3,"_2":-0.854604947280623},{"_1":4,"_2":-0.4116935595133253}],"genId":"282958803"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/barChart'], \n function(playground, _magicbarChart) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magicbarChart,\n \"o\": {\"x\":\"_1\",\"y\":\"_2\",\"width\":600,\"height\":400}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <span class=\"chart-total-item-count\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anon986ac9ccfeaf419c27cb21bab7e70328","initialValue":"5"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p> entries total</span>\n <span class=\"chart-sampling-warning\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anon3c7aba80a5ed321b327097d18e5a74c3","initialValue":""}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p></span>\n <div>\n </div>\n </div></div>\n </div><div class=\"tab-pane\" id=\"tab1471824747-4\">\n <div>\n <script data-this=\"{"dataId":"anon91cc20d2de8576bb45d2b1c12af5f456","dataInit":[{"_1":0,"_2":0.049278734204755884},{"_1":1,"_2":0.7943494262237427},{"_1":2,"_2":-0.19549539852199105},{"_1":3,"_2":-0.854604947280623},{"_1":4,"_2":-0.4116935595133253}],"genId":"376036594"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/req(['../javascripts/notebook/playground','../javascripts/notebook/magic/pivotChart'], \n function(playground, _magicpivotChart) {\n // data ==> data-this (in observable.js's scopedEval) ==> this in JS => { dataId, dataInit, ... }\n // this ==> scope (in observable.js's scopedEval) ==> this.parentElement ==> div.container below (toHtml)\n\n playground.call(data,\n this\n ,\n {\n \"f\": _magicpivotChart,\n \"o\": {\"width\":600,\"height\":400,\"derivedAttributes\":{},\"extraOptions\":{}}\n }\n \n \n \n );\n }\n );/*]]>*/</script>\n <div>\n <span class=\"chart-total-item-count\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anonbd3bf02a6fd97418f3d3508eb7e7ddb6","initialValue":"5"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p> entries total</span>\n <span class=\"chart-sampling-warning\"><p data-bind=\"text: value\"><script data-this=\"{"valueId":"anonb69927e6243588239763c7d22951642d","initialValue":""}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId, initialValue)\n },\n this\n );\n});\n /*]]>*/</script></p></span>\n <div>\n </div>\n </div></div>\n </div></div>\n </div>\n </div></div>" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 8, | |
| "time" : "Took: 983 milliseconds, at 2016-4-28 19:47" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "622702ED1D5D451280F515203008D8E3" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "import org.apache.spark.rdd.RDD\n@transient val lines = scala.collection.mutable.Queue[RDD[Double]]()\n@transient val d = ssc.queueStream(lines)", | |
| "outputs" : [ { | |
| "name" : "stdout", | |
| "output_type" : "stream", | |
| "text" : "import org.apache.spark.rdd.RDD\nlines: scala.collection.mutable.Queue[org.apache.spark.rdd.RDD[Double]] = Queue()\nd: org.apache.spark.streaming.dstream.InputDStream[Double] = org.apache.spark.streaming.dstream.QueueInputDStream@73c095f\n" | |
| }, { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 9, | |
| "time" : "Took: 424 milliseconds, at 2016-4-28 19:48" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : true, | |
| "collapsed" : false, | |
| "id" : "5D664DEA2E254EE7A7316E0030529583" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "import notebook.front.widgets._\nimport notebook.front.DataConnectedWidget\nimport notebook.{JsonCodec, Codec}\nimport play.api.libs.json._\n\n\nclass A extends DataConnectedWidget[String] with Serializable {\n @transient implicit val singleCodec:Codec[JsValue, String] = JsonCodec.strings\n\n @transient val capacity = 10\n \n @transient val initData = Seq.empty[String]\n @transient val prefill:Option[String]=None\n \n @transient var data = (initData.size, prefill) match {\n case (0, None) => Seq.empty[String]\n case (x, None) => initData\n case (0, Some(i)) => Seq.fill(capacity)(i)\n case (x, Some(i)) => initData.padTo(capacity, i)\n }\n\n apply(data)\n\n @transient lazy val toHtml = <ul data-bind=\"foreach: value\">\n <li data-bind=\"html: $data\"></li>{\n scopedScript(\n \"\"\"\n |req(\n |['observable', 'knockout'],\n |function (O, ko) {\n | ko.applyBindings({\n | value: O.makeObservable(valueId)\n | },\n | this\n | );\n |});\n \"\"\"stripMargin,\n Json.obj(\"valueId\" -> dataConnection.id)\n )\n }</ul>\n\n override def apply(d:Seq[String]) {\n data = if (d.size > capacity) {\n d.drop(d.size - capacity)\n } else {\n d\n }\n super.apply(data)\n }\n\n def append(s:String) {\n apply(data :+ s)\n }\n\n def appendAll(s:Seq[String]) {\n apply(data ++ s)\n }\n }\n@transient val myUL = new A()", | |
| "outputs" : [ { | |
| "name" : "stdout", | |
| "output_type" : "stream", | |
| "text" : "<console>:73: warning: postfix operator stripMargin should be enabled\nby making the implicit value scala.language.postfixOps visible.\nThis can be achieved by adding the import clause 'import scala.language.postfixOps'\nor by setting the compiler option -language:postfixOps.\nSee the Scala docs for value scala.language.postfixOps for a discussion\nwhy the feature should be explicitly enabled.\n \"\"\"stripMargin,\n ^\nimport notebook.front.widgets._\nimport notebook.front.DataConnectedWidget\nimport notebook.{JsonCodec, Codec}\nimport play.api.libs.json._\ndefined class A\nmyUL: A = <A widget>\n" | |
| }, { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 10, | |
| "time" : "Took: 1 second 307 milliseconds, at 2016-4-28 19:48" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "90F77BF19F27457D8EADFBF4ADA7347F" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "myUL", | |
| "outputs" : [ { | |
| "name" : "stdout", | |
| "output_type" : "stream", | |
| "text" : "res19: A = <A widget>\n" | |
| }, { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "<ul data-bind=\"foreach: value\">\n <li data-bind=\"html: $data\"></li><script data-this=\"{"valueId":"anonf6278274778efc52f00eb09fb668d1e8"}\" type=\"text/x-scoped-javascript\">/*<![CDATA[*/\nreq(\n['observable', 'knockout'],\nfunction (O, ko) {\n ko.applyBindings({\n value: O.makeObservable(valueId)\n },\n this\n );\n});\n /*]]>*/</script></ul>" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 11, | |
| "time" : "Took: 535 milliseconds, at 2016-4-28 19:48" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "80CD84F6919D402D801765B2B6DDCFCA" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "myUL( Seq(\"c : 1\", \"d : 2\"))", | |
| "outputs" : [ { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 13, | |
| "time" : "Took: 567 milliseconds, at 2016-4-28 19:49" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "151D01224CE94DE5B0CD819B5CBA298F" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "val runningSums = d.map((dbl: Double) => (f\"$dbl%.2f\", 1)).updateStateByKey((newValues: Seq[Int], partialSum: Option[Int]) => {\n val currentSum = partialSum.getOrElse(0)\n Some(currentSum + newValues.sum)\n})\n", | |
| "outputs" : [ { | |
| "name" : "stdout", | |
| "output_type" : "stream", | |
| "text" : "runningSums: org.apache.spark.streaming.dstream.DStream[(String, Int)] = org.apache.spark.streaming.dstream.StateDStream@1bec65e3\n" | |
| }, { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 14, | |
| "time" : "Took: 637 milliseconds, at 2016-4-28 19:50" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "DA774D74BE9449329B7C162C4A73A794" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "runningSums.foreachRDD(rdd => {\n val size = (\"size\", rdd.count())\n val elems = rdd.collect()\n val sum = (\"sum\", elems.map(_._2).sum )\n val mean = (\"mean\", sum._2 / size._2 )\n val least = (\"least seen\", elems.minBy(_._2))\n val most = (\"most seen\", elems.maxBy(_._2))\n val r = List(size, sum, mean, least, most).map{case (key, count) => s\"$key: $count\"}\n myUL(r)\n})\n", | |
| "outputs" : [ { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 15, | |
| "time" : "Took: 744 milliseconds, at 2016-4-28 19:56" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "FA2D60D619E5404985C3E391DEE42A9D" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "ssc.start()", | |
| "outputs" : [ { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 16, | |
| "time" : "Took: 685 milliseconds, at 2016-4-28 19:57" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "F260D0AC81E6425C8C6B8F8AE7A5978E" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "// Create and push some RDDs into\nfor (i <- 1 to 30) {\n lines.synchronized {\n lines += u()\n }\n Thread.sleep(4)\n}", | |
| "outputs" : [ { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 17, | |
| "time" : "Took: 671 milliseconds, at 2016-4-28 19:57" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "0C4D4F76718944A98EACC21EDCBD1AB4" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "lines.size", | |
| "outputs" : [ { | |
| "name" : "stdout", | |
| "output_type" : "stream", | |
| "text" : "res32: Int = 18\n" | |
| }, { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "18" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 18, | |
| "time" : "Took: 753 milliseconds, at 2016-4-28 19:58" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : false, | |
| "id" : "5066DD69C53345F4B3B6789BA392B51A" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "ssc.stop()", | |
| "outputs" : [ { | |
| "metadata" : { }, | |
| "data" : { | |
| "text/html" : "" | |
| }, | |
| "output_type" : "execute_result", | |
| "execution_count" : 19, | |
| "time" : "Took: 615 milliseconds, at 2016-4-28 20:1" | |
| } ] | |
| }, { | |
| "metadata" : { | |
| "trusted" : true, | |
| "input_collapsed" : false, | |
| "collapsed" : true, | |
| "id" : "64596F0D7FC24E6D8E48872ACAE0FC20" | |
| }, | |
| "cell_type" : "code", | |
| "source" : "", | |
| "outputs" : [ ] | |
| } ], | |
| "nbformat" : 4 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment