Last active
December 6, 2021 00:35
-
-
Save rtempleton/1ae550e490b666c81b2e3a932ff9a8de to your computer and use it in GitHub Desktop.
Zeppelin Notebook for use in Hortonworks IOT workshop
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "paragraphs": [ | |
| { | |
| "text": "%md\n### Run simple queries to check the count of both the iotdata and iotrollup tables.\nClick the play button within each of the paragraphs to rerun/update the content\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:03:40-0400", | |
| "config": { | |
| "tableHide": false, | |
| "editorSetting": { | |
| "language": "markdown", | |
| "editOnDblClick": true, | |
| "completionKey": "TAB", | |
| "completionSupport": false | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/markdown", | |
| "fontSize": 9, | |
| "editorHide": true, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575214_20159231", | |
| "id": "20180913-162552_592167779", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:03:41-0400", | |
| "dateFinished": "2018-10-05T19:03:41-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "focus": true, | |
| "$$hashKey": "object:18738" | |
| }, | |
| { | |
| "text": "%jdbc(phoenix)\nselect count(*) from iotrollup as rollup_records;", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:03:41-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "sql", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 6, | |
| "editorMode": "ace/mode/sql", | |
| "fontSize": 9, | |
| "editorHide": false, | |
| "results": { | |
| "0": { | |
| "graph": { | |
| "mode": "table", | |
| "height": 98, | |
| "optionOpen": false, | |
| "setting": { | |
| "table": { | |
| "tableGridState": {}, | |
| "tableColumnTypeState": { | |
| "names": { | |
| "COUNT(1)": "string" | |
| }, | |
| "updated": false | |
| }, | |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", | |
| "tableOptionValue": { | |
| "useFilter": false, | |
| "showPagination": false, | |
| "showAggregationFooter": false | |
| }, | |
| "updated": false, | |
| "initialized": false | |
| } | |
| }, | |
| "commonSetting": {} | |
| } | |
| }, | |
| "1": { | |
| "graph": { | |
| "mode": "table", | |
| "height": 114, | |
| "optionOpen": false, | |
| "setting": { | |
| "table": { | |
| "tableGridState": {}, | |
| "tableColumnTypeState": { | |
| "names": { | |
| "COUNT(1)": "string" | |
| }, | |
| "updated": false | |
| }, | |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", | |
| "tableOptionValue": { | |
| "useFilter": false, | |
| "showPagination": false, | |
| "showAggregationFooter": false | |
| }, | |
| "updated": false, | |
| "initialized": false | |
| } | |
| }, | |
| "commonSetting": {} | |
| } | |
| } | |
| }, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575216_1063679957", | |
| "id": "20180912-185008_1297885400", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:03:41-0400", | |
| "dateFinished": "2018-10-05T19:03:41-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18739" | |
| }, | |
| { | |
| "text": "%jdbc(phoenix)\nselect count(*) from iotdata as rollup_records;\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:03:41-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "sql", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 6, | |
| "editorMode": "ace/mode/sql", | |
| "fontSize": 9, | |
| "editorHide": true, | |
| "results": { | |
| "0": { | |
| "graph": { | |
| "mode": "table", | |
| "height": 98, | |
| "optionOpen": false, | |
| "setting": { | |
| "table": { | |
| "tableGridState": {}, | |
| "tableColumnTypeState": { | |
| "names": { | |
| "COUNT(1)": "string" | |
| }, | |
| "updated": false | |
| }, | |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", | |
| "tableOptionValue": { | |
| "useFilter": false, | |
| "showPagination": false, | |
| "showAggregationFooter": false | |
| }, | |
| "updated": false, | |
| "initialized": false | |
| } | |
| }, | |
| "commonSetting": {} | |
| } | |
| } | |
| }, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575216_-788297054", | |
| "id": "20180913-125056_1399369949", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:03:42-0400", | |
| "dateFinished": "2018-10-05T19:03:43-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18740" | |
| }, | |
| { | |
| "text": "%md\n### Query the rollup table and visualize readings from Random7 and Random8 tags over the last 6 hours\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:03:43-0400", | |
| "config": { | |
| "tableHide": false, | |
| "editorSetting": { | |
| "language": "markdown", | |
| "editOnDblClick": true, | |
| "completionKey": "TAB", | |
| "completionSupport": false | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/markdown", | |
| "fontSize": 9, | |
| "editorHide": true, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575216_2004081939", | |
| "id": "20180913-131407_1132797990", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:03:43-0400", | |
| "dateFinished": "2018-10-05T19:03:43-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18741" | |
| }, | |
| { | |
| "text": "%jdbc(phoenix)\nselect tag, bucket, val_avg, val_stddev from iotrollup r, iottag t where bucket >= timemachine(to_timestamp(to_char(now())), 60, cast(-6 as INTEGER)) and tag in ('Random7','Random8') and r.tagid = t.tagid", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:04:59-0400", | |
| "config": { | |
| "tableHide": false, | |
| "editorSetting": { | |
| "language": "sql", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/sql", | |
| "fontSize": 9, | |
| "editorHide": false, | |
| "results": { | |
| "0": { | |
| "graph": { | |
| "mode": "lineChart", | |
| "height": 300, | |
| "optionOpen": false, | |
| "setting": { | |
| "table": { | |
| "tableGridState": {}, | |
| "tableColumnTypeState": { | |
| "names": { | |
| "TAG": "string", | |
| "BUCKET": "string", | |
| "VAL_AVG": "string", | |
| "VAL_STDDEV": "string" | |
| }, | |
| "updated": false | |
| }, | |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", | |
| "tableOptionValue": { | |
| "useFilter": false, | |
| "showPagination": false, | |
| "showAggregationFooter": false | |
| }, | |
| "updated": false, | |
| "initialized": false | |
| }, | |
| "lineChart": { | |
| "rotate": { | |
| "degree": "-45" | |
| }, | |
| "xLabelStatus": "hide", | |
| "forceY": true, | |
| "isDateFormat": true, | |
| "lineWithFocus": false | |
| } | |
| }, | |
| "commonSetting": {}, | |
| "keys": [ | |
| { | |
| "name": "BUCKET", | |
| "index": 1, | |
| "aggr": "sum" | |
| } | |
| ], | |
| "groups": [ | |
| { | |
| "name": "TAG", | |
| "index": 0, | |
| "aggr": "sum" | |
| } | |
| ], | |
| "values": [ | |
| { | |
| "name": "VAL_AVG", | |
| "index": 2, | |
| "aggr": "sum" | |
| }, | |
| { | |
| "name": "VAL_STDDEV", | |
| "index": 3, | |
| "aggr": "sum" | |
| } | |
| ] | |
| }, | |
| "helium": {} | |
| } | |
| }, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575217_1080341470", | |
| "id": "20180912-190028_103864504", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:03:44-0400", | |
| "dateFinished": "2018-10-05T19:03:44-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18742" | |
| }, | |
| { | |
| "text": "%md\n### Drill into details of Random7 over the last 30 minutes.\nUse the Zoom bar below the results to zoom in and scrub the results back and forth", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:03:44-0400", | |
| "config": { | |
| "tableHide": false, | |
| "editorSetting": { | |
| "language": "markdown", | |
| "editOnDblClick": true, | |
| "completionKey": "TAB", | |
| "completionSupport": false | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/markdown", | |
| "fontSize": 9, | |
| "editorHide": false, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575217_112027173", | |
| "id": "20180913-133518_331397953", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:03:44-0400", | |
| "dateFinished": "2018-10-05T19:03:44-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18743" | |
| }, | |
| { | |
| "text": "%jdbc(phoenix)\nselect tag, bucket, val_avg, val_stddev \nfrom iotrollup d\ninner join iottag t on t.tagid = d.tagid\nwhere bucket >= timemachine(to_timestamp(to_char(now())), 60, cast(-6 as INTEGER)) and tag = 'Random7'", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:05:00-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "sql", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/sql", | |
| "fontSize": 9, | |
| "editorHide": false, | |
| "results": { | |
| "0": { | |
| "graph": { | |
| "mode": "lineChart", | |
| "height": 300, | |
| "optionOpen": false, | |
| "setting": { | |
| "table": { | |
| "tableGridState": {}, | |
| "tableColumnTypeState": { | |
| "names": { | |
| "TAG": "string", | |
| "ts": "string", | |
| "VAL": "string" | |
| }, | |
| "updated": false | |
| }, | |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", | |
| "tableOptionValue": { | |
| "useFilter": false, | |
| "showPagination": false, | |
| "showAggregationFooter": false | |
| }, | |
| "updated": false, | |
| "initialized": false | |
| }, | |
| "lineChart": { | |
| "rotate": { | |
| "degree": "-45" | |
| }, | |
| "xLabelStatus": "default", | |
| "isDateFormat": false, | |
| "forceY": true, | |
| "lineWithFocus": true | |
| } | |
| }, | |
| "commonSetting": {}, | |
| "keys": [], | |
| "groups": [ | |
| { | |
| "name": "TAG", | |
| "index": 0, | |
| "aggr": "sum" | |
| } | |
| ], | |
| "values": [] | |
| }, | |
| "helium": {} | |
| } | |
| }, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575218_714303461", | |
| "id": "20180913-133506_130833614", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:03:45-0400", | |
| "dateFinished": "2018-10-05T19:03:45-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18744" | |
| }, | |
| { | |
| "text": "%md\n### Use Spark to let data scientists connect to the data and being exploring, looking for correlations, etc...\nBegin by creating a dataframe which leverages the sql context to load directly from pheonix.\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:03:45-0400", | |
| "config": { | |
| "tableHide": false, | |
| "editorSetting": { | |
| "language": "markdown", | |
| "editOnDblClick": true, | |
| "completionKey": "TAB", | |
| "completionSupport": false | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/markdown", | |
| "fontSize": 9, | |
| "editorHide": true, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575219_702917396", | |
| "id": "20180913-143829_2065687106", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:03:47-0400", | |
| "dateFinished": "2018-10-05T19:03:47-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18745" | |
| }, | |
| { | |
| "text": "%spark\nimport org.apache.spark.SparkContext\nimport org.apache.spark.sql.SQLContext\nimport org.apache.phoenix.spark._\n\nval sqlContext = new SQLContext(sc)\n\n//note we load the whole table here\nval df = sqlContext.load(\n \"org.apache.phoenix.spark\", \n Map(\"table\" -> \"IOTDATA\", \"zkUrl\" -> \"localhost:2181\")\n)\n\n//here we only view a subset of the data\ndf.filter(df(\"tagid\")===\"User2\").select(\"tagid\",\"val\", \"ts\").show\n\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:03:47-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "scala", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/scala", | |
| "fontSize": 9, | |
| "editorHide": false, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575219_64338595", | |
| "id": "20180912-191311_236699153", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:03:47-0400", | |
| "dateFinished": "2018-10-05T19:03:49-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18746" | |
| }, | |
| { | |
| "text": "%md\n####Look at the summary of our data and examine some statistical aspects\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:03:49-0400", | |
| "config": { | |
| "tableHide": false, | |
| "editorSetting": { | |
| "language": "markdown", | |
| "editOnDblClick": true, | |
| "completionKey": "TAB", | |
| "completionSupport": false | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/markdown", | |
| "fontSize": 9, | |
| "editorHide": true, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575220_-1255720051", | |
| "id": "20180914-094622_1941538171", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:03:50-0400", | |
| "dateFinished": "2018-10-05T19:03:50-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18747" | |
| }, | |
| { | |
| "text": "%spark\n\ndf.filter(df(\"tagid\")===\"112\").describe(\"val\").show()\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:18:15-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "scala", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/scala", | |
| "fontSize": 9, | |
| "results": {}, | |
| "enabled": true, | |
| "editorHide": false | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "runtimeInfos": { | |
| "jobUrl": { | |
| "propertyName": "jobUrl", | |
| "label": "SPARK JOB", | |
| "tooltip": "View in Spark web UI", | |
| "group": "spark", | |
| "values": [ | |
| "http://demo.hortonworks.com:4040/jobs/job?id=17" | |
| ], | |
| "interpreterSettingId": "spark2" | |
| } | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575220_1692650422", | |
| "id": "20180914-094450_2101463490", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:18:15-0400", | |
| "dateFinished": "2018-10-05T19:18:18-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18748" | |
| }, | |
| { | |
| "text": "%md\n\n###Outlier Detection\n####Examine the variability of the data\n####Calculate the first and third quartile values and the IQR (Interquartile Range) which is the \"spread\" or difference between the third and first quartiles. \n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:04:17-0400", | |
| "config": { | |
| "tableHide": false, | |
| "editorSetting": { | |
| "language": "markdown", | |
| "editOnDblClick": true, | |
| "completionKey": "TAB", | |
| "completionSupport": false | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/markdown", | |
| "fontSize": 9, | |
| "editorHide": true, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575222_-980146959", | |
| "id": "20180914-093625_1587907826", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:04:17-0400", | |
| "dateFinished": "2018-10-05T19:04:18-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18749" | |
| }, | |
| { | |
| "text": "%spark\nval quartiles = df.stat.approxQuantile(\"val\",\n Array(0.25,0.75),0.0)\nval Q1 = quartiles(0)\nval Q3 = quartiles(1)\nval IQR = Q3 - Q1\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:04:18-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "scala", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/scala", | |
| "fontSize": 9, | |
| "results": {}, | |
| "enabled": true, | |
| "editorHide": false | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "runtimeInfos": { | |
| "jobUrl": { | |
| "propertyName": "jobUrl", | |
| "label": "SPARK JOB", | |
| "tooltip": "View in Spark web UI", | |
| "group": "spark", | |
| "values": [ | |
| "http://demo.hortonworks.com:4040/jobs/job?id=8" | |
| ], | |
| "interpreterSettingId": "spark2" | |
| } | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575222_-894723382", | |
| "id": "20180914-092831_1497371958", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:04:18-0400", | |
| "dateFinished": "2018-10-05T19:04:38-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18750" | |
| }, | |
| { | |
| "text": "%md\n####Now calculate the ranges. Then filter the data using data frame filters.\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:04:38-0400", | |
| "config": { | |
| "tableHide": false, | |
| "editorSetting": { | |
| "language": "markdown", | |
| "editOnDblClick": true, | |
| "completionKey": "TAB", | |
| "completionSupport": false | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/markdown", | |
| "fontSize": 9, | |
| "editorHide": true, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575223_-51913206", | |
| "id": "20180914-093318_1189241828", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:04:38-0400", | |
| "dateFinished": "2018-10-05T19:04:38-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18751" | |
| }, | |
| { | |
| "text": "%spark\nval lowerRange = Q1 - 1.5*IQR\nval upperRange = Q3+ 1.5*IQR\n\nval outliers = df.filter(s\"val < $lowerRange or val > $upperRange\")\noutliers.show()", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:04:38-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "scala", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/scala", | |
| "fontSize": 9, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "runtimeInfos": { | |
| "jobUrl": { | |
| "propertyName": "jobUrl", | |
| "label": "SPARK JOB", | |
| "tooltip": "View in Spark web UI", | |
| "group": "spark", | |
| "values": [ | |
| "http://demo.hortonworks.com:4040/jobs/job?id=9" | |
| ], | |
| "interpreterSettingId": "spark2" | |
| } | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575223_877978994", | |
| "id": "20180914-092851_297687867", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:04:39-0400", | |
| "dateFinished": "2018-10-05T19:04:41-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18752" | |
| }, | |
| { | |
| "text": "%spark\noutliers.select(\"TS\", \"VAL\").show()", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:04:41-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "scala", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/scala", | |
| "fontSize": 9, | |
| "results": {}, | |
| "enabled": true, | |
| "editorHide": false | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "runtimeInfos": { | |
| "jobUrl": { | |
| "propertyName": "jobUrl", | |
| "label": "SPARK JOB", | |
| "tooltip": "View in Spark web UI", | |
| "group": "spark", | |
| "values": [ | |
| "http://demo.hortonworks.com:4040/jobs/job?id=10" | |
| ], | |
| "interpreterSettingId": "spark2" | |
| } | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575224_-514228838", | |
| "id": "20180914-093103_1660198976", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:04:41-0400", | |
| "dateFinished": "2018-10-05T19:04:42-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18753" | |
| }, | |
| { | |
| "text": "%md\n### Python Method\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:04:43-0400", | |
| "config": { | |
| "tableHide": false, | |
| "editorSetting": { | |
| "language": "markdown", | |
| "editOnDblClick": true, | |
| "completionKey": "TAB", | |
| "completionSupport": false | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/markdown", | |
| "fontSize": 9, | |
| "editorHide": true, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575224_157540119", | |
| "id": "20180914-103445_977837612", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:04:43-0400", | |
| "dateFinished": "2018-10-05T19:04:43-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18754" | |
| }, | |
| { | |
| "text": "%pyspark\n\ndf = sqlContext.read.format(\"org.apache.phoenix.spark\").option(\"table\", \"IOTDATA\").option(\"zkUrl\", \"localhost:2181e\").load()\ndf.show()\n\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:04:43-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "python", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/python", | |
| "fontSize": 9, | |
| "results": {}, | |
| "enabled": true, | |
| "editorHide": false | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "runtimeInfos": { | |
| "jobUrl": { | |
| "propertyName": "jobUrl", | |
| "label": "SPARK JOB", | |
| "tooltip": "View in Spark web UI", | |
| "group": "spark", | |
| "values": [ | |
| "http://demo.hortonworks.com:4040/jobs/job?id=11" | |
| ], | |
| "interpreterSettingId": "spark2" | |
| } | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575225_-1132601176", | |
| "id": "20180914-102819_756250805", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:04:43-0400", | |
| "dateFinished": "2018-10-05T19:04:44-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18755" | |
| }, | |
| { | |
| "text": "%pyspark\ndf.describe().show()", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-05T19:04:44-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "python", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/python", | |
| "fontSize": 9, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "runtimeInfos": { | |
| "jobUrl": { | |
| "propertyName": "jobUrl", | |
| "label": "SPARK JOB", | |
| "tooltip": "View in Spark web UI", | |
| "group": "spark", | |
| "values": [ | |
| "http://demo.hortonworks.com:4040/jobs/job?id=12" | |
| ], | |
| "interpreterSettingId": "spark2" | |
| } | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575225_-1441956018", | |
| "id": "20180914-103148_1567218585", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-05T19:04:44-0400", | |
| "dateFinished": "2018-10-05T19:04:58-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18756" | |
| }, | |
| { | |
| "text": "%pyspark\nquartiles = df.approxQuantile(\"VAL\", [0.25, 0.75], 0.00)\n\nQ1 = quartiles[0]\nQ3 = quartiles[1]\nIQR = Q3 - Q1\n\nlowerRange = Q1 - 1.5*IQR\nupperRange = Q3+ 1.5*IQR\n\nprint(Q1, Q3, IQR, lowerRange, upperRange)", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-06T13:08:39-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "python", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/python", | |
| "fontSize": 9, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "runtimeInfos": { | |
| "jobUrl": { | |
| "propertyName": "jobUrl", | |
| "label": "SPARK JOB", | |
| "tooltip": "View in Spark web UI", | |
| "group": "spark", | |
| "values": [ | |
| "http://demo.hortonworks.com:4040/jobs/job?id=18" | |
| ], | |
| "interpreterSettingId": "spark2" | |
| } | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575226_-2012718816", | |
| "id": "20180914-104434_1987892281", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-06T13:08:40-0400", | |
| "dateFinished": "2018-10-06T13:09:00-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18757" | |
| }, | |
| { | |
| "text": "%pyspark\n\noutliers = df.filter((df.VAL < lowerRange) | (df.VAL > upperRange))\noutliers.show()", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-06T13:08:44-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "python", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/python", | |
| "fontSize": 9, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "runtimeInfos": { | |
| "jobUrl": { | |
| "propertyName": "jobUrl", | |
| "label": "SPARK JOB", | |
| "tooltip": "View in Spark web UI", | |
| "group": "spark", | |
| "values": [ | |
| "http://demo.hortonworks.com:4040/jobs/job?id=19" | |
| ], | |
| "interpreterSettingId": "spark2" | |
| } | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575227_-867899665", | |
| "id": "20180914-103953_84968504", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-06T13:08:45-0400", | |
| "dateFinished": "2018-10-06T13:09:00-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18758" | |
| }, | |
| { | |
| "text": "%pyspark\noutliers.describe().show()", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-06T13:08:52-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "python", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/python", | |
| "fontSize": 9, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "runtimeInfos": { | |
| "jobUrl": { | |
| "propertyName": "jobUrl", | |
| "label": "SPARK JOB", | |
| "tooltip": "View in Spark web UI", | |
| "group": "spark", | |
| "values": [ | |
| "http://demo.hortonworks.com:4040/jobs/job?id=20" | |
| ], | |
| "interpreterSettingId": "spark2" | |
| } | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575227_-198930696", | |
| "id": "20180914-112733_298758573", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-06T13:09:00-0400", | |
| "dateFinished": "2018-10-06T13:09:08-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18759" | |
| }, | |
| { | |
| "text": "%sql\nselect TS, VAL, TAGID from outliers\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-06T13:12:56-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "sql", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/sql", | |
| "fontSize": 9, | |
| "results": { | |
| "0": { | |
| "graph": { | |
| "mode": "lineChart", | |
| "height": 300, | |
| "optionOpen": false, | |
| "setting": { | |
| "table": { | |
| "tableGridState": {}, | |
| "tableColumnTypeState": { | |
| "names": { | |
| "TS": "string", | |
| "VAL": "string" | |
| }, | |
| "updated": false | |
| }, | |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", | |
| "tableOptionValue": { | |
| "useFilter": false, | |
| "showPagination": false, | |
| "showAggregationFooter": false | |
| }, | |
| "updated": false, | |
| "initialized": false | |
| }, | |
| "multiBarChart": { | |
| "rotate": { | |
| "degree": "-45" | |
| }, | |
| "xLabelStatus": "default" | |
| }, | |
| "stackedAreaChart": { | |
| "rotate": { | |
| "degree": "-45" | |
| }, | |
| "xLabelStatus": "default", | |
| "style": "stream" | |
| }, | |
| "lineChart": { | |
| "xLabelStatus": "default", | |
| "rotate": { | |
| "degree": "-45" | |
| }, | |
| "forceY": true, | |
| "isDateFormat": true, | |
| "lineWithFocus": true | |
| }, | |
| "pieChart": {} | |
| }, | |
| "commonSetting": {}, | |
| "keys": [ | |
| { | |
| "name": "TS", | |
| "index": 0, | |
| "aggr": "sum" | |
| } | |
| ], | |
| "groups": [], | |
| "values": [ | |
| { | |
| "name": "VAL", | |
| "index": 1, | |
| "aggr": "sum" | |
| } | |
| ] | |
| }, | |
| "helium": {} | |
| } | |
| }, | |
| "enabled": true, | |
| "editorHide": false | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "runtimeInfos": { | |
| "jobUrl": { | |
| "propertyName": "jobUrl", | |
| "label": "SPARK JOB", | |
| "tooltip": "View in Spark web UI", | |
| "group": "spark", | |
| "values": [ | |
| "http://demo.hortonworks.com:4040/jobs/job?id=21" | |
| ], | |
| "interpreterSettingId": "spark2" | |
| } | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575229_294963071", | |
| "id": "20180914-105904_1042135454", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-06T13:12:54-0400", | |
| "dateFinished": "2018-10-06T13:12:55-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18760" | |
| }, | |
| { | |
| "text": "%sql\nselect TS, VAL, TAGID from sampleOutliers", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-06T13:13:34-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "sql", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/sql", | |
| "fontSize": 9, | |
| "results": { | |
| "0": { | |
| "graph": { | |
| "mode": "lineChart", | |
| "height": 300, | |
| "optionOpen": false, | |
| "setting": { | |
| "table": { | |
| "tableGridState": {}, | |
| "tableColumnTypeState": { | |
| "names": { | |
| "TS": "string", | |
| "VAL": "string" | |
| }, | |
| "updated": false | |
| }, | |
| "tableOptionSpecHash": "[{\"name\":\"useFilter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable filter for columns\"},{\"name\":\"showPagination\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable pagination for better navigation\"},{\"name\":\"showAggregationFooter\",\"valueType\":\"boolean\",\"defaultValue\":false,\"widget\":\"checkbox\",\"description\":\"Enable a footer for displaying aggregated values\"}]", | |
| "tableOptionValue": { | |
| "useFilter": false, | |
| "showPagination": false, | |
| "showAggregationFooter": false | |
| }, | |
| "updated": false, | |
| "initialized": false | |
| }, | |
| "lineChart": { | |
| "rotate": { | |
| "degree": "-45" | |
| }, | |
| "xLabelStatus": "default", | |
| "forceY": true, | |
| "isDateFormat": true, | |
| "lineWithFocus": true | |
| } | |
| }, | |
| "commonSetting": {}, | |
| "keys": [ | |
| { | |
| "name": "TS", | |
| "index": 0, | |
| "aggr": "sum" | |
| } | |
| ], | |
| "groups": [], | |
| "values": [ | |
| { | |
| "name": "VAL", | |
| "index": 1, | |
| "aggr": "sum" | |
| } | |
| ] | |
| }, | |
| "helium": {} | |
| } | |
| }, | |
| "enabled": true, | |
| "editorHide": false | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "runtimeInfos": { | |
| "jobUrl": { | |
| "propertyName": "jobUrl", | |
| "label": "SPARK JOB", | |
| "tooltip": "View in Spark web UI", | |
| "group": "spark", | |
| "values": [ | |
| "http://demo.hortonworks.com:4040/jobs/job?id=22", | |
| "http://demo.hortonworks.com:4040/jobs/job?id=23" | |
| ], | |
| "interpreterSettingId": "spark2" | |
| } | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575229_-1629613433", | |
| "id": "20180914-111832_568295790", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "dateStarted": "2018-10-06T13:13:27-0400", | |
| "dateFinished": "2018-10-06T13:13:30-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18761" | |
| }, | |
| { | |
| "text": "%pyspark\n", | |
| "user": "admin", | |
| "dateUpdated": "2018-10-06T13:10:40-0400", | |
| "config": { | |
| "editorSetting": { | |
| "language": "python", | |
| "editOnDblClick": false, | |
| "completionKey": "TAB", | |
| "completionSupport": true | |
| }, | |
| "colWidth": 12, | |
| "editorMode": "ace/mode/python", | |
| "fontSize": 9, | |
| "results": {}, | |
| "enabled": true | |
| }, | |
| "settings": { | |
| "params": {}, | |
| "forms": {} | |
| }, | |
| "apps": [], | |
| "jobName": "paragraph_1538776575230_1177044147", | |
| "id": "20180914-114602_1186665316", | |
| "dateCreated": "2018-10-05T17:56:15-0400", | |
| "status": "FINISHED", | |
| "errorMessage": "", | |
| "progressUpdateIntervalMs": 500, | |
| "$$hashKey": "object:18762" | |
| } | |
| ], | |
| "name": "IOTWorkshop", | |
| "id": "2DTMXSVQ8", | |
| "noteParams": {}, | |
| "noteForms": {}, | |
| "angularObjects": { | |
| "md:shared_process": [], | |
| "jdbc:shared_process": [], | |
| "spark2:shared_process": [] | |
| }, | |
| "config": { | |
| "isZeppelinNotebookCronEnable": false, | |
| "looknfeel": "default", | |
| "personalizedMode": "false" | |
| }, | |
| "info": {} | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment