paladini/data.json Secret

## data.json
{"values": [[1359788300000, 13.2], [1359788400000, 123], [1359788410000, 23.1]], "name": "teste"}
{"values": [[1438123834470, 2.4599999999999937], [1438123838470, 2], [1438123875470, 2.4200000000000017], [1438123879470, 2], [1438123883470, 2], [1438123919470, 2.469999999999999], [1438123922470, 2], [1438123926470, 2.469999999999999], [1438123930470, 2], [1438123944470, 2.489999999999995], [1438123948470, 2], [1438123982470, 2.4399999999999977], [1438123986470, 2], [1438123990470, 2.4399999999999977], [1438123994470, 2], [1438123998470, 2.4399999999999977], [1438124002470, 2], [1438124024470, 69], [1438124036470, 59.51], [1438124040470, 59], [1438124044470, 60.04], [1438124048470, 60], [1438124053470, 59], [1438124057470, 71.42], [1438124062470, 71], [1438124066470, 46], [1438183430470, 70.35], [1438183435470, 70], [1438183440470, 66.49000000000001], [1438183445470, 66], [1438183450470, 62.4], [1438183454470, 62], [1438183487470, 65], [1438183502470, 64], [1438183517470, 64.77000000000001], [1438183522470, 64], [1438183540470, 64.3], [1438183545470, 64], [1438183550470, 64.18], [1438183554470, 64], [1438183563470, 65.02000000000001], [1438183567470, 65], [1438183581470, 64], [1438183586470, 65], [1438183591470, 64], [1438183604470, 66], [1438183613470, 65.19], [1438183617470, 65], [1438183636470, 65.50999999999999], [1438183641470, 65], [1438183646470, 65], [1438183674470, 65.22999999999999], [1438183683470, 65], [1438183701470, 65.05], [1438183705470, 65], [1438183710470, 62], [1438183720470, 22], [1438257607470, 50], [1438257774470, 83], [1438257778470, 67], [1438257787470, 84], [1438257791470, 77.98], [1438257796470, 77], [1438257800470, 71], [1438257804470, 73.23], [1438257812470, 73], [1438257816470, 74], [1438257821470, 64], [1438257825470, 74], [1438257829470, 64], [1438257833470, 71], [1438257837470, 81.53999999999999], [1438257841470, 81], [1438257845470, 69], [1438257849470, 73], [1438257853470, 71], [1438257857470, 72], [1438257865470, 71.53999999999999], [1438257869470, 71], [1438257873470, 71.84], [1438257877470, 71], [1438257893470, 72], [1438257898470, 72.35], [1438257902470, 72.12], [1438257906470, 72], [1438257910470, 72.37], [1438429513078, 50], [1438893821992, 10], [1438893832646, 90], [1438893845375, 50], [1438899340797, 65], [1438950743353, 50], [1438951909470, 65], [1438965857473, 1], [1439207914353, 50], [1439207934218, 65], [1439293245353, 50], [1439293265261, 65], [1439301212353, 50], [1439301291725, 0], [1439302099645, 65], [1439326483908, 50], [1439381831055, 65], [1439399048852, 50], [1439404120476, 54], [1439404122152, 65], [1439404163852, 50], [1439404248488, 65], [1439412201852, 50], [1439465362677, 65], [1439465862852, 50], [1439465899700, 65], [1439466068852, 50], [1439466108800, 65], [1439479427852, 50], [1439479468022, 65], [1439483520852, 50], [1439483536852, 50], [1439483541498, 65], [1439483541852, 65], [1440360650991, 50], [1440367197236, 97], [1440367203236, 62], [1440367209236, 60], [1440367215236, 94], [1440367221236, 79], [1440367234236, 75], [1440367241236, 74.28], [1440367248236, 74], [1440367273236, 74.33], [1440367278236, 74], [1440367308236, 72], [1440367343236, 71.82], [1440367356236, 71], [1440367362236, 71.19], [1440367369236, 71], [1440367381236, 71.19], [1440367387236, 71], [1440367456236, 71.21], [1440367462236, 71], [1440367467236, 71.21], [1440367473236, 71], [1440367491236, 71.19], [1440367497236, 71.23], [1440367503236, 71], [1440367509236, 84.49], [1440367514236, 84], [1440367530236, 85], [1440367536236, 85.26], [1440367542236, 85], [1440367548236, 85.25], [1440367553236, 85.25], [1440367558236, 85], [1440367857236, 78.65], [1440367862236, 78], [1440367872236, 85], [1440367882236, 77.44], [1440367888236, 77], [1440367907236, 90.35], [1440367913236, 90], [1440367920236, 79.37], [1440367926236, 79], [1440367932236, 78], [1440367945236, 75], [1440367951236, 73.51], [1440367958236, 73], [1440367965236, 72.54], [1440367972236, 72], [1440367986236, 71.79], [1440367993236, 71.18], [1440367999236, 71.16], [1440368005236, 71], [1440368011236, 71.16], [1440368018236, 71.18], [1440368025236, 71.18], [1440368032236, 71], [1440368109236, 71.18], [1440368115236, 71], [1440368127236, 71.16], [1440368134236, 71], [1440368201236, 71.63], [1440368207236, 71.25], [1440368213236, 71], [1440368219236, 71.21], [1440368225236, 71], [1440368233236, 71.25], [1440368239236, 71.28], [1440368245236, 71], [1440368369236, 71.67], [1440369607171, 50], [1440369623171, 96.91], [1440369628171, 96], [1440369639171, 96.96], [1440369645171, 96.82], [1440369650171, 96.77], [1440369655171, 96], [1440369660171, 81], [1440369671171, 79], [1440369681171, 77.84], [1440369687171, 77], [1440369692171, 76], [1440369697171, 75.89], [1440369703171, 75.3], [1440369708171, 75], [1440369713171, 41], [1440369718171, 25], [1440369723171, 22], [1440369779171, 93.16], [1440369785171, 93], [1440369790171, 42.16], [1440369795171, 42], [1440369800171, 23], [1440369812171, 22], [1440369818171, 22.61], [1440369824171, 22], [1440369835171, 22.7], [1440369840171, 22], [1440369847171, 22], [1440369864171, 22.63], [1440369868171, 22], [1440369879171, 22.7], [1440369884171, 22], [1440369895171, 22.65], [1440369900171, 22], [1440369919171, 22.68], [1440369924171, 22], [1440369941171, 76.63], [1440369945171, 76], [1440369950171, 75], [1440369967171, 17], [1440369986171, 32.56], [1440369997171, 32], [1440370006171, 33], [1440370011171, 76], [1440370025171, 59], [1440370029171, 68.37], [1440370033171, 68], [1440370049171, 65], [1440370230171, 65.81], [1440370235171, 65.84], [1440370241171, 65], [1440370279171, 65.77], [1440370285171, 65.72], [1440370290171, 65], [1440370370171, 11], [1440370376171, 76], [1440370380171, 69.4], [1440370384171, 69], [1440370477171, 10.89], [1440370482171, 77.42], [1440370487171, 77], [1440370492171, 60], [1440370841171, 33.44], [1440370846171, 33], [1440370871171, 75], [1440370877171, 69.02], [1440370882171, 69], [1440371376171, 12.4], [1440371381171, 12], [1440371385171, 76.47], [1440371390171, 76], [1440371395171, 60], [1440371523171, 9.91], [1440371528171, 76.3], [1440371532171, 76], [1440371547171, 60], [1440374859044, 50], [1440416833212, 65], [1440525820046, 0], [1440525841163, 50], [1440525877973, 100], [1440525906223, 65], [1440529856044, 20], [1440530799252, 0], [1440531873984, 50], [1440535009276, 90], [1440535306306, 0], [1440604366938, 50], [1440638352580, 0], [1440638368580, 100], [1440638383194, 50], [1440638556474, 0], [1440638585285, 50], [1441214058065, 80.28], [1441214063065, 80], [1441214068065, 66], [1441222495065, 47.75], [1441222500065, 47], [1441222517065, 75.33], [1441222523065, 75], [1441222530065, 64], [1441222719065, 3.56], [1441222725065, 78.79], [1441222731065, 78], [1441223289065, 8.28], [1441223294065, 8], [1441223300065, 78.58], [1441223305065, 78], [1441223310065, 61], [1441223477065, 42.4], [1441223482065, 42], [1441223498065, 76], [1441223505065, 62], [1441225272065, 41], [1441225324065, 75], [1441225329065, 61], [1442318926725, 67.77], [1442318934725, 67], [1442514704759, 67.77], [1442514713759, 67], [1442700857152, 67.77], [1442700865152, 67], [1442701531152, 67.77], [1442701539152, 67], [1442775795847, 50], [1442775975257, 67.77], [1442775981257, 67], [1442776905239, 50], [1442780049803, 67], [1442780120803, 67.77], [1442780127803, 67], [1442780525177, 25], [1442780533877, 67], [1442780536577, 25], [1442780599589, 67], [1442780743627, 25], [1442780933509, 67.77], [1442780937009, 67], [1442781878337, 67.77], [1442781888337, 67], [1442782892051, 67], [1442782919769, 67.77], [1442782931051, 67], [1442783141646, 25], [1442783146315, 10], [1442783152770, 0], [1442783321211, 25], [1442783332110, 100], [1442783395949, 67.77], [1442783399949, 67], [1442783444067, 67.77], [1442783445949, 67], [1442786607492, 30.93], [1442786611492, 45.89], [1442786615492, 45], [1442786619492, 17], [1442786623492, 46.77], [1442786627492, 46], [1442786655492, 86.32], [1442786659492, 21.79], [1442786663492, 21], [1442786667492, 47], [1442787299492, 50], [1442787304492, 2.44], [1442787308492, 31.68], [1442787311492, 31], [1442787371492, 52.4], [1442787375492, 64.44], [1442787378492, 64], [1442787381492, 81], [1442787385492, 32.7], [1442787389492, 32], [1442787396492, 47.56], [1442787399492, 47], [1442787413492, 66.63], [1442787417492, 81.86], [1442787421492, 81], [1442787425492, 21.93], [1442787428492, 47.4], [1442787432492, 47], [1442787792492, 31.37], [1442787799492, 31], [1442790809518, 50], [1442793063768, 36.05], [1442793067768, 36], [1442860101910, 73], [1442860107910, 98.42], [1442860113410, 98], [1442860118910, 3], [1442860124910, 80], [1442860131410, 55], [1442860136910, 73], [1442866262254, 68.88], [1442866271254, 68], [1442869494754, 27.58], [1442869501254, 72.77], [1442869508254, 72], [1442869515754, 47.74], [1442869522754, 47], [1442869615754, 73.56], [1442869622754, 73], [1442869663254, 51.89], [1442869669754, 51], [1442869939254, 74.56], [1442869945754, 74], [1442869966754, 57.37], [1442869973254, 57], [1442869980254, 73.53], [1442869987254, 73], [1442870080254, 37.54], [1442870087254, 37], [1442870094754, 52.33], [1442870101254, 52], [1442870121754, 2.4], [1442870128254, 2], [1442870625254, 66.44], [1442870631754, 66], [1442870638254, 24], [1442870644754, 74.75], [1442870651254, 74], [1442871468254, 89.75], [1442871475254, 89], [1442883014728, 89.74], [1442883028228, 89], [1442948800228, 78], [1442948837728, 5.26], [1442948843728, 5], [1442948863728, 95.63], [1442948869228, 95], [1442949436228, 50]], "name": "DP_058424"}
{"values": [[1359788300000, 13.2], [1359788400000, 123], [1359788410000, 23.1]], "name": "teste_gzip"}
{"values": [[1440532091016, 153.55], [1440532096016, 152.98], [1440532101016, 153.58], [1440532107016, 153.76], [1440532113016, 154.04], [1440532119016, 153.79], [1440532125016, 152.41], [1440532130016, 152.52], [1440532135016, 152.43], [1440532141016, 151.76], [1440532147016, 153.4], [1440532152016, 153.28], [1440532157016, 152.77], [1440532163016, 151.53], [1440532169016, 151.5], [1440532174016, 152.22], [1440532180016, 152.87], [1440532186016, 151.5], [1440532192016, 149.2], [1440532197016, 149.93], [1440532203016, 149.33], [1440532208016, 151.32], [1440532214016, 150.6], [1440532219016, 151.66], [1440532225016, 152.34], [1440532231016, 151.46], [1440532236016, 149.92], [1440532242016, 150.01], [1440532247016, 147.59], [1440532253016, 147.81], [1440532258016, 148.96], [1440532264016, 150.44], [1440532269016, 150.45], [1440532275016, 152.14], [1440532281016, 152.25], [1440532287016, 150.48], [1440532292016, 150.35], [1440532297016, 151.14], [1440532302016, 152.63], [1440532308016, 153.34], [1440532313016, 153.36], [1440532319016, 152.18], [1440532325016, 151.02], [1440532331016, 149.94], [1440532337016, 151.46], [1440532342016, 152.52], [1440532348016, 154.92], [1440532354016, 155.6], [1440532360016, 155.62], [1440532366016, 155.76], [1440532371016, 163.25], [1440532377016, 167.33], [1440532382016, 160.1], [1440532388016, 155.67], [1440532394016, 140.1], [1440532400016, 139.69], [1440532405016, 135.49], [1440532411016, 131.95], [1440532416016, 130.16], [1440532422016, 128.82], [1440532427016, 128.31], [1440532432016, 127.68], [1440532437016, 127.15], [1440532442016, 125.88], [1440532447016, 125.8], [1440532452016, 125.97], [1440532457016, 125.65], [1440532463016, 125.97], [1440532468016, 125.75], [1440532474016, 125.95], [1440532480016, 124.17], [1440532486016, 123.66], [1440532492016, 123.38], [1440532498016, 123.55], [1440532504016, 121.63], [1440532509016, 123.08], [1440532515016, 123.61], [1440532521016, 123], [1440532526016, 121.94], [1440532532016, 121.83], [1440532537016, 121.38], [1440532543016, 121.6], [1440532548016, 121.66], [1440532554016, 122.66], [1440532559016, 121.65], [1440532565016, 121.67], [1440532571016, 121.96], [1440532576016, 121.98], [1440532582016, 121.52], [1440532587016, 122.67], [1440532592016, 123.11], [1440532597016, 123.1], [1440532603016, 123.3], [1440532608016, 123.78], [1440532613016, 124.1], [1440532619016, 124.27], [1440532625016, 123.33], [1440532631016, 123.69], [1440532637016, 123.9], [1440532642016, 122.86], [1440532648016, 121.84], [1440532654016, 122.73], [1440532660016, 123.32], [1440532665016, 123.6], [1440532671016, 124.55], [1440532676016, 124.61], [1440532681016, 124.04], [1440532687016, 123.76], [1440532693016, 123.43], [1440532699016, 123.87], [1440532705016, 123.83], [1440532710016, 124.02], [1440532715016, 123.62], [1440532720016, 123.31], [1440532726016, 123.7], [1440532731016, 124.02]], "name": "DP_063321"}

## reading_and_parsing_json_from_spark.py
# Starting Spark Context and DataFrames
sc = SparkContext("local", appName="JSON to DataFrame")
sqlContext = SQLContext(sc)
dataframe = sqlContext.read.json("data.json")
dataframe.show()

## spark_submit_dataframe_show.log
[paladini@starstuff smartx-fontaine]$ spark-submit spark/examples/loading_dataframe_from_kairos.py
Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
15/10/05 16:04:32 INFO SparkContext: Running Spark version 1.5.0
15/10/05 16:04:33 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
15/10/05 16:04:33 INFO SecurityManager: Changing view acls to: paladini
15/10/05 16:04:33 INFO SecurityManager: Changing modify acls to: paladini
15/10/05 16:04:33 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(paladini); users with modify permissions: Set(paladini)
15/10/05 16:04:34 INFO Slf4jLogger: Slf4jLogger started
15/10/05 16:04:34 INFO Remoting: Starting remoting
15/10/05 16:04:34 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriver@192.168.1.111:35680]
15/10/05 16:04:34 INFO Utils: Successfully started service 'sparkDriver' on port 35680.
15/10/05 16:04:34 INFO SparkEnv: Registering MapOutputTracker
15/10/05 16:04:34 INFO SparkEnv: Registering BlockManagerMaster
15/10/05 16:04:34 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-40087457-5737-4b1f-b821-77a4dd59e3a9
15/10/05 16:04:34 INFO MemoryStore: MemoryStore started with capacity 530.0 MB
15/10/05 16:04:34 INFO HttpFileServer: HTTP File server directory is /tmp/spark-fd210c43-0d33-4094-a669-1d477262040d/httpd-2fbdd9ed-7fb7-4bd3-843e-70d6cae3ea82
15/10/05 16:04:34 INFO HttpServer: Starting HTTP Server
15/10/05 16:04:34 INFO Utils: Successfully started service 'HTTP file server' on port 32822.
15/10/05 16:04:34 INFO SparkEnv: Registering OutputCommitCoordinator
15/10/05 16:04:34 INFO Utils: Successfully started service 'SparkUI' on port 4040.
15/10/05 16:04:34 INFO SparkUI: Started SparkUI at http://192.168.1.111:4040
15/10/05 16:04:34 INFO Utils: Copying /home/paladini/ufxc/lisha/smartx-fontaine/spark/examples/loading_dataframe_from_kairos.py to /tmp/spark-fd210c43-0d33-4094-a669-1d477262040d/userFiles-3a4ad2dc-6422-4ce4-8854-74e62817613b/loading_dataframe_from_kairos.py
15/10/05 16:04:34 INFO SparkContext: Added file file:/home/paladini/ufxc/lisha/smartx-fontaine/spark/examples/loading_dataframe_from_kairos.py at file:/home/paladini/ufxc/lisha/smartx-fontaine/spark/examples/loading_dataframe_from_kairos.py with timestamp 1444071874860
15/10/05 16:04:35 WARN MetricsSystem: Using default name DAGScheduler for source because spark.app.id is not set.
15/10/05 16:04:35 INFO Executor: Starting executor ID driver on host localhost
15/10/05 16:04:35 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 46437.
15/10/05 16:04:35 INFO NettyBlockTransferService: Server created on 46437
15/10/05 16:04:35 INFO BlockManagerMaster: Trying to register BlockManager
15/10/05 16:04:35 INFO BlockManagerMasterEndpoint: Registering block manager localhost:46437 with 530.0 MB RAM, BlockManagerId(driver, localhost, 46437)
15/10/05 16:04:35 INFO BlockManagerMaster: Registered BlockManager
15/10/05 16:04:36 INFO JSONRelation: Listing file:/home/paladini/ufxc/lisha/smartx-fontaine/data.json on driver
15/10/05 16:04:36 INFO MemoryStore: ensureFreeSpace(213992) called with curMem=0, maxMem=555755765
15/10/05 16:04:36 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 209.0 KB, free 529.8 MB)
15/10/05 16:04:36 INFO MemoryStore: ensureFreeSpace(19910) called with curMem=213992, maxMem=555755765
15/10/05 16:04:36 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 19.4 KB, free 529.8 MB)
15/10/05 16:04:36 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on localhost:46437 (size: 19.4 KB, free: 530.0 MB)
15/10/05 16:04:36 INFO SparkContext: Created broadcast 0 from json at NativeMethodAccessorImpl.java:-2
15/10/05 16:04:36 INFO FileInputFormat: Total input paths to process : 1
15/10/05 16:04:36 INFO SparkContext: Starting job: json at NativeMethodAccessorImpl.java:-2
15/10/05 16:04:36 INFO DAGScheduler: Got job 0 (json at NativeMethodAccessorImpl.java:-2) with 1 output partitions
15/10/05 16:04:36 INFO DAGScheduler: Final stage: ResultStage 0(json at NativeMethodAccessorImpl.java:-2)
15/10/05 16:04:36 INFO DAGScheduler: Parents of final stage: List()
15/10/05 16:04:36 INFO DAGScheduler: Missing parents: List()
15/10/05 16:04:36 INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[3] at json at NativeMethodAccessorImpl.java:-2), which has no missing parents
15/10/05 16:04:36 INFO MemoryStore: ensureFreeSpace(4368) called with curMem=233902, maxMem=555755765
15/10/05 16:04:36 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 529.8 MB)
15/10/05 16:04:36 INFO MemoryStore: ensureFreeSpace(2460) called with curMem=238270, maxMem=555755765
15/10/05 16:04:36 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 529.8 MB)
15/10/05 16:04:36 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on localhost:46437 (size: 2.4 KB, free: 530.0 MB)
15/10/05 16:04:36 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:861
15/10/05 16:04:36 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at json at NativeMethodAccessorImpl.java:-2)
15/10/05 16:04:36 INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks
15/10/05 16:04:36 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2269 bytes)
15/10/05 16:04:36 INFO Executor: Running task 0.0 in stage 0.0 (TID 0)
15/10/05 16:04:37 INFO Executor: Fetching file:/home/paladini/ufxc/lisha/smartx-fontaine/spark/examples/loading_dataframe_from_kairos.py with timestamp 1444071874860
15/10/05 16:04:37 INFO Utils: /home/paladini/ufxc/lisha/smartx-fontaine/spark/examples/loading_dataframe_from_kairos.py has been previously copied to /tmp/spark-fd210c43-0d33-4094-a669-1d477262040d/userFiles-3a4ad2dc-6422-4ce4-8854-74e62817613b/loading_dataframe_from_kairos.py
15/10/05 16:04:37 INFO HadoopRDD: Input split: file:/home/paladini/ufxc/lisha/smartx-fontaine/data.json:0+12565
15/10/05 16:04:37 INFO deprecation: mapred.tip.id is deprecated. Instead, use mapreduce.task.id
15/10/05 16:04:37 INFO deprecation: mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id
15/10/05 16:04:37 INFO deprecation: mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap
15/10/05 16:04:37 INFO deprecation: mapred.task.partition is deprecated. Instead, use mapreduce.task.partition
15/10/05 16:04:37 INFO deprecation: mapred.job.id is deprecated. Instead, use mapreduce.job.id
15/10/05 16:04:38 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 3706 bytes result sent to driver
15/10/05 16:04:38 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1076 ms on localhost (1/1)
15/10/05 16:04:38 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool
15/10/05 16:04:38 INFO DAGScheduler: ResultStage 0 (json at NativeMethodAccessorImpl.java:-2) finished in 1.099 s
15/10/05 16:04:38 INFO DAGScheduler: Job 0 finished: json at NativeMethodAccessorImpl.java:-2, took 1.177533 s
15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(63280) called with curMem=240730, maxMem=555755765
15/10/05 16:04:38 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 61.8 KB, free 529.7 MB)
15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(19788) called with curMem=304010, maxMem=555755765
15/10/05 16:04:38 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 19.3 KB, free 529.7 MB)
15/10/05 16:04:38 INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on localhost:46437 (size: 19.3 KB, free: 530.0 MB)
15/10/05 16:04:38 INFO SparkContext: Created broadcast 2 from showString at NativeMethodAccessorImpl.java:-2
15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(213992) called with curMem=323798, maxMem=555755765
15/10/05 16:04:38 INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 209.0 KB, free 529.5 MB)
15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(19910) called with curMem=537790, maxMem=555755765
15/10/05 16:04:38 INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 19.4 KB, free 529.5 MB)
15/10/05 16:04:38 INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on localhost:46437 (size: 19.4 KB, free: 530.0 MB)
15/10/05 16:04:38 INFO SparkContext: Created broadcast 3 from showString at NativeMethodAccessorImpl.java:-2
15/10/05 16:04:38 INFO FileInputFormat: Total input paths to process : 1
15/10/05 16:04:38 INFO SparkContext: Starting job: showString at NativeMethodAccessorImpl.java:-2
15/10/05 16:04:38 INFO DAGScheduler: Got job 1 (showString at NativeMethodAccessorImpl.java:-2) with 1 output partitions
15/10/05 16:04:38 INFO DAGScheduler: Final stage: ResultStage 1(showString at NativeMethodAccessorImpl.java:-2)
15/10/05 16:04:38 INFO DAGScheduler: Parents of final stage: List()
15/10/05 16:04:38 INFO DAGScheduler: Missing parents: List()
15/10/05 16:04:38 INFO DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[7] at showString at NativeMethodAccessorImpl.java:-2), which has no missing parents
15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(4800) called with curMem=557700, maxMem=555755765
15/10/05 16:04:38 INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 4.7 KB, free 529.5 MB)
15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(2715) called with curMem=562500, maxMem=555755765
15/10/05 16:04:38 INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.7 KB, free 529.5 MB)
15/10/05 16:04:38 INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on localhost:46437 (size: 2.7 KB, free: 529.9 MB)
15/10/05 16:04:38 INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:861
15/10/05 16:04:38 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at showString at NativeMethodAccessorImpl.java:-2)
15/10/05 16:04:38 INFO TaskSchedulerImpl: Adding task set 1.0 with 1 tasks
15/10/05 16:04:38 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2269 bytes)
15/10/05 16:04:38 INFO Executor: Running task 0.0 in stage 1.0 (TID 1)
15/10/05 16:04:38 INFO HadoopRDD: Input split: file:/home/paladini/ufxc/lisha/smartx-fontaine/data.json:0+12565
15/10/05 16:04:38 INFO Executor: Finished task 0.0 in stage 1.0 (TID 1). 26671 bytes result sent to driver
15/10/05 16:04:38 INFO DAGScheduler: ResultStage 1 (showString at NativeMethodAccessorImpl.java:-2) finished in 0.116 s
15/10/05 16:04:38 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 122 ms on localhost (1/1)
15/10/05 16:04:38 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool
15/10/05 16:04:38 INFO DAGScheduler: Job 1 finished: showString at NativeMethodAccessorImpl.java:-2, took 0.145138 s
+----------+--------------------+
|      name|              values|
+----------+--------------------+
|     teste|[WrappedArray(1.3...|
| DP_058424|[WrappedArray(1.4...|
|teste_gzip|[WrappedArray(1.3...|
| DP_063321|[WrappedArray(1.4...|
+----------+--------------------+

15/10/05 16:04:38 INFO SparkContext: Invoking stop() from shutdown hook
15/10/05 16:04:38 INFO SparkUI: Stopped Spark web UI at http://192.168.1.111:4040
15/10/05 16:04:38 INFO DAGScheduler: Stopping DAGScheduler
15/10/05 16:04:38 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
15/10/05 16:04:38 INFO MemoryStore: MemoryStore cleared
15/10/05 16:04:38 INFO BlockManager: BlockManager stopped
15/10/05 16:04:38 INFO BlockManagerMaster: BlockManagerMaster stopped
15/10/05 16:04:38 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
15/10/05 16:04:38 INFO SparkContext: Successfully stopped SparkContext
15/10/05 16:04:38 INFO ShutdownHookManager: Shutdown hook called
15/10/05 16:04:38 INFO ShutdownHookManager: Deleting directory /tmp/spark-fd210c43-0d33-4094-a669-1d477262040d
[paladini@starstuff smartx-fontaine]$
	{"values": [[1359788300000, 13.2], [1359788400000, 123], [1359788410000, 23.1]], "name": "teste"}
	{"values": [[1438123834470, 2.4599999999999937], [1438123838470, 2], [1438123875470, 2.4200000000000017], [1438123879470, 2], [1438123883470, 2], [1438123919470, 2.469999999999999], [1438123922470, 2], [1438123926470, 2.469999999999999], [1438123930470, 2], [1438123944470, 2.489999999999995], [1438123948470, 2], [1438123982470, 2.4399999999999977], [1438123986470, 2], [1438123990470, 2.4399999999999977], [1438123994470, 2], [1438123998470, 2.4399999999999977], [1438124002470, 2], [1438124024470, 69], [1438124036470, 59.51], [1438124040470, 59], [1438124044470, 60.04], [1438124048470, 60], [1438124053470, 59], [1438124057470, 71.42], [1438124062470, 71], [1438124066470, 46], [1438183430470, 70.35], [1438183435470, 70], [1438183440470, 66.49000000000001], [1438183445470, 66], [1438183450470, 62.4], [1438183454470, 62], [1438183487470, 65], [1438183502470, 64], [1438183517470, 64.77000000000001], [1438183522470, 64], [1438183540470, 64.3], [1438183545470, 64], [1438183550470, 64.18], [1438183554470, 64], [1438183563470, 65.02000000000001], [1438183567470, 65], [1438183581470, 64], [1438183586470, 65], [1438183591470, 64], [1438183604470, 66], [1438183613470, 65.19], [1438183617470, 65], [1438183636470, 65.50999999999999], [1438183641470, 65], [1438183646470, 65], [1438183674470, 65.22999999999999], [1438183683470, 65], [1438183701470, 65.05], [1438183705470, 65], [1438183710470, 62], [1438183720470, 22], [1438257607470, 50], [1438257774470, 83], [1438257778470, 67], [1438257787470, 84], [1438257791470, 77.98], [1438257796470, 77], [1438257800470, 71], [1438257804470, 73.23], [1438257812470, 73], [1438257816470, 74], [1438257821470, 64], [1438257825470, 74], [1438257829470, 64], [1438257833470, 71], [1438257837470, 81.53999999999999], [1438257841470, 81], [1438257845470, 69], [1438257849470, 73], [1438257853470, 71], [1438257857470, 72], [1438257865470, 71.53999999999999], [1438257869470, 71], [1438257873470, 71.84], [1438257877470, 71], [1438257893470, 72], [1438257898470, 72.35], [1438257902470, 72.12], [1438257906470, 72], [1438257910470, 72.37], [1438429513078, 50], [1438893821992, 10], [1438893832646, 90], [1438893845375, 50], [1438899340797, 65], [1438950743353, 50], [1438951909470, 65], [1438965857473, 1], [1439207914353, 50], [1439207934218, 65], [1439293245353, 50], [1439293265261, 65], [1439301212353, 50], [1439301291725, 0], [1439302099645, 65], [1439326483908, 50], [1439381831055, 65], [1439399048852, 50], [1439404120476, 54], [1439404122152, 65], [1439404163852, 50], [1439404248488, 65], [1439412201852, 50], [1439465362677, 65], [1439465862852, 50], [1439465899700, 65], [1439466068852, 50], [1439466108800, 65], [1439479427852, 50], [1439479468022, 65], [1439483520852, 50], [1439483536852, 50], [1439483541498, 65], [1439483541852, 65], [1440360650991, 50], [1440367197236, 97], [1440367203236, 62], [1440367209236, 60], [1440367215236, 94], [1440367221236, 79], [1440367234236, 75], [1440367241236, 74.28], [1440367248236, 74], [1440367273236, 74.33], [1440367278236, 74], [1440367308236, 72], [1440367343236, 71.82], [1440367356236, 71], [1440367362236, 71.19], [1440367369236, 71], [1440367381236, 71.19], [1440367387236, 71], [1440367456236, 71.21], [1440367462236, 71], [1440367467236, 71.21], [1440367473236, 71], [1440367491236, 71.19], [1440367497236, 71.23], [1440367503236, 71], [1440367509236, 84.49], [1440367514236, 84], [1440367530236, 85], [1440367536236, 85.26], [1440367542236, 85], [1440367548236, 85.25], [1440367553236, 85.25], [1440367558236, 85], [1440367857236, 78.65], [1440367862236, 78], [1440367872236, 85], [1440367882236, 77.44], [1440367888236, 77], [1440367907236, 90.35], [1440367913236, 90], [1440367920236, 79.37], [1440367926236, 79], [1440367932236, 78], [1440367945236, 75], [1440367951236, 73.51], [1440367958236, 73], [1440367965236, 72.54], [1440367972236, 72], [1440367986236, 71.79], [1440367993236, 71.18], [1440367999236, 71.16], [1440368005236, 71], [1440368011236, 71.16], [1440368018236, 71.18], [1440368025236, 71.18], [1440368032236, 71], [1440368109236, 71.18], [1440368115236, 71], [1440368127236, 71.16], [1440368134236, 71], [1440368201236, 71.63], [1440368207236, 71.25], [1440368213236, 71], [1440368219236, 71.21], [1440368225236, 71], [1440368233236, 71.25], [1440368239236, 71.28], [1440368245236, 71], [1440368369236, 71.67], [1440369607171, 50], [1440369623171, 96.91], [1440369628171, 96], [1440369639171, 96.96], [1440369645171, 96.82], [1440369650171, 96.77], [1440369655171, 96], [1440369660171, 81], [1440369671171, 79], [1440369681171, 77.84], [1440369687171, 77], [1440369692171, 76], [1440369697171, 75.89], [1440369703171, 75.3], [1440369708171, 75], [1440369713171, 41], [1440369718171, 25], [1440369723171, 22], [1440369779171, 93.16], [1440369785171, 93], [1440369790171, 42.16], [1440369795171, 42], [1440369800171, 23], [1440369812171, 22], [1440369818171, 22.61], [1440369824171, 22], [1440369835171, 22.7], [1440369840171, 22], [1440369847171, 22], [1440369864171, 22.63], [1440369868171, 22], [1440369879171, 22.7], [1440369884171, 22], [1440369895171, 22.65], [1440369900171, 22], [1440369919171, 22.68], [1440369924171, 22], [1440369941171, 76.63], [1440369945171, 76], [1440369950171, 75], [1440369967171, 17], [1440369986171, 32.56], [1440369997171, 32], [1440370006171, 33], [1440370011171, 76], [1440370025171, 59], [1440370029171, 68.37], [1440370033171, 68], [1440370049171, 65], [1440370230171, 65.81], [1440370235171, 65.84], [1440370241171, 65], [1440370279171, 65.77], [1440370285171, 65.72], [1440370290171, 65], [1440370370171, 11], [1440370376171, 76], [1440370380171, 69.4], [1440370384171, 69], [1440370477171, 10.89], [1440370482171, 77.42], [1440370487171, 77], [1440370492171, 60], [1440370841171, 33.44], [1440370846171, 33], [1440370871171, 75], [1440370877171, 69.02], [1440370882171, 69], [1440371376171, 12.4], [1440371381171, 12], [1440371385171, 76.47], [1440371390171, 76], [1440371395171, 60], [1440371523171, 9.91], [1440371528171, 76.3], [1440371532171, 76], [1440371547171, 60], [1440374859044, 50], [1440416833212, 65], [1440525820046, 0], [1440525841163, 50], [1440525877973, 100], [1440525906223, 65], [1440529856044, 20], [1440530799252, 0], [1440531873984, 50], [1440535009276, 90], [1440535306306, 0], [1440604366938, 50], [1440638352580, 0], [1440638368580, 100], [1440638383194, 50], [1440638556474, 0], [1440638585285, 50], [1441214058065, 80.28], [1441214063065, 80], [1441214068065, 66], [1441222495065, 47.75], [1441222500065, 47], [1441222517065, 75.33], [1441222523065, 75], [1441222530065, 64], [1441222719065, 3.56], [1441222725065, 78.79], [1441222731065, 78], [1441223289065, 8.28], [1441223294065, 8], [1441223300065, 78.58], [1441223305065, 78], [1441223310065, 61], [1441223477065, 42.4], [1441223482065, 42], [1441223498065, 76], [1441223505065, 62], [1441225272065, 41], [1441225324065, 75], [1441225329065, 61], [1442318926725, 67.77], [1442318934725, 67], [1442514704759, 67.77], [1442514713759, 67], [1442700857152, 67.77], [1442700865152, 67], [1442701531152, 67.77], [1442701539152, 67], [1442775795847, 50], [1442775975257, 67.77], [1442775981257, 67], [1442776905239, 50], [1442780049803, 67], [1442780120803, 67.77], [1442780127803, 67], [1442780525177, 25], [1442780533877, 67], [1442780536577, 25], [1442780599589, 67], [1442780743627, 25], [1442780933509, 67.77], [1442780937009, 67], [1442781878337, 67.77], [1442781888337, 67], [1442782892051, 67], [1442782919769, 67.77], [1442782931051, 67], [1442783141646, 25], [1442783146315, 10], [1442783152770, 0], [1442783321211, 25], [1442783332110, 100], [1442783395949, 67.77], [1442783399949, 67], [1442783444067, 67.77], [1442783445949, 67], [1442786607492, 30.93], [1442786611492, 45.89], [1442786615492, 45], [1442786619492, 17], [1442786623492, 46.77], [1442786627492, 46], [1442786655492, 86.32], [1442786659492, 21.79], [1442786663492, 21], [1442786667492, 47], [1442787299492, 50], [1442787304492, 2.44], [1442787308492, 31.68], [1442787311492, 31], [1442787371492, 52.4], [1442787375492, 64.44], [1442787378492, 64], [1442787381492, 81], [1442787385492, 32.7], [1442787389492, 32], [1442787396492, 47.56], [1442787399492, 47], [1442787413492, 66.63], [1442787417492, 81.86], [1442787421492, 81], [1442787425492, 21.93], [1442787428492, 47.4], [1442787432492, 47], [1442787792492, 31.37], [1442787799492, 31], [1442790809518, 50], [1442793063768, 36.05], [1442793067768, 36], [1442860101910, 73], [1442860107910, 98.42], [1442860113410, 98], [1442860118910, 3], [1442860124910, 80], [1442860131410, 55], [1442860136910, 73], [1442866262254, 68.88], [1442866271254, 68], [1442869494754, 27.58], [1442869501254, 72.77], [1442869508254, 72], [1442869515754, 47.74], [1442869522754, 47], [1442869615754, 73.56], [1442869622754, 73], [1442869663254, 51.89], [1442869669754, 51], [1442869939254, 74.56], [1442869945754, 74], [1442869966754, 57.37], [1442869973254, 57], [1442869980254, 73.53], [1442869987254, 73], [1442870080254, 37.54], [1442870087254, 37], [1442870094754, 52.33], [1442870101254, 52], [1442870121754, 2.4], [1442870128254, 2], [1442870625254, 66.44], [1442870631754, 66], [1442870638254, 24], [1442870644754, 74.75], [1442870651254, 74], [1442871468254, 89.75], [1442871475254, 89], [1442883014728, 89.74], [1442883028228, 89], [1442948800228, 78], [1442948837728, 5.26], [1442948843728, 5], [1442948863728, 95.63], [1442948869228, 95], [1442949436228, 50]], "name": "DP_058424"}
	{"values": [[1359788300000, 13.2], [1359788400000, 123], [1359788410000, 23.1]], "name": "teste_gzip"}
	{"values": [[1440532091016, 153.55], [1440532096016, 152.98], [1440532101016, 153.58], [1440532107016, 153.76], [1440532113016, 154.04], [1440532119016, 153.79], [1440532125016, 152.41], [1440532130016, 152.52], [1440532135016, 152.43], [1440532141016, 151.76], [1440532147016, 153.4], [1440532152016, 153.28], [1440532157016, 152.77], [1440532163016, 151.53], [1440532169016, 151.5], [1440532174016, 152.22], [1440532180016, 152.87], [1440532186016, 151.5], [1440532192016, 149.2], [1440532197016, 149.93], [1440532203016, 149.33], [1440532208016, 151.32], [1440532214016, 150.6], [1440532219016, 151.66], [1440532225016, 152.34], [1440532231016, 151.46], [1440532236016, 149.92], [1440532242016, 150.01], [1440532247016, 147.59], [1440532253016, 147.81], [1440532258016, 148.96], [1440532264016, 150.44], [1440532269016, 150.45], [1440532275016, 152.14], [1440532281016, 152.25], [1440532287016, 150.48], [1440532292016, 150.35], [1440532297016, 151.14], [1440532302016, 152.63], [1440532308016, 153.34], [1440532313016, 153.36], [1440532319016, 152.18], [1440532325016, 151.02], [1440532331016, 149.94], [1440532337016, 151.46], [1440532342016, 152.52], [1440532348016, 154.92], [1440532354016, 155.6], [1440532360016, 155.62], [1440532366016, 155.76], [1440532371016, 163.25], [1440532377016, 167.33], [1440532382016, 160.1], [1440532388016, 155.67], [1440532394016, 140.1], [1440532400016, 139.69], [1440532405016, 135.49], [1440532411016, 131.95], [1440532416016, 130.16], [1440532422016, 128.82], [1440532427016, 128.31], [1440532432016, 127.68], [1440532437016, 127.15], [1440532442016, 125.88], [1440532447016, 125.8], [1440532452016, 125.97], [1440532457016, 125.65], [1440532463016, 125.97], [1440532468016, 125.75], [1440532474016, 125.95], [1440532480016, 124.17], [1440532486016, 123.66], [1440532492016, 123.38], [1440532498016, 123.55], [1440532504016, 121.63], [1440532509016, 123.08], [1440532515016, 123.61], [1440532521016, 123], [1440532526016, 121.94], [1440532532016, 121.83], [1440532537016, 121.38], [1440532543016, 121.6], [1440532548016, 121.66], [1440532554016, 122.66], [1440532559016, 121.65], [1440532565016, 121.67], [1440532571016, 121.96], [1440532576016, 121.98], [1440532582016, 121.52], [1440532587016, 122.67], [1440532592016, 123.11], [1440532597016, 123.1], [1440532603016, 123.3], [1440532608016, 123.78], [1440532613016, 124.1], [1440532619016, 124.27], [1440532625016, 123.33], [1440532631016, 123.69], [1440532637016, 123.9], [1440532642016, 122.86], [1440532648016, 121.84], [1440532654016, 122.73], [1440532660016, 123.32], [1440532665016, 123.6], [1440532671016, 124.55], [1440532676016, 124.61], [1440532681016, 124.04], [1440532687016, 123.76], [1440532693016, 123.43], [1440532699016, 123.87], [1440532705016, 123.83], [1440532710016, 124.02], [1440532715016, 123.62], [1440532720016, 123.31], [1440532726016, 123.7], [1440532731016, 124.02]], "name": "DP_063321"}
	# Starting Spark Context and DataFrames
	sc = SparkContext("local", appName="JSON to DataFrame")
	sqlContext = SQLContext(sc)
	dataframe = sqlContext.read.json("data.json")
	dataframe.show()
	[paladini@starstuff smartx-fontaine]$ spark-submit spark/examples/loading_dataframe_from_kairos.py
	Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
	15/10/05 16:04:32 INFO SparkContext: Running Spark version 1.5.0
	15/10/05 16:04:33 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
	15/10/05 16:04:33 INFO SecurityManager: Changing view acls to: paladini
	15/10/05 16:04:33 INFO SecurityManager: Changing modify acls to: paladini
	15/10/05 16:04:33 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(paladini); users with modify permissions: Set(paladini)
	15/10/05 16:04:34 INFO Slf4jLogger: Slf4jLogger started
	15/10/05 16:04:34 INFO Remoting: Starting remoting
	15/10/05 16:04:34 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriver@192.168.1.111:35680]
	15/10/05 16:04:34 INFO Utils: Successfully started service 'sparkDriver' on port 35680.
	15/10/05 16:04:34 INFO SparkEnv: Registering MapOutputTracker
	15/10/05 16:04:34 INFO SparkEnv: Registering BlockManagerMaster
	15/10/05 16:04:34 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-40087457-5737-4b1f-b821-77a4dd59e3a9
	15/10/05 16:04:34 INFO MemoryStore: MemoryStore started with capacity 530.0 MB
	15/10/05 16:04:34 INFO HttpFileServer: HTTP File server directory is /tmp/spark-fd210c43-0d33-4094-a669-1d477262040d/httpd-2fbdd9ed-7fb7-4bd3-843e-70d6cae3ea82
	15/10/05 16:04:34 INFO HttpServer: Starting HTTP Server
	15/10/05 16:04:34 INFO Utils: Successfully started service 'HTTP file server' on port 32822.
	15/10/05 16:04:34 INFO SparkEnv: Registering OutputCommitCoordinator
	15/10/05 16:04:34 INFO Utils: Successfully started service 'SparkUI' on port 4040.
	15/10/05 16:04:34 INFO SparkUI: Started SparkUI at http://192.168.1.111:4040
	15/10/05 16:04:34 INFO Utils: Copying /home/paladini/ufxc/lisha/smartx-fontaine/spark/examples/loading_dataframe_from_kairos.py to /tmp/spark-fd210c43-0d33-4094-a669-1d477262040d/userFiles-3a4ad2dc-6422-4ce4-8854-74e62817613b/loading_dataframe_from_kairos.py
	15/10/05 16:04:34 INFO SparkContext: Added file file:/home/paladini/ufxc/lisha/smartx-fontaine/spark/examples/loading_dataframe_from_kairos.py at file:/home/paladini/ufxc/lisha/smartx-fontaine/spark/examples/loading_dataframe_from_kairos.py with timestamp 1444071874860
	15/10/05 16:04:35 WARN MetricsSystem: Using default name DAGScheduler for source because spark.app.id is not set.
	15/10/05 16:04:35 INFO Executor: Starting executor ID driver on host localhost
	15/10/05 16:04:35 INFO Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 46437.
	15/10/05 16:04:35 INFO NettyBlockTransferService: Server created on 46437
	15/10/05 16:04:35 INFO BlockManagerMaster: Trying to register BlockManager
	15/10/05 16:04:35 INFO BlockManagerMasterEndpoint: Registering block manager localhost:46437 with 530.0 MB RAM, BlockManagerId(driver, localhost, 46437)
	15/10/05 16:04:35 INFO BlockManagerMaster: Registered BlockManager
	15/10/05 16:04:36 INFO JSONRelation: Listing file:/home/paladini/ufxc/lisha/smartx-fontaine/data.json on driver
	15/10/05 16:04:36 INFO MemoryStore: ensureFreeSpace(213992) called with curMem=0, maxMem=555755765
	15/10/05 16:04:36 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 209.0 KB, free 529.8 MB)
	15/10/05 16:04:36 INFO MemoryStore: ensureFreeSpace(19910) called with curMem=213992, maxMem=555755765
	15/10/05 16:04:36 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 19.4 KB, free 529.8 MB)
	15/10/05 16:04:36 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on localhost:46437 (size: 19.4 KB, free: 530.0 MB)
	15/10/05 16:04:36 INFO SparkContext: Created broadcast 0 from json at NativeMethodAccessorImpl.java:-2
	15/10/05 16:04:36 INFO FileInputFormat: Total input paths to process : 1
	15/10/05 16:04:36 INFO SparkContext: Starting job: json at NativeMethodAccessorImpl.java:-2
	15/10/05 16:04:36 INFO DAGScheduler: Got job 0 (json at NativeMethodAccessorImpl.java:-2) with 1 output partitions
	15/10/05 16:04:36 INFO DAGScheduler: Final stage: ResultStage 0(json at NativeMethodAccessorImpl.java:-2)
	15/10/05 16:04:36 INFO DAGScheduler: Parents of final stage: List()
	15/10/05 16:04:36 INFO DAGScheduler: Missing parents: List()
	15/10/05 16:04:36 INFO DAGScheduler: Submitting ResultStage 0 (MapPartitionsRDD[3] at json at NativeMethodAccessorImpl.java:-2), which has no missing parents
	15/10/05 16:04:36 INFO MemoryStore: ensureFreeSpace(4368) called with curMem=233902, maxMem=555755765
	15/10/05 16:04:36 INFO MemoryStore: Block broadcast_1 stored as values in memory (estimated size 4.3 KB, free 529.8 MB)
	15/10/05 16:04:36 INFO MemoryStore: ensureFreeSpace(2460) called with curMem=238270, maxMem=555755765
	15/10/05 16:04:36 INFO MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 2.4 KB, free 529.8 MB)
	15/10/05 16:04:36 INFO BlockManagerInfo: Added broadcast_1_piece0 in memory on localhost:46437 (size: 2.4 KB, free: 530.0 MB)
	15/10/05 16:04:36 INFO SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:861
	15/10/05 16:04:36 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 0 (MapPartitionsRDD[3] at json at NativeMethodAccessorImpl.java:-2)
	15/10/05 16:04:36 INFO TaskSchedulerImpl: Adding task set 0.0 with 1 tasks
	15/10/05 16:04:36 INFO TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, PROCESS_LOCAL, 2269 bytes)
	15/10/05 16:04:36 INFO Executor: Running task 0.0 in stage 0.0 (TID 0)
	15/10/05 16:04:37 INFO Executor: Fetching file:/home/paladini/ufxc/lisha/smartx-fontaine/spark/examples/loading_dataframe_from_kairos.py with timestamp 1444071874860
	15/10/05 16:04:37 INFO Utils: /home/paladini/ufxc/lisha/smartx-fontaine/spark/examples/loading_dataframe_from_kairos.py has been previously copied to /tmp/spark-fd210c43-0d33-4094-a669-1d477262040d/userFiles-3a4ad2dc-6422-4ce4-8854-74e62817613b/loading_dataframe_from_kairos.py
	15/10/05 16:04:37 INFO HadoopRDD: Input split: file:/home/paladini/ufxc/lisha/smartx-fontaine/data.json:0+12565
	15/10/05 16:04:37 INFO deprecation: mapred.tip.id is deprecated. Instead, use mapreduce.task.id
	15/10/05 16:04:37 INFO deprecation: mapred.task.id is deprecated. Instead, use mapreduce.task.attempt.id
	15/10/05 16:04:37 INFO deprecation: mapred.task.is.map is deprecated. Instead, use mapreduce.task.ismap
	15/10/05 16:04:37 INFO deprecation: mapred.task.partition is deprecated. Instead, use mapreduce.task.partition
	15/10/05 16:04:37 INFO deprecation: mapred.job.id is deprecated. Instead, use mapreduce.job.id
	15/10/05 16:04:38 INFO Executor: Finished task 0.0 in stage 0.0 (TID 0). 3706 bytes result sent to driver
	15/10/05 16:04:38 INFO TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 1076 ms on localhost (1/1)
	15/10/05 16:04:38 INFO TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool
	15/10/05 16:04:38 INFO DAGScheduler: ResultStage 0 (json at NativeMethodAccessorImpl.java:-2) finished in 1.099 s
	15/10/05 16:04:38 INFO DAGScheduler: Job 0 finished: json at NativeMethodAccessorImpl.java:-2, took 1.177533 s
	15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(63280) called with curMem=240730, maxMem=555755765
	15/10/05 16:04:38 INFO MemoryStore: Block broadcast_2 stored as values in memory (estimated size 61.8 KB, free 529.7 MB)
	15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(19788) called with curMem=304010, maxMem=555755765
	15/10/05 16:04:38 INFO MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 19.3 KB, free 529.7 MB)
	15/10/05 16:04:38 INFO BlockManagerInfo: Added broadcast_2_piece0 in memory on localhost:46437 (size: 19.3 KB, free: 530.0 MB)
	15/10/05 16:04:38 INFO SparkContext: Created broadcast 2 from showString at NativeMethodAccessorImpl.java:-2
	15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(213992) called with curMem=323798, maxMem=555755765
	15/10/05 16:04:38 INFO MemoryStore: Block broadcast_3 stored as values in memory (estimated size 209.0 KB, free 529.5 MB)
	15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(19910) called with curMem=537790, maxMem=555755765
	15/10/05 16:04:38 INFO MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 19.4 KB, free 529.5 MB)
	15/10/05 16:04:38 INFO BlockManagerInfo: Added broadcast_3_piece0 in memory on localhost:46437 (size: 19.4 KB, free: 530.0 MB)
	15/10/05 16:04:38 INFO SparkContext: Created broadcast 3 from showString at NativeMethodAccessorImpl.java:-2
	15/10/05 16:04:38 INFO FileInputFormat: Total input paths to process : 1
	15/10/05 16:04:38 INFO SparkContext: Starting job: showString at NativeMethodAccessorImpl.java:-2
	15/10/05 16:04:38 INFO DAGScheduler: Got job 1 (showString at NativeMethodAccessorImpl.java:-2) with 1 output partitions
	15/10/05 16:04:38 INFO DAGScheduler: Final stage: ResultStage 1(showString at NativeMethodAccessorImpl.java:-2)
	15/10/05 16:04:38 INFO DAGScheduler: Parents of final stage: List()
	15/10/05 16:04:38 INFO DAGScheduler: Missing parents: List()
	15/10/05 16:04:38 INFO DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[7] at showString at NativeMethodAccessorImpl.java:-2), which has no missing parents
	15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(4800) called with curMem=557700, maxMem=555755765
	15/10/05 16:04:38 INFO MemoryStore: Block broadcast_4 stored as values in memory (estimated size 4.7 KB, free 529.5 MB)
	15/10/05 16:04:38 INFO MemoryStore: ensureFreeSpace(2715) called with curMem=562500, maxMem=555755765
	15/10/05 16:04:38 INFO MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 2.7 KB, free 529.5 MB)
	15/10/05 16:04:38 INFO BlockManagerInfo: Added broadcast_4_piece0 in memory on localhost:46437 (size: 2.7 KB, free: 529.9 MB)
	15/10/05 16:04:38 INFO SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:861
	15/10/05 16:04:38 INFO DAGScheduler: Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[7] at showString at NativeMethodAccessorImpl.java:-2)
	15/10/05 16:04:38 INFO TaskSchedulerImpl: Adding task set 1.0 with 1 tasks
	15/10/05 16:04:38 INFO TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, localhost, PROCESS_LOCAL, 2269 bytes)
	15/10/05 16:04:38 INFO Executor: Running task 0.0 in stage 1.0 (TID 1)
	15/10/05 16:04:38 INFO HadoopRDD: Input split: file:/home/paladini/ufxc/lisha/smartx-fontaine/data.json:0+12565
	15/10/05 16:04:38 INFO Executor: Finished task 0.0 in stage 1.0 (TID 1). 26671 bytes result sent to driver
	15/10/05 16:04:38 INFO DAGScheduler: ResultStage 1 (showString at NativeMethodAccessorImpl.java:-2) finished in 0.116 s
	15/10/05 16:04:38 INFO TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 122 ms on localhost (1/1)
	15/10/05 16:04:38 INFO TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool
	15/10/05 16:04:38 INFO DAGScheduler: Job 1 finished: showString at NativeMethodAccessorImpl.java:-2, took 0.145138 s
	+----------+--------------------+
	\| name\| values\|
	+----------+--------------------+
	\| teste\|[WrappedArray(1.3...\|
	\| DP_058424\|[WrappedArray(1.4...\|
	\|teste_gzip\|[WrappedArray(1.3...\|
	\| DP_063321\|[WrappedArray(1.4...\|
	+----------+--------------------+

	15/10/05 16:04:38 INFO SparkContext: Invoking stop() from shutdown hook
	15/10/05 16:04:38 INFO SparkUI: Stopped Spark web UI at http://192.168.1.111:4040
	15/10/05 16:04:38 INFO DAGScheduler: Stopping DAGScheduler
	15/10/05 16:04:38 INFO MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
	15/10/05 16:04:38 INFO MemoryStore: MemoryStore cleared
	15/10/05 16:04:38 INFO BlockManager: BlockManager stopped
	15/10/05 16:04:38 INFO BlockManagerMaster: BlockManagerMaster stopped
	15/10/05 16:04:38 INFO OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
	15/10/05 16:04:38 INFO SparkContext: Successfully stopped SparkContext
	15/10/05 16:04:38 INFO ShutdownHookManager: Shutdown hook called
	15/10/05 16:04:38 INFO ShutdownHookManager: Deleting directory /tmp/spark-fd210c43-0d33-4094-a669-1d477262040d
	[paladini@starstuff smartx-fontaine]$