jupyter

 #%pylab inline

In [2]:

import dataiku
import dataiku.spark as dkuspark
import pyspark
from pyspark.sql import SQLContext

In [3]:

# Load PySpark
sc = pyspark.SparkContext()
sqlContext = SQLContext(sc)

In [4]:

# Example: Read the descriptor of a Dataiku dataset
mydataset = dataiku.Dataset("csvisit_99k_prepared")
# And read it as a Spark dataframe
df = dkuspark.get_dataframe(sqlContext, mydataset)
# df = mydataset.get_dataframe()

In [5]:

# Example: Get the count of records in the dataframe
df.count()

Out[5]:

96218

In [6]:

import pandas as pd
z = df.select("visit_duration","page_views_num").toPandas()
#z.plot(kind='line')
#z.plot(kind='bar')
#z.plot(kind='hist')
#z.plot(kind='hist')
z.plot(kind='line', x='visit_duration', y='page_views_num', c='visit_duration');

In [7]:

import pandas as pd
z = df.select("visit_duration","page_views_num").toPandas()
z.plot()

Out[7]:

<matplotlib.axes._subplots.AxesSubplot at 0x58f76d0>