5 Sparkify listen_freq

sparkify_5_listen_freq.py

# Create a new aggreated dataframe called listen_freq 
# (stands for listening frequency) for each user 

listen_freq = data.select('userId','sessionId', 'timeStamp')\
    .groupBy('userId','sessionId')\
    .agg(F.min('timeStamp').alias('sessionTime'))\
    .orderBy('userId', 'sessionId')\
    .groupBy('userId')\
    .agg(F.min('sessionTime').alias('minSessionTime'), 
         F.max('sessionTime').alias('maxSessionTime'),
         F.count('sessionId').alias('sessionCount'))\
    .withColumn('sessionsFreqDay', F.datediff('maxSessionTime', 'minSessionTime')/col('sessionCount'))\
    .orderBy('userId')

listen_freq.cache()
listen_freq.show(10)