Skip to content

Instantly share code, notes, and snippets.

@vperron
Created September 5, 2017 14:34
Show Gist options
  • Save vperron/52f65e9127c53a4f56726d7f5dbbf8af to your computer and use it in GitHub Desktop.
Save vperron/52f65e9127c53a4f56726d7f5dbbf8af to your computer and use it in GitHub Desktop.
# -*- coding: utf-8 -*-
from __future__ import division
import logging
import os
import json
import pydash
from decimal import Decimal
from celery import task
from urllib import urlencode
from urlparse import urljoin
from datetime import time
from premailer import Premailer
from email.MIMEImage import MIMEImage
from django.conf import settings
from django.utils import translation
from django.core.mail import EmailMultiAlternatives
from django.core.exceptions import ValidationError
from django.utils.formats import dateformat
from django.template.loader import render_to_string
from django.utils.translation import ugettext as _
from django.utils.timezone import datetime, timedelta
from django.db.models import Count
from api.utils import xpercent, xcurrency, xtimedelta, xmul
from api.utils.browsers import get_webdriver
from api.utils.dates import week_range
from api.utils.periods.definitions import LAST_WEEK, TWO_WEEKS_AGO, WEEK
from api.organizations.models import OrganizationMember
from api.performances.models import (ConversionKPI, AverageShopTimeKPI,
RepeatShoppersKPI, CaptureKPI)
from api.utils.periods.serializers import PeriodSerializer
from .constants import DEFAULT_BOUNCE_TRESHOLD
from .models import ShoppingSession, ShopDailyAnalytics
logger = logging.getLogger(__name__)
def get_frequencies(organization, start, stop):
first_shop = organization.shops.first()
if not first_shop:
return []
tz = first_shop.timezone
xstart = datetime.combine(start, time())
xstop = datetime.combine(stop, time())
return ShoppingSession.objects.filter(
analytics__shop__organization__uid=organization.uid,
start_at__range=[tz.localize(xstart), tz.localize(xstop)],
).values(
'dvuid',
).annotate(
frequency=Count('uid')
).order_by().values_list(
'frequency', flat=True,
)
def get_average_frequency(organization, start, stop):
first_shop = organization.shops.first()
if not first_shop:
return []
tz = first_shop.timezone
xstart = datetime.combine(start, time())
xstop = datetime.combine(stop, time())
agg = ShoppingSession.objects.filter(
analytics__shop__organization__uid=organization.uid,
start_at__range=[tz.localize(xstart), tz.localize(xstop)],
).aggregate(
visits=Count('uid'),
unique_devices=Count('dvuid', distinct=True),
)
return Decimal(agg['visits'] / agg['unique_devices']).quantize(Decimal('.01'))
def get_bounce_rate(organization, start, stop, bounce_treshold):
first_shop = organization.shops.first()
if not first_shop:
return []
tz = first_shop.timezone
xstart = datetime.combine(start, time())
xstop = datetime.combine(stop, time())
quick_sessions_cnt = ShoppingSession.objects.filter(
analytics__shop__organization__uid=organization.uid,
start_at__range=[tz.localize(xstart), tz.localize(xstop)],
duration__lt=bounce_treshold,
).count()
long_sessions_cnt = ShoppingSession.objects.filter(
analytics__shop__organization__uid=organization.uid,
start_at__range=[tz.localize(xstart), tz.localize(xstop)],
duration__gte=bounce_treshold,
).count()
return float(quick_sessions_cnt) / float(long_sessions_cnt)
def attach_image_data(msg, name, data, ext='png'):
msg_img = MIMEImage(data)
msg_img.add_header('Content-ID', '<{name}>'.format(name=name))
msg_img.add_header('X-Attachment-Id', '{name}'.format(name=name))
msg_img.add_header(
'Content-Disposition', 'inline; filename="{name}.{ext}"'.format(name=name, ext=ext)
)
msg_img.replace_header(
'Content-Type', 'image/{ext}; name="{name}.{ext}"'.format(name=name, ext=ext)
)
msg.attach(msg_img)
def insert_inline_image(msg, img_path):
cid = os.path.splitext(os.path.basename(img_path))[0]
filename = os.path.join(settings.STATIC_ROOT, img_path)
with open(filename, 'rb') as f:
data = f.read()
attach_image_data(msg, cid, data)
def get_foot_traffic_screenshot(driver, visits, cvisits):
main_title = _("Traffic per Week Day")
weekdays = map(lambda k: dateformat.format(k, 'D'), week_range())
week_labels = [_("Last Week"), _("Two Weeks Ago")]
data = []
for i, d in enumerate(weekdays):
data.append({'x': i, 'y': 0, 'v': 0 if i >= len(visits) else visits[i]})
data.append({'x': i, 'y': 1, 'v': 0 if i >= len(cvisits) else cvisits[i]})
url = "{base_url}?{query}".format(
base_url=urljoin(settings.DASHBOARD_URL, '/#/headless/foot_traffic'),
query=urlencode(dict(
data=json.dumps(data),
params=json.dumps(dict(
weekdays=weekdays,
mainTitle=main_title,
week_labels=week_labels,
))
))
)
driver.get(url)
return driver.get_screenshot_as_png()
def get_conversion_screenshot(driver, visits, transactions, sales, currency):
main_title = _("Weekly Conversion & Sales")
left_title = _("People")
right_title = unicode(currency)
legends = [{
'text': _("Visits"),
'color': '#00afc4',
}, {
'text': _("Transactions"),
'color': '#cc2676',
}, {
'text': _("Sales"),
'color': '#ed5564',
}]
weekdays = map(lambda k: dateformat.format(k, 'D'), week_range())
data = []
for i, d in enumerate(weekdays):
data.append({
'xLabel': d,
'sales': 0 if i >= len(sales) else sales[i],
'visits': 0 if i >= len(visits) else visits[i],
'transactions': 0 if i >= len(transactions) else transactions[i],
})
url = "{base_url}?{query}".format(
base_url=urljoin(settings.DASHBOARD_URL, '/#/headless/conversion'),
query=urlencode(dict(
data=json.dumps(data),
params=json.dumps(dict(
mainTitle=main_title,
leftTitle=left_title,
rightTitle=right_title,
legends=legends,
))
))
)
driver.get(url)
return driver.get_screenshot_as_png()
def list2frequencies(lst, min_freq=0, max_freq=5):
dct = {i: 0 for i in xrange(min_freq, max_freq + 1)}
groups = pydash.group_by(lst)
for k, v in groups.iteritems():
if k >= max_freq:
dct[max_freq] += len(v)
elif k < min_freq:
continue
else:
dct[k] = len(v)
return dct
def get_loyalty_screenshot(driver, current_freqs, comparison_freqs, min_freq=2, max_freq=5):
main_title = _("Visit Frequency")
left_title = _("People")
legends = [{
'text': _("Last Week"),
'color': '#00afc4',
}, {
'text': _("Two Weeks Ago"),
'color': '#cc2676',
}]
current_histogram = list2frequencies(current_freqs, min_freq=min_freq, max_freq=max_freq)
comparison_histogram = list2frequencies(comparison_freqs, min_freq=min_freq, max_freq=max_freq)
data = []
for k in current_histogram.keys():
data.append({
'label': k if k != max_freq else "%s+" % max_freq,
'values': [current_histogram[k], comparison_histogram[k]]
})
url = "{base_url}?{query}".format(
base_url=urljoin(settings.DASHBOARD_URL, '/#/headless/loyalty'),
query=urlencode(dict(
data=json.dumps(data),
params=json.dumps(dict(
mainTitle=main_title,
leftTitle=left_title,
legends=legends,
))
))
)
driver.get(url)
return driver.get_screenshot_as_png()
def boolean2color(b):
return 'locariseBlue' if b else 'locarisePink'
def timedelta2color(td):
td = td if td else timedelta(0)
return boolean2color(td >= timedelta(0))
@task
def generate_weekly_report():
"""
Generate an analytics weekly digest for every organization that
has at least an `OrganizationMember` with its `weekly_digest` attribute
Truthy.
"""
for member in OrganizationMember.objects.filter(weekly_digest=True):
send_weekly_report(member)
def send_weekly_report(member):
"""
Send a weekly report to an Organization Member.
"""
orga = member.organization
locale = member.user.locale
# retrieve current and comparison queryset based on period parameters.
period_parameters = {
'period_type': WEEK,
'current_period': LAST_WEEK,
'comparison_period': TWO_WEEKS_AGO
}
queryset = ShopDailyAnalytics.objects.select_related(
'analytics'
).filter(
analytics__shop__organization__uid=member.organization.uid # all shops
)
analytics_start_at = queryset.get_analytics_start_at()
serializer = PeriodSerializer(
analytics_start_at=analytics_start_at,
data=period_parameters
)
if not serializer.is_valid() or not serializer.periods_are_available():
raise ValidationError(u'Last week or 2 weeks ago querysets are empty')
current_qs = queryset.period_filter(serializer.object['current_period'])
comparison_qs = queryset.period_filter(serializer.object['comparison_period'])
current_data = current_qs.get_aggregated_data()
compare_data = comparison_qs.get_aggregated_data()
start_date = current_qs.first().date
end_date = current_qs.last().date
current_freq = get_frequencies(orga, start_date, end_date)
comparison_freq = get_frequencies(
orga, comparison_qs.first().date, comparison_qs.last().date
)
bounce_rate = get_bounce_rate(
orga, start_date, end_date, DEFAULT_BOUNCE_TRESHOLD
)
avg_freq = get_average_frequency(orga, start_date, end_date)
cavg_freq = get_average_frequency(orga, comparison_qs.first().date, comparison_qs.last().date)
capture = CaptureKPI.from_dailies([current_data], [compare_data])
dwell_time = AverageShopTimeKPI.from_dailies([current_data], [compare_data])
conversion_rate = ConversionKPI.from_dailies([current_data], [compare_data])
repeat_shoppers = RepeatShoppersKPI.from_dailies([current_data], [compare_data])
gross_return_percent = int(round(xmul(repeat_shoppers.current_value, 10.0)))
email_data = {
'organization_name': orga.name,
'start_date': start_date,
'end_date': end_date,
'dashboard_url': urljoin(settings.DASHBOARD_URL, '/#/app/%s' % orga.uid),
'total_visits': current_data['total_visits'],
'total_passersby': current_data['total_passersby'],
'window_conversion': xpercent(capture.current_value),
'comp_window_conversion': xpercent(capture.get_relative_change(), True),
'comp_window_conversion_colorclass': boolean2color(capture.get_relative_change() >= 0),
'has_return_ratio': repeat_shoppers.current_value is not None, # not used yet
'return_ratio': xpercent(repeat_shoppers.current_value),
'comp_return_ratio': xpercent(repeat_shoppers.get_relative_change(), True),
'comp_return_ratio_colorclass': boolean2color(repeat_shoppers.get_relative_change() >= 0),
'new_visitors': range(10 - gross_return_percent),
'return_visitors': range(gross_return_percent),
'total_new': current_data['unique_devices'] - current_data['returns'],
'total_returns': current_data['returns'],
'has_dwell_time': dwell_time.current_value is not None, # not used yet
'dwell_time': xtimedelta(dwell_time.current_value),
'comp_dwell_time': xtimedelta(dwell_time.get_relative_change(), True),
'comp_dwell_time_colorclass': timedelta2color(dwell_time.get_relative_change()),
'bounce_rate': xpercent(bounce_rate),
'has_revenues': current_data['total_transactions'] is not None,
'total_transactions': current_data['total_transactions'],
'total_sales': xcurrency(current_data['total_sales'], orga.currency),
'conversion_rate': xpercent(conversion_rate.current_value),
'comp_conversion_rate': xpercent(conversion_rate.get_relative_change(), True),
'avg_visit_frequency': avg_freq,
'comp_avg_visit_frequency': cavg_freq,
}
# Insert aggregated data into translatable template
translation.activate(locale)
subject = render_to_string('email/weekly_digest_title.txt', email_data)
subject = " ".join(subject.splitlines()).strip().encode('utf-8')
text_message = render_to_string('email/weekly_digest.txt', email_data)
html_message = render_to_string('email/weekly_digest.html', email_data)
# Inline style inside HTML tags
inlined_html = Premailer(
html_message, disable_validation=True
).transform(True).encode('utf-8')
# Generate the multipart email with static images embedded
msg = EmailMultiAlternatives(
subject, text_message, settings.DEFAULT_FROM_EMAIL, [member.user.email]
)
msg.attach_alternative(inlined_html, 'text/html')
# https://www.vlent.nl/weblog/2014/01/15/sending-emails-with-embedded-images-in-django/
msg.mixed_subtype = 'related'
# Attach static icons & logos
icon_filenames = ['logo-horizontal-128', 'weekly-top-logo',
'visitor_new', 'dwell_time', 'window_conversion',
'visitor_return', 'crown']
for filename in icon_filenames:
insert_inline_image(msg, 'img/email/%s.png' % filename)
# Generate charts with headless dashboard
sales = [int(d.total_sales) for d in current_qs]
visits = [d.total_visits for d in current_qs]
transactions = [d.total_transactions for d in current_qs]
comparison_visits = [d.total_visits for d in comparison_qs]
with get_webdriver() as driver:
foot_traffic_image = get_foot_traffic_screenshot(
driver, visits, comparison_visits
)
attach_image_data(msg, 'foot_traffic', foot_traffic_image)
conversion_image = get_conversion_screenshot(
driver, visits, transactions, sales, orga.currency
)
attach_image_data(msg, 'conversion', conversion_image)
loyalty_image = get_loyalty_screenshot(
driver, current_freq, comparison_freq
)
attach_image_data(msg, 'loyalty', loyalty_image)
logger.info(u"Send weekly report to {}".format(member.user.email))
msg.send()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment