send_emails.py

# -*- coding: utf-8 -*-
from __future__ import division
import logging

import os
import json
import pydash

from decimal import Decimal
from celery import task
from urllib import urlencode
from urlparse import urljoin
from datetime import time
from premailer import Premailer

from email.MIMEImage import MIMEImage
from django.conf import settings
from django.utils import translation
from django.core.mail import EmailMultiAlternatives
from django.core.exceptions import ValidationError
from django.utils.formats import dateformat
from django.template.loader import render_to_string
from django.utils.translation import ugettext as _
from django.utils.timezone import datetime, timedelta
from django.db.models import Count

from api.utils import xpercent, xcurrency, xtimedelta, xmul
from api.utils.browsers import get_webdriver
from api.utils.dates import week_range
from api.utils.periods.definitions import LAST_WEEK, TWO_WEEKS_AGO, WEEK
from api.organizations.models import OrganizationMember
from api.performances.models import (ConversionKPI, AverageShopTimeKPI,
                                     RepeatShoppersKPI, CaptureKPI)
from api.utils.periods.serializers import PeriodSerializer

from .constants import DEFAULT_BOUNCE_TRESHOLD
from .models import ShoppingSession, ShopDailyAnalytics


logger = logging.getLogger(__name__)


def get_frequencies(organization, start, stop):
    first_shop = organization.shops.first()
    if not first_shop:
        return []
    tz = first_shop.timezone
    xstart = datetime.combine(start, time())
    xstop = datetime.combine(stop, time())
    return ShoppingSession.objects.filter(
        analytics__shop__organization__uid=organization.uid,
        start_at__range=[tz.localize(xstart), tz.localize(xstop)],
    ).values(
        'dvuid',
    ).annotate(
        frequency=Count('uid')
    ).order_by().values_list(
        'frequency', flat=True,
    )


def get_average_frequency(organization, start, stop):
    first_shop = organization.shops.first()
    if not first_shop:
        return []

    tz = first_shop.timezone
    xstart = datetime.combine(start, time())
    xstop = datetime.combine(stop, time())

    agg = ShoppingSession.objects.filter(
        analytics__shop__organization__uid=organization.uid,
        start_at__range=[tz.localize(xstart), tz.localize(xstop)],
    ).aggregate(
        visits=Count('uid'),
        unique_devices=Count('dvuid', distinct=True),
    )

    return Decimal(agg['visits'] / agg['unique_devices']).quantize(Decimal('.01'))


def get_bounce_rate(organization, start, stop, bounce_treshold):
    first_shop = organization.shops.first()
    if not first_shop:
        return []
    tz = first_shop.timezone
    xstart = datetime.combine(start, time())
    xstop = datetime.combine(stop, time())
    quick_sessions_cnt = ShoppingSession.objects.filter(
        analytics__shop__organization__uid=organization.uid,
        start_at__range=[tz.localize(xstart), tz.localize(xstop)],
        duration__lt=bounce_treshold,
    ).count()
    long_sessions_cnt = ShoppingSession.objects.filter(
        analytics__shop__organization__uid=organization.uid,
        start_at__range=[tz.localize(xstart), tz.localize(xstop)],
        duration__gte=bounce_treshold,
    ).count()
    return float(quick_sessions_cnt) / float(long_sessions_cnt)


def attach_image_data(msg, name, data, ext='png'):
    msg_img = MIMEImage(data)
    msg_img.add_header('Content-ID', '<{name}>'.format(name=name))
    msg_img.add_header('X-Attachment-Id', '{name}'.format(name=name))
    msg_img.add_header(
        'Content-Disposition', 'inline; filename="{name}.{ext}"'.format(name=name, ext=ext)
    )
    msg_img.replace_header(
        'Content-Type', 'image/{ext}; name="{name}.{ext}"'.format(name=name, ext=ext)
    )
    msg.attach(msg_img)


def insert_inline_image(msg, img_path):
    cid = os.path.splitext(os.path.basename(img_path))[0]
    filename = os.path.join(settings.STATIC_ROOT, img_path)
    with open(filename, 'rb') as f:
        data = f.read()
    attach_image_data(msg, cid, data)


def get_foot_traffic_screenshot(driver, visits, cvisits):
    main_title = _("Traffic per Week Day")
    weekdays = map(lambda k: dateformat.format(k, 'D'), week_range())
    week_labels = [_("Last Week"), _("Two Weeks Ago")]
    data = []
    for i, d in enumerate(weekdays):
        data.append({'x': i, 'y': 0, 'v': 0 if i >= len(visits) else visits[i]})
        data.append({'x': i, 'y': 1, 'v': 0 if i >= len(cvisits) else cvisits[i]})

    url = "{base_url}?{query}".format(
        base_url=urljoin(settings.DASHBOARD_URL, '/#/headless/foot_traffic'),
        query=urlencode(dict(
            data=json.dumps(data),
            params=json.dumps(dict(
                weekdays=weekdays,
                mainTitle=main_title,
                week_labels=week_labels,
            ))
        ))
    )

    driver.get(url)
    return driver.get_screenshot_as_png()


def get_conversion_screenshot(driver, visits, transactions, sales, currency):
    main_title = _("Weekly Conversion & Sales")
    left_title = _("People")
    right_title = unicode(currency)
    legends = [{
        'text': _("Visits"),
        'color': '#00afc4',
    }, {
        'text': _("Transactions"),
        'color': '#cc2676',
    }, {
        'text': _("Sales"),
        'color': '#ed5564',
    }]
    weekdays = map(lambda k: dateformat.format(k, 'D'), week_range())
    data = []
    for i, d in enumerate(weekdays):
        data.append({
            'xLabel': d,
            'sales': 0 if i >= len(sales) else sales[i],
            'visits': 0 if i >= len(visits) else visits[i],
            'transactions': 0 if i >= len(transactions) else transactions[i],
        })
    url = "{base_url}?{query}".format(
        base_url=urljoin(settings.DASHBOARD_URL, '/#/headless/conversion'),
        query=urlencode(dict(
            data=json.dumps(data),
            params=json.dumps(dict(
                mainTitle=main_title,
                leftTitle=left_title,
                rightTitle=right_title,
                legends=legends,
            ))
        ))
    )

    driver.get(url)
    return driver.get_screenshot_as_png()


def list2frequencies(lst, min_freq=0, max_freq=5):
    dct = {i: 0 for i in xrange(min_freq, max_freq + 1)}
    groups = pydash.group_by(lst)
    for k, v in groups.iteritems():
        if k >= max_freq:
            dct[max_freq] += len(v)
        elif k < min_freq:
            continue
        else:
            dct[k] = len(v)
    return dct


def get_loyalty_screenshot(driver, current_freqs, comparison_freqs, min_freq=2, max_freq=5):
    main_title = _("Visit Frequency")
    left_title = _("People")
    legends = [{
        'text': _("Last Week"),
        'color': '#00afc4',
    }, {
        'text': _("Two Weeks Ago"),
        'color': '#cc2676',
    }]
    current_histogram = list2frequencies(current_freqs, min_freq=min_freq, max_freq=max_freq)
    comparison_histogram = list2frequencies(comparison_freqs, min_freq=min_freq, max_freq=max_freq)
    data = []
    for k in current_histogram.keys():
        data.append({
            'label': k if k != max_freq else "%s+" % max_freq,
            'values': [current_histogram[k], comparison_histogram[k]]
        })
    url = "{base_url}?{query}".format(
        base_url=urljoin(settings.DASHBOARD_URL, '/#/headless/loyalty'),
        query=urlencode(dict(
            data=json.dumps(data),
            params=json.dumps(dict(
                mainTitle=main_title,
                leftTitle=left_title,
                legends=legends,
            ))
        ))
    )

    driver.get(url)
    return driver.get_screenshot_as_png()


def boolean2color(b):
    return 'locariseBlue' if b else 'locarisePink'


def timedelta2color(td):
    td = td if td else timedelta(0)
    return boolean2color(td >= timedelta(0))


@task
def generate_weekly_report():
    """
    Generate an analytics weekly digest for every organization that
    has at least an `OrganizationMember` with its `weekly_digest` attribute
    Truthy.
    """
    for member in OrganizationMember.objects.filter(weekly_digest=True):
        send_weekly_report(member)


def send_weekly_report(member):
    """
    Send a weekly report to an Organization Member.
    """
    orga = member.organization
    locale = member.user.locale

    # retrieve current and comparison queryset based on period parameters.

    period_parameters = {
        'period_type': WEEK,
        'current_period': LAST_WEEK,
        'comparison_period': TWO_WEEKS_AGO
    }

    queryset = ShopDailyAnalytics.objects.select_related(
        'analytics'
    ).filter(
        analytics__shop__organization__uid=member.organization.uid  # all shops
    )
    analytics_start_at = queryset.get_analytics_start_at()

    serializer = PeriodSerializer(
        analytics_start_at=analytics_start_at,
        data=period_parameters
    )
    if not serializer.is_valid() or not serializer.periods_are_available():
        raise ValidationError(u'Last week or 2 weeks ago querysets are empty')

    current_qs = queryset.period_filter(serializer.object['current_period'])
    comparison_qs = queryset.period_filter(serializer.object['comparison_period'])

    current_data = current_qs.get_aggregated_data()
    compare_data = comparison_qs.get_aggregated_data()

    start_date = current_qs.first().date
    end_date = current_qs.last().date

    current_freq = get_frequencies(orga, start_date, end_date)
    comparison_freq = get_frequencies(
        orga, comparison_qs.first().date, comparison_qs.last().date
    )
    bounce_rate = get_bounce_rate(
        orga, start_date, end_date, DEFAULT_BOUNCE_TRESHOLD
    )
    avg_freq = get_average_frequency(orga, start_date, end_date)
    cavg_freq = get_average_frequency(orga, comparison_qs.first().date, comparison_qs.last().date)

    capture = CaptureKPI.from_dailies([current_data], [compare_data])
    dwell_time = AverageShopTimeKPI.from_dailies([current_data], [compare_data])
    conversion_rate = ConversionKPI.from_dailies([current_data], [compare_data])
    repeat_shoppers = RepeatShoppersKPI.from_dailies([current_data], [compare_data])

    gross_return_percent = int(round(xmul(repeat_shoppers.current_value, 10.0)))

    email_data = {
        'organization_name': orga.name,
        'start_date': start_date,
        'end_date': end_date,
        'dashboard_url': urljoin(settings.DASHBOARD_URL, '/#/app/%s' % orga.uid),
        'total_visits': current_data['total_visits'],
        'total_passersby': current_data['total_passersby'],
        'window_conversion': xpercent(capture.current_value),
        'comp_window_conversion': xpercent(capture.get_relative_change(), True),
        'comp_window_conversion_colorclass': boolean2color(capture.get_relative_change() >= 0),

        'has_return_ratio': repeat_shoppers.current_value is not None,  # not used yet
        'return_ratio': xpercent(repeat_shoppers.current_value),
        'comp_return_ratio': xpercent(repeat_shoppers.get_relative_change(), True),
        'comp_return_ratio_colorclass': boolean2color(repeat_shoppers.get_relative_change() >= 0),

        'new_visitors': range(10 - gross_return_percent),
        'return_visitors': range(gross_return_percent),

        'total_new': current_data['unique_devices'] - current_data['returns'],
        'total_returns': current_data['returns'],

        'has_dwell_time': dwell_time.current_value is not None,  # not used yet
        'dwell_time': xtimedelta(dwell_time.current_value),
        'comp_dwell_time': xtimedelta(dwell_time.get_relative_change(), True),
        'comp_dwell_time_colorclass': timedelta2color(dwell_time.get_relative_change()),

        'bounce_rate': xpercent(bounce_rate),

        'has_revenues': current_data['total_transactions'] is not None,
        'total_transactions': current_data['total_transactions'],
        'total_sales': xcurrency(current_data['total_sales'], orga.currency),
        'conversion_rate': xpercent(conversion_rate.current_value),
        'comp_conversion_rate': xpercent(conversion_rate.get_relative_change(), True),

        'avg_visit_frequency': avg_freq,
        'comp_avg_visit_frequency': cavg_freq,
    }

    # Insert aggregated data into translatable template
    translation.activate(locale)
    subject = render_to_string('email/weekly_digest_title.txt', email_data)
    subject = " ".join(subject.splitlines()).strip().encode('utf-8')
    text_message = render_to_string('email/weekly_digest.txt', email_data)
    html_message = render_to_string('email/weekly_digest.html', email_data)

    # Inline style inside HTML tags
    inlined_html = Premailer(
        html_message, disable_validation=True
    ).transform(True).encode('utf-8')

    # Generate the multipart email with static images embedded
    msg = EmailMultiAlternatives(
        subject, text_message, settings.DEFAULT_FROM_EMAIL, [member.user.email]
    )
    msg.attach_alternative(inlined_html, 'text/html')
    # https://www.vlent.nl/weblog/2014/01/15/sending-emails-with-embedded-images-in-django/
    msg.mixed_subtype = 'related'

    # Attach static icons & logos
    icon_filenames = ['logo-horizontal-128', 'weekly-top-logo',
                      'visitor_new', 'dwell_time', 'window_conversion',
                      'visitor_return', 'crown']
    for filename in icon_filenames:
        insert_inline_image(msg, 'img/email/%s.png' % filename)

    # Generate charts with headless dashboard
    sales = [int(d.total_sales) for d in current_qs]
    visits = [d.total_visits for d in current_qs]
    transactions = [d.total_transactions for d in current_qs]
    comparison_visits = [d.total_visits for d in comparison_qs]

    with get_webdriver() as driver:

        foot_traffic_image = get_foot_traffic_screenshot(
            driver, visits, comparison_visits
        )
        attach_image_data(msg, 'foot_traffic', foot_traffic_image)

        conversion_image = get_conversion_screenshot(
            driver, visits, transactions, sales, orga.currency
        )
        attach_image_data(msg, 'conversion', conversion_image)

        loyalty_image = get_loyalty_screenshot(
            driver, current_freq, comparison_freq
        )
        attach_image_data(msg, 'loyalty', loyalty_image)

    logger.info(u"Send weekly report to {}".format(member.user.email))
    msg.send()