create_emails_webpage.py

#Copyright Timothy Clemans 2014 released under GPL
import os
import shutil
os.system('python gyb.py --email policevideorequests@gmail.com --search "to:kingcounty.gov OR from:kingcounty.gov"')
os.system('rm emails/*')
for dirpath, dirnames, filenames in os.walk("GYB-GMail-Backup-policevideorequests@gmail.com"):
    for filename in filenames:
        try:
            shutil.copy(os.path.join(dirpath, filename), os.path.join('/root/got-your-back-0.29/emails/', filename))
        except OSError:
            print ("Could not move %s " % os.join(dirpath, filename))
import sys  

reload(sys)  
sys.setdefaultencoding('utf8')
"""read and display mail info"""
from dateutil import tz
import sys
import os
import optparse
import locale
emails = []
try:
    import pyzmail
except ImportError:
    if os.path.isdir('../pyzmail'):
        sys.path.append(os.path.abspath('..'))
    elif os.path.isdir('pyzmail'):
        sys.path.append(os.path.abspath('.'))
    import pyzmail
#print [f for f in os.listdir('/root/got-your-back-0.29/emails/') if os.path.isfile(f) and f.endswith('.eml')]
files = sorted([f for f in os.listdir('/root/got-your-back-0.29/emails/') if f.endswith('.eml')])[::-1]
for f in files:
    
    msg=pyzmail.PyzMessage.factory(open(os.path.join('/root/got-your-back-0.29/emails/', f), 'rb'))
    cols = {'Date': '%s' % (msg.get('date', ''), ), 'Subject': '%s' % (msg.get_subject().strip(), ), 'From':'%s <%s>' % (msg.get_address('from')[0], msg.get_address('from')[1]), 'To':'%s <%s>' % (msg.get_address('to')[0], msg.get_address('to')[1])}
    cols["attachments"] = []
    cols['body'] = ''
    #print 'Subject: %r' % (msg.get_subject(), )
    #print 'From: %r' % (msg.get_address('from'), )
    #print 'To: %r' % (msg.get_addresses('to'), )
    #print 'Cc: %r' % (msg.get_addresses('cc'), )
    #print 'Date: %r' % (msg.get('date', ''), )
    #print 'Message-Id: %r' % (msg.get('message-id', ''), )
    import re

    case_numbers = []

    for mailpart in msg.mailparts:
        
        if mailpart.filename:
            import random
            import string
            randomid = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8))
            cols["attachments"].append('<a href="%s">%s</a>' % (randomid+mailpart.filename, mailpart.filename))
            
        
            open(os.path.join('emails', randomid+mailpart.filename), 'wb').write(mailpart.get_payload())

            case_numbers_m = re.search('\d{2}-\d{6}', str(mailpart.filename))
            if case_numbers_m:
                case_numbers.append(case_numbers_m.group(0))

        if mailpart.is_body=='text/plain':
            # print first 3 lines
            payload, used_charset=pyzmail.decode_text(mailpart.get_payload(), mailpart.charset, None)
            cols['body'] = payload
            for line in payload.split('\n'):
                # be careful console can be unable to display unicode characters
                if line:
                    #print '       >', line
                    case_numbers_m = re.findall('\d{2}-\d{6}', line)
                    case_numbers.extend(case_numbers_m)
    #print case_numbers
    cols["case_numbers"] = ', '.join(list(set(case_numbers)))
    emails.append(cols)
#print emails
html = """
<style>
body,td,th {
font:.8em Arial;
}
table {
border-collapse:collapse;
}
th,td {
border:1px solid #000;
padding:5px;
text-align:left;
vertical-align:top;
}
th {
font-weight:bold;
}
</style>
<script src="https://code.jquery.com/jquery-1.11.2.min.js"></script>
<script>
$(function() {

     $("#drop").change(function () {
        $('tr').hide();
        $('.header').show();
        $('.'+$('#drop option:selected').val()).show();
    });
});
</script>
<title>Realtime feed of emails from King County Sheriff's Office</title>
<h1>Realtime feed of PDR response emails with records from King County Sheriff's Office to parties not mentioned in the records they are requesting</h1>

<p>Effective December 17, 2014 the King County Sheriff's Office copies me on every public disclosure response that has responsive records if the requester is not involved in the records he or she requests. Files placed on disc are provided to me via Microsoft OneDrive.</p>
<p>The notice KCSO appends to responses is: "The King County Sheriff's Office has received the following request from an anonymous requestor, "Anon Anon" at policevideorequests@gmail.com :
"I request records provided one calendar day prior to date of this email being sent to any non-involved third parties be provided via email or OneDrive."
This is an automated daily public disclosure request. In an effort toward transparency and efficiency, Anon Anon is being copied on all public disclosure correspondence where we are releasing information to requestors. This includes, but is not limited to, requesters who are non-involved third parties as it relates to the records they are seeking.
If you have questions, please let us know!"</p>"""

# Tue, 02 Dec 2014 17:00:24 -0000
import email.utils
from datetime import datetime
import time
emails = [e for e in emails if e["Date"]]
emails = sorted(emails, reverse=True, key=lambda x: email.utils.parsedate_tz(x["Date"]))
import hashlib
def hash(item):
    m = hashlib.md5()
    m.update(item)
    return m.hexdigest()
tos = sorted(list(set([(hash(i["To"]), i["To"]) for i in emails if not 'kingcounty.gov' in i["To"].lower()])))
dropdown = ' Show just emails sent to: <select id="drop"><option></option>'
for i in tos:
    dropdown += '<option value="%s">%s</option>' % (i[0], i[1])
dropdown += "</select>"
html += '<input type="button" value="Toggle email content" onclick="$(\'.email_body\').toggle()" />%s' %  dropdown
html += '<table style="width:100%;">'
html += '<tr class="header"><th>Date</th><th>From</th><th>To</th><th>Subject</th><th>Case numbers</th><th>Attachments</th></tr>'
for i, e in enumerate(emails):
    print i
    attachments = '<ul>'
    for attachment in e['attachments']:
        if attachment:
            attachments += '<li>'+attachment+'</li>'
    attachments += '</ul>'
    localdate = datetime.fromtimestamp(email.utils.mktime_tz(email.utils.parsedate_tz(e['Date'])))
    from_zone = tz.gettz('UTC')
    to_zone = tz.gettz('Antarctica/Rothera') # this is the timezone I had to set in order for the right timezone to come out (west coast)
    utc = localdate.replace(tzinfo=from_zone)
    localdate = utc.astimezone(to_zone).strftime('%m/%d/%y %H:%M')
    html += '<tr class="%s"><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td></tr>' % (hash(e['To']), localdate, e['From'], e['To'], e['Subject'], e['case_numbers'], attachments)
    html += '<tr class="%s"><td colspan="6"><input type="button" value="Show email" onclick="$(\'#email_body_%s\').show()" /><div class="email_body" id="email_body_%s" style="display:none">%s</div></td></tr>' % (hash(e['To']), i, i, '<br>'.join(e["body"].encode('utf-8').replace('<', '&lt;').split('\n')))
html += "</table>"
f = open('/root/got-your-back-0.29/emails/index.html', 'w')
f.write( html.encode('utf-8'))
f.close()
os.system('cp emails/* /var/www/html/')
print 'done'