velenux/extract_timeline_appearance.py

## extract_timeline_appearance.py
# encoding: utf-8

import json
import pprint
import codecs
import iso8601
import datetime

# this is the json from https://elezioni.tracking.exposed/campaign/opendata/
filename = 'archive.json'

symbols = {
#    'photo': '&#x1f5bc;',
#    'post': '&#x1f5b9;',
#    'video': '&#x1f39e;'
    'photo': '&#9632;',
    'post': '&#9679;',
    'video': '&#9654;'
}

colors = {
    '1': [
        '#fff',
        '#fffe50',
        '#ffca2d',
        '#ff9f2d',
        '#ff6602',
        '#f00'
    ],
    '2': [
        '#e6fdff',
        '#50d7ff',
        '#2db0ff',
        '#2d87ff',
        '#0234ff',
        '#00f'
    ],
    '3': [
        '#ffe6fd',
        '#ff50ed',
        '#f92dff',
        '#e32dff',
        '#bc02ff',
        '#8d02ff'
    ],
}

# ~~ post format ~~
#  {
#    "_id": {
#      "$oid": "5ab919042076215038537c58"
#    },
#    "impressionOrder": 4,
#    "impressionTime": {
#      "$date": "2018-01-10T07:01:18.000Z"
#    },
#    "profileName": "Antonietta",
#    "profileAlign": "left",
#    "publicationTime": {
#      "$date": "2018-01-09T17:19:24.000Z"
#    },
#    "visualizationDiff": 49314,
#    "postId": "2166240993390135",
#    "utype": "post",
#    "displayName": "Il Fatto Quotidiano",
#    "externals": 1,
#    "timelineId": "61c77482e39041164f8ed7dde112cbf3e05032dc",
#    "permaLink": "/ilFattoQuotidiano/posts/2166240993390135",
#    "rtotal": "315",
#    "comments": "205",
#    "shares": "20",
#    "id": "994bb8641ab8f68918f8be9079a221708e04c888",
#    "by": "fbtrex",
#    "publisherName": "Il Fatto Quotidiano",
#    "publisherOrientation": "M5S",
#    "observed": 2
#  },


data = {}

def get_formatted_entry(entry_data, symbols, colors):
    """Returns the formatted entry."""
    #pprint.pprint(entry_data)
    retstr = ''
    staleness = entry_data['staleness']
    if staleness > 3:
        staleness = 3
    distance = entry_data['distance']
    color = colors[str(staleness)][distance]
    symbol = entry_data['type']
    #pprint.pprint(color)
    #pprint.pprint(symbol)
    retstr += "<span style=\"color: {}\">{}</span>".format(color, symbols[symbol])
    return retstr


def order_data_types(types):
    """Return an array of alphabetically sorted data types."""
    #pprint.pprint(types)
    #pprint.pprint(types.sort())
    return types.sort()

def build_heatmap_csv_string(heatmap_types, data_types):
    """Return a comma separated string of heatmap values sorted by type."""
    retstr = ''
    for post_type in data_types:
        retstr += "{},".format(heatmap_types[post_type])
    return retstr

def get_date(date_string):
    """Return a date in the format YYYYMMDDHH."""
    return iso8601.parse_date(date_string).strftime('%Y%m%d%H')

def get_distance(profile_align, publisher_align):
    """Returns an integer distance between profile and publisher political orientation."""
    align = {
        'left': -2,
        'center-left': -1,
        'undecided': 0,
        'M5S': 1,
        'right': 2,
        'far-right': 3
    }
    profile_value = align[profile_align]
    publisher_value = align[publisher_align]
    if profile_value == publisher_value:
        return 0
    elif profile_value > publisher_value:
        return profile_value - publisher_value
    else:
        return publisher_value - profile_value


# carica il file
src = json.load(codecs.open(filename,'rU','utf-8'))

# parsa le entry
for post in src:
    if post['impressionOrder'] > 9:
        continue
    else:
        d = get_date(post['impressionTime']['$date'])
        u = post['profileName']
        o = {}
        o['type'] = post['utype']
        o['distance'] = get_distance(post['profileAlign'], post['publisherOrientation'])
        o['staleness'] = post['observed']
        if not u in data:
            data[u] = {}
        if not d in data[u]:
            data[u][d] = []
        # insert the hash to the specific index
        data[u][d].insert(int(post['impressionOrder']), o)
        ##pprint.pprint(o)

# FIXME: sarebbe da fare con un template engine come jinja2 o simili
print("<html><head>\
<style>\
td { max-width: 1em !important; vertical-align: top; }\
th { writing-mode: vertical-rl; font-weight: thin; font-size: 0.7em; color: #aaa }\
body { background-color: #666; color: #fff; font-family: monospace; }\
p { font-family: sans-serif; }\
</style></head><body>")

for sym in symbols:
    print("<p>{} {}</p>".format(symbols[sym], sym))

print("<p>Colors, political distance increasing from left to right:</p>")

for color_shade in sorted(colors.keys()):
    print("<p>Posts viewed {} times: ".format(color_shade))
    for specific_color in colors[color_shade]:
        print("<span style=\"background-color: {}\">&nbsp;&nbsp;&nbsp;</span>".format(specific_color))

print("<p></p><h1>Data</h1>")

for user in data:
    print("<table><thead><th>Utente</th>")
    dates = sorted(data[user].keys())
    for date in dates:
        print("<th>{}</th>".format(date))
    print("</thead><tr><td style=\"width: 100% !important; max-width: 100% !important\">{}</td>".format(user))
    for date in dates:
        print("<td>")
        for entry in data[user][date]:
            print(get_formatted_entry(entry, symbols, colors))
        print("</td>")
    print("</tr></table>")

print("</body></html>")
#pprint.pprint(data)
	# encoding: utf-8

	import json
	import pprint
	import codecs
	import iso8601
	import datetime

	# this is the json from https://elezioni.tracking.exposed/campaign/opendata/
	filename = 'archive.json'

	symbols = {
	# 'photo': '🖼',
	# 'post': '🖹',
	# 'video': '🎞'
	'photo': '■',
	'post': '●',
	'video': '▶'
	}

	colors = {
	'1': [
	'#fff',
	'#fffe50',
	'#ffca2d',
	'#ff9f2d',
	'#ff6602',
	'#f00'
	],
	'2': [
	'#e6fdff',
	'#50d7ff',
	'#2db0ff',
	'#2d87ff',
	'#0234ff',
	'#00f'
	],
	'3': [
	'#ffe6fd',
	'#ff50ed',
	'#f92dff',
	'#e32dff',
	'#bc02ff',
	'#8d02ff'
	],
	}

	# ~~ post format ~~
	# {
	# "_id": {
	# "$oid": "5ab919042076215038537c58"
	# },
	# "impressionOrder": 4,
	# "impressionTime": {
	# "$date": "2018-01-10T07:01:18.000Z"
	# },
	# "profileName": "Antonietta",
	# "profileAlign": "left",
	# "publicationTime": {
	# "$date": "2018-01-09T17:19:24.000Z"
	# },
	# "visualizationDiff": 49314,
	# "postId": "2166240993390135",
	# "utype": "post",
	# "displayName": "Il Fatto Quotidiano",
	# "externals": 1,
	# "timelineId": "61c77482e39041164f8ed7dde112cbf3e05032dc",
	# "permaLink": "/ilFattoQuotidiano/posts/2166240993390135",
	# "rtotal": "315",
	# "comments": "205",
	# "shares": "20",
	# "id": "994bb8641ab8f68918f8be9079a221708e04c888",
	# "by": "fbtrex",
	# "publisherName": "Il Fatto Quotidiano",
	# "publisherOrientation": "M5S",
	# "observed": 2
	# },


	data = {}

	def get_formatted_entry(entry_data, symbols, colors):
	"""Returns the formatted entry."""
	#pprint.pprint(entry_data)
	retstr = ''
	staleness = entry_data['staleness']
	if staleness > 3:
	staleness = 3
	distance = entry_data['distance']
	color = colors[str(staleness)][distance]
	symbol = entry_data['type']
	#pprint.pprint(color)
	#pprint.pprint(symbol)
	retstr += "<span style=\"color: {}\">{}</span>".format(color, symbols[symbol])
	return retstr



	def order_data_types(types):
	"""Return an array of alphabetically sorted data types."""
	#pprint.pprint(types)
	#pprint.pprint(types.sort())
	return types.sort()

	def build_heatmap_csv_string(heatmap_types, data_types):
	"""Return a comma separated string of heatmap values sorted by type."""
	retstr = ''
	for post_type in data_types:
	retstr += "{},".format(heatmap_types[post_type])
	return retstr

	def get_date(date_string):
	"""Return a date in the format YYYYMMDDHH."""
	return iso8601.parse_date(date_string).strftime('%Y%m%d%H')

	def get_distance(profile_align, publisher_align):
	"""Returns an integer distance between profile and publisher political orientation."""
	align = {
	'left': -2,
	'center-left': -1,
	'undecided': 0,
	'M5S': 1,
	'right': 2,
	'far-right': 3
	}
	profile_value = align[profile_align]
	publisher_value = align[publisher_align]
	if profile_value == publisher_value:
	return 0
	elif profile_value > publisher_value:
	return profile_value - publisher_value
	else:
	return publisher_value - profile_value


	# carica il file
	src = json.load(codecs.open(filename,'rU','utf-8'))

	# parsa le entry
	for post in src:
	if post['impressionOrder'] > 9:
	continue
	else:
	d = get_date(post['impressionTime']['$date'])
	u = post['profileName']
	o = {}
	o['type'] = post['utype']
	o['distance'] = get_distance(post['profileAlign'], post['publisherOrientation'])
	o['staleness'] = post['observed']
	if not u in data:
	data[u] = {}
	if not d in data[u]:
	data[u][d] = []
	# insert the hash to the specific index
	data[u][d].insert(int(post['impressionOrder']), o)
	##pprint.pprint(o)

	# FIXME: sarebbe da fare con un template engine come jinja2 o simili
	print("<html><head>\
	<style>\
	td { max-width: 1em !important; vertical-align: top; }\
	th { writing-mode: vertical-rl; font-weight: thin; font-size: 0.7em; color: #aaa }\
	body { background-color: #666; color: #fff; font-family: monospace; }\
	p { font-family: sans-serif; }\
	</style></head><body>")

	for sym in symbols:
	print("<p>{} {}</p>".format(symbols[sym], sym))

	print("<p>Colors, political distance increasing from left to right:</p>")

	for color_shade in sorted(colors.keys()):
	print("<p>Posts viewed {} times: ".format(color_shade))
	for specific_color in colors[color_shade]:
	print("<span style=\"background-color: {}\">   </span>".format(specific_color))

	print("<p></p><h1>Data</h1>")

	for user in data:
	print("<table><thead><th>Utente</th>")
	dates = sorted(data[user].keys())
	for date in dates:
	print("<th>{}</th>".format(date))
	print("</thead><tr><td style=\"width: 100% !important; max-width: 100% !important\">{}</td>".format(user))
	for date in dates:
	print("<td>")
	for entry in data[user][date]:
	print(get_formatted_entry(entry, symbols, colors))
	print("</td>")
	print("</tr></table>")

	print("</body></html>")
	#pprint.pprint(data)