Skip to content

Instantly share code, notes, and snippets.

@andylolz andylolz/.block forked from mbostock/.block
Last active May 21, 2019

Embed
What would you like to do?
IATI traceability
license: gpl-3.0
/venv/
*.rdb
iatikit.ini

Click to drag; scroll to zoom. Red links are downstream; black links are upstream.

Link to full screen demo

Some initial takehomes:

  • You can see "clouds" of black arrows around the funders who obligate their recipients to publish IATI data: DFID; Dutch MFA (minbuza_nl); Belgian MFA (be_dgd).
  • It looks like some big consultancies publish links to their implementers – see the clouds of red arrows around Crown Agents and MannionDaniels
  • There are more black (upstream) links than red (downstream) links. That’s to be expected – IATI "STRONGLY RECOMMENDS" publishing upstream links, but downstream links are recorded "if possible"
  • The graph is incomplete! Lots of big funders are missing completely, and it’s likely that most of their implementers don’t publish IATI data at all.

Data used, and the code to generate it from IATI is available in this gist.

Implemented in D3.js.

Forked from Mike Bostock’s gist.

<!DOCTYPE html>
<meta charset="utf-8">
<style>
.link {
fill: #666;
stroke: #666;
stroke-width: 1.5px;
}
.link.downstream {
stroke: red;
fill: red;
}
.nodes circle {
fill: #ccc;
stroke: #333;
stroke-width: 1.5px;
}
text {
font: 10px sans-serif;
pointer-events: none;
text-shadow: 0 1px 0 #fff, 1px 0 0 #fff, 0 -1px 0 #fff, -1px 0 0 #fff;
}
</style>
<svg width="100%"></svg>
<script src="//d3js.org/d3.v4.min.js"></script>
<script>
var width = window.innerWidth,
height = window.innerHeight;
var svg = d3.select("svg")
.attr("width", width)
.attr("height", height);
var radius = 5;
d3.csv("links.csv", function (links) {
var nodes = {};
links.forEach(function (link) {
link.source = nodes[link.source] || (nodes[link.source] = {name: link.source});
link.target = nodes[link.target] || (nodes[link.target] = {name: link.target});
});
nodes = d3.values(nodes);
//set up the simulation and add forces
var simulation = d3.forceSimulation()
.nodes(nodes);
var link_force = d3.forceLink(links)
.id(function(d) { return d.name; });
var charge_force = d3.forceManyBody()
.strength(-100);
var center_force = d3.forceCenter(width / 2, height / 2);
simulation
.force("charge_force", charge_force)
.force("center_force", center_force)
.force("links",link_force)
;
//add tick instructions:
simulation.on("tick", tickActions );
//add encompassing group for the zoom
var g = svg.append("g")
.attr("class", "everything");
//draw lines for the links
var path = g.append("g")
.attr("class", "links")
.attr("fill", function(d) { return "none"; })
.attr("stroke-width", function(d) { return "1.5px"; })
.attr("stroke", function(d) { return "#666"; })
.selectAll("path")
.data(links)
.enter().append("path")
.attr("class", function(d) { return "link " + d.type; })
.attr("marker-end", function(d) { return "url(#" + d.type + ")"; });
//draw circles for the nodes
var node = g.append("g")
.attr("class", "nodes")
.selectAll("circle")
.data(nodes)
.enter()
.append("circle")
.attr("r", radius)
.attr("fill", 'blue');
var text = g.append("g")
.attr("class", "text")
.selectAll("circle")
.data(nodes)
.enter().append("text")
.attr("x", 8)
.attr("y", ".31em")
.text(function(d) { return d.name; });
//add drag capabilities
var drag_handler = d3.drag()
.on("start", drag_start)
.on("drag", drag_drag)
.on("end", drag_end);
drag_handler(node);
//add zoom capabilities
var zoom_handler = d3.zoom()
.on("zoom", zoom_actions);
zoom_handler(svg);
/** Functions **/
//Drag functions
//d is the node
function drag_start(d) {
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
}
//make sure you can't drag the circle outside the box
function drag_drag(d) {
d.fx = d3.event.x;
d.fy = d3.event.y;
}
function drag_end(d) {
if (!d3.event.active) simulation.alphaTarget(0);
d.fx = null;
d.fy = null;
}
//Zoom functions
function zoom_actions(){
g.attr("transform", d3.event.transform)
}
function tickActions() {
path.attr("d", function(d) {
var dx = d.target.x - d.source.x,
dy = d.target.y - d.source.y,
dr = Math.sqrt(dx * dx + dy * dy),
theta = Math.atan2(dy, dx) + Math.PI / 7.85,
d90 = Math.PI / 2,
dtxs = d.target.x - 6 * Math.cos(theta),
dtys = d.target.y - 6 * Math.sin(theta);
return "M" + d.source.x + "," + d.source.y + "A" + dr + "," + dr + " 0 0 1," + d.target.x + "," + d.target.y + "A" + dr + "," + dr + " 0 0 0," + d.source.x + "," + d.source.y + "M" + dtxs + "," + dtys + "l" + (3.5 * Math.cos(d90 - theta) - 10 * Math.cos(theta)) + "," + (-3.5 * Math.sin(d90 - theta) - 10 * Math.sin(theta)) + "L" + (dtxs - 3.5 * Math.cos(d90 - theta) - 10 * Math.cos(theta)) + "," + (dtys + 3.5 * Math.sin(d90 - theta) - 10 * Math.sin(theta)) + "z";
});
//update circle positions each tick of the simulation
node
.attr("cx", function(d) { return d.x; })
.attr("cy", function(d) { return d.y; });
text
.attr("x", function(d) { return d.x; })
.attr("y", function(d) { return d.y; });
}
});
</script>
import csv
from collections import defaultdict
from itertools import zip_longest
import iatikit
import redis
r = redis.Redis()
for pub in iatikit.data().publishers:
for act in pub.activities:
if act.id is not None:
_ = r.set(act.id, pub.name)
store = []
for pub in iatikit.data().publishers:
for act in pub.activities:
receiver_ids = act.etree.xpath(
'transaction/receiver-org/@receiver-activity-id')
provider_ids = act.etree.xpath(
'transaction/provider-org/@provider-activity-id')
links = []
for provider_id, receiver_id in zip_longest(provider_ids, receiver_ids,
fillvalue=act.id):
if not receiver_id:
continue
if not provider_id:
continue
receiver = r.get(receiver_id)
if receiver:
receiver = receiver.decode()
else:
continue
provider = r.get(provider_id)
if provider:
provider = provider.decode()
else:
continue
if provider == pub.name:
direction = 'downstream'
if receiver == pub.name:
direction = 'upstream'
if receiver != pub.name or provider != pub.name:
tup = (direction, provider, receiver)
links.append(tup)
links = list(set(links))
for link in links:
print(link)
store.append(link)
dedupe_store = defaultdict(int)
for link in store:
dedupe_store[link] += 1
dedupe_store = [{
'type': k[0],
'source': k[1],
'target': k[2],
'weight': v,
} for k, v in dedupe_store.items()]
fieldnames = ['source', 'target', 'type', 'weight']
with open('links.csv', 'w') as handler:
writer = csv.DictWriter(handler, fieldnames=fieldnames)
writer.writeheader()
_ = [writer.writerow(x) for x in dedupe_store]
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.