Skip to content

Instantly share code, notes, and snippets.

@andylolz

andylolz/.block

Forked from mbostock/.block
Last active November 8, 2023 14:21
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save andylolz/34dbc9e1d3ae04c5a331af1f978849f2 to your computer and use it in GitHub Desktop.
Save andylolz/34dbc9e1d3ae04c5a331af1f978849f2 to your computer and use it in GitHub Desktop.
Download ZIP
IATI traceability
Raw
.block
license: gpl-3.0
Raw
.gitignore
/venv/
*.rdb
iatikit.ini
Raw
README.md

Click to drag; scroll to zoom. Red links are downstream; black links are upstream.

Link to full screen demo

Some initial takehomes:

  • You can see "clouds" of black arrows around the funders who obligate their recipients to publish IATI data: DFID; Dutch MFA (minbuza_nl); Belgian MFA (be_dgd).
  • It looks like some big consultancies publish links to their implementers – see the clouds of red arrows around Crown Agents and MannionDaniels
  • There are more black (upstream) links than red (downstream) links. That’s to be expected – IATI "STRONGLY RECOMMENDS" publishing upstream links, but downstream links are recorded "if possible"
  • The graph is incomplete! Lots of big funders are missing completely, and it’s likely that most of their implementers don’t publish IATI data at all.

Data used, and the code to generate it from IATI is available in this gist.

Implemented in D3.js.

Forked from Mike Bostock’s gist.

Raw
index.html
<!DOCTYPE html>
<meta charset="utf-8">
<style>
.link {
fill: #666;
stroke: #666;
stroke-width: 1.5px;
}
.link.downstream {
stroke: red;
fill: red;
}
.nodes circle {
fill: #ccc;
stroke: #333;
stroke-width: 1.5px;
}
text {
font: 10px sans-serif;
pointer-events: none;
text-shadow: 0 1px 0 #fff, 1px 0 0 #fff, 0 -1px 0 #fff, -1px 0 0 #fff;
}
</style>
<svg width="100%"></svg>
<script src="//d3js.org/d3.v4.min.js"></script>
<script>
var width = window.innerWidth,
height = window.innerHeight;
var svg = d3.select("svg")
.attr("width", width)
.attr("height", height);
var radius = 5;
d3.csv("links.csv", function (links) {
var nodes = {};
links.forEach(function (link) {
link.source = nodes[link.source] || (nodes[link.source] = {name: link.source});
link.target = nodes[link.target] || (nodes[link.target] = {name: link.target});
});
nodes = d3.values(nodes);
//set up the simulation and add forces
var simulation = d3.forceSimulation()
.nodes(nodes);
var link_force = d3.forceLink(links)
.id(function(d) { return d.name; });
var charge_force = d3.forceManyBody()
.strength(-100);
var center_force = d3.forceCenter(width / 2, height / 2);
simulation
.force("charge_force", charge_force)
.force("center_force", center_force)
.force("links",link_force)
;
//add tick instructions:
simulation.on("tick", tickActions );
//add encompassing group for the zoom
var g = svg.append("g")
.attr("class", "everything");
//draw lines for the links
var path = g.append("g")
.attr("class", "links")
.attr("fill", function(d) { return "none"; })
.attr("stroke-width", function(d) { return "1.5px"; })
.attr("stroke", function(d) { return "#666"; })
.selectAll("path")
.data(links)
.enter().append("path")
.attr("class", function(d) { return "link " + d.type; })
.attr("marker-end", function(d) { return "url(#" + d.type + ")"; });
//draw circles for the nodes
var node = g.append("g")
.attr("class", "nodes")
.selectAll("circle")
.data(nodes)
.enter()
.append("circle")
.attr("r", radius)
.attr("fill", 'blue');
var text = g.append("g")
.attr("class", "text")
.selectAll("circle")
.data(nodes)
.enter().append("text")
.attr("x", 8)
.attr("y", ".31em")
.text(function(d) { return d.name; });
//add drag capabilities
var drag_handler = d3.drag()
.on("start", drag_start)
.on("drag", drag_drag)
.on("end", drag_end);
drag_handler(node);
//add zoom capabilities
var zoom_handler = d3.zoom()
.on("zoom", zoom_actions);
zoom_handler(svg);
/** Functions **/
//Drag functions
//d is the node
function drag_start(d) {
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
}
//make sure you can't drag the circle outside the box
function drag_drag(d) {
d.fx = d3.event.x;
d.fy = d3.event.y;
}
function drag_end(d) {
if (!d3.event.active) simulation.alphaTarget(0);
d.fx = null;
d.fy = null;
}
//Zoom functions
function zoom_actions(){
g.attr("transform", d3.event.transform)
}
function tickActions() {
path.attr("d", function(d) {
var dx = d.target.x - d.source.x,
dy = d.target.y - d.source.y,
dr = Math.sqrt(dx * dx + dy * dy),
theta = Math.atan2(dy, dx) + Math.PI / 7.85,
d90 = Math.PI / 2,
dtxs = d.target.x - 6 * Math.cos(theta),
dtys = d.target.y - 6 * Math.sin(theta);
return "M" + d.source.x + "," + d.source.y + "A" + dr + "," + dr + " 0 0 1," + d.target.x + "," + d.target.y + "A" + dr + "," + dr + " 0 0 0," + d.source.x + "," + d.source.y + "M" + dtxs + "," + dtys + "l" + (3.5 * Math.cos(d90 - theta) - 10 * Math.cos(theta)) + "," + (-3.5 * Math.sin(d90 - theta) - 10 * Math.sin(theta)) + "L" + (dtxs - 3.5 * Math.cos(d90 - theta) - 10 * Math.cos(theta)) + "," + (dtys + 3.5 * Math.sin(d90 - theta) - 10 * Math.sin(theta)) + "z";
});
//update circle positions each tick of the simulation
node
.attr("cx", function(d) { return d.x; })
.attr("cy", function(d) { return d.y; });
text
.attr("x", function(d) { return d.x; })
.attr("y", function(d) { return d.y; });
}
});
</script>
Raw
traceability.py
import csv
from collections import defaultdict
from itertools import zip_longest
import iatikit
import redis
r = redis.Redis()
for pub in iatikit.data().publishers:
for act in pub.activities:
if act.id is not None:
_ = r.set(act.id, pub.name)
store = []
for pub in iatikit.data().publishers:
for act in pub.activities:
receiver_ids = act.etree.xpath(
'transaction/receiver-org/@receiver-activity-id')
provider_ids = act.etree.xpath(
'transaction/provider-org/@provider-activity-id')
links = []
for provider_id, receiver_id in zip_longest(provider_ids, receiver_ids,
fillvalue=act.id):
if not receiver_id:
continue
if not provider_id:
continue
receiver = r.get(receiver_id)
if receiver:
receiver = receiver.decode()
else:
continue
provider = r.get(provider_id)
if provider:
provider = provider.decode()
else:
continue
if provider == pub.name:
direction = 'downstream'
if receiver == pub.name:
direction = 'upstream'
if receiver != pub.name or provider != pub.name:
tup = (direction, provider, receiver)
links.append(tup)
links = list(set(links))
for link in links:
print(link)
store.append(link)
dedupe_store = defaultdict(int)
for link in store:
dedupe_store[link] += 1
dedupe_store = [{
'type': k[0],
'source': k[1],
'target': k[2],
'weight': v,
} for k, v in dedupe_store.items()]
fieldnames = ['source', 'target', 'type', 'weight']
with open('links.csv', 'w') as handler:
writer = csv.DictWriter(handler, fieldnames=fieldnames)
writer.writeheader()
_ = [writer.writerow(x) for x in dedupe_store]
@alexjtilley
Copy link

Hello Andy!

We're planning a presentation on traceability at the IATI VCE later this month. We'd like to present a summary of current links to show how much this is being used - how easy/difficult would it be to produce an updated [links.csv] file like the one here?

Thanks!
Alex

@andylolz
Copy link
Author

andylolz commented Nov 1, 2023

Hi @alexjtilley!

Yes sure, I can look into this! Please nudge me here next week if you haven’t heard back!

@alexjtilley
Copy link

Great - thank you Andy!

Hope all is well.

@alexjtilley
Copy link

Hello Andy - just giving you a nudge in case you'd forgotten about this! Cheers, Alex

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment