Bevry member geolocation consolidation script

README.md

Bevry member geolocation consolidation script

This is an example script of doing some advanced data operations with high-level APIs

• chainy.coffee is my upcoming way of doing this, using chainy
• desired.js is my desired way of doing this, not yet possible
• highland.js is a mostly-working implementation using highland

The goal of the script is to:

1. Fetch the public members from multiple github organisations
2. Fetch the members complete profiles
3. Convert each member's location to lat long coordinates
4. Convert the results into a single geojson file for all

If you know any better ways of doing this, or any libraries that could help accomplish this with super nice APIs, or whatever, then do leave your feedback, suggestions, and shoutouts in the comments.

CC0 Licensed. Attribution appreciated.

chainy.coffee

{Chainy} = require('chainy')

Chainy.create()
	.add(['bevry','browserstate','ideashare','interconnectapp','docpad'])

	.request (org) ->
		return "https://api.github.com/orgs/#{org}/public_members"

	.flatten().count()

	.removeDuplicates('id').count()

	.request (user) ->
		return user.url

	.hasField('location').count()

	.map (user, complete) ->
		Chainy.create()
			.add([user])
			.request (user) ->
				return "https://api.tiles.mapbox.com/v3/examples.map-zr0njcqy/geocode/#{user.location}.json"
			.map (geo) ->
				result = geo.results[0][0]
				user.coordinates = [result.lon, result.lat]  if result
				return geo
			.fn -> complete(null, user)

	.hasField('coordinates').count()

	.map (user) ->
		return {
			type: 'Feature'
			properties:
				githubUsername: user.login
			geometry:
				type: 'Point'
				coordinates: user.coordinates
		}

	.replace (data) ->
		return {
			type: 'FeatureCollection'
			features: data
		}

	.log()

	.replace (data) ->
		JSON.stringify(data, null, '\t')

	.pipe(
		require('fs').createWriteStream('./out.geojson')
	)

desired.js

// This is my wishful way of doing it

var _ = require('whatever');

_()
	// Add the oranisations
	.add(['bevry', 'browserstate', 'interconnectapp', 'docpad', 'ideashare'])
	
	// Fetch the members of the organisations
	.request(function(org){
		return 'https://api.github.com/orgs/'+org+'/public_members';
	})

	// Remove duplicate members as they could be in multiple orgs
	.removeDuplicates('id')

	// Fetch their details
	.request(function(user){
		return user.url;
	})

	// Reject those with no location
	.hasField('location')

	// Inject the coordinates field
	.map(function(user, next){
		return _()
			.add(user)
			.request(function(user){
				return 'https://api.tiles.mapbox.com/v3/examples.map-zr0njcqy/geocode/'+user.location+'.json';
			})
			.map(function(geocode){
				var result = geocode.results[0][0];
				user.coordinates = [result.lon, result.lat];
			})
			.done(next);
	})

	// Reject those with no coordinates
	.hasField('coordinates')

	// Convert into a geojson cormat
	.map(function(user){
		return {
			type: "Feature",
			properties: {
				githubUsername: user.login
			},
			geometry: {
				type: "Point",
				coordinates: user.coordinates
			}
		};
	})
	.collect()
	.map(function(features){
		return {
			type: "FeatureCollection",
			features: features
		};
	})
	.writeFile('./members.geojson')
	;

highland.js

// This is how you do it in highland
// npm install highland feedr

var _ = require('highland');
var feedr = require('feedr').create({cache:'preferred'});
var readFeed = _.wrapCallback(feedr.readFeed.bind(feedr));
var log = function(i){
	console.log(i);
	return i;
};
var fetchMembers = function(org){
	return readFeed({
		url: 'https://api.github.com/orgs/'+org+'/public_members',
		parse: 'json'
	});
};
var uniqueFielder = function(field){
	return function(items){
		var counts = {};
		return items.filter(function(item){
			var id = item[field];
			counts[id] = counts[id] || 0;
			++counts[id];
			return counts[id] === 1;
		});
	};
};
var expandArray = function(err, items, push, next){
	if ( err ) {
		push(err);
		next();
	}
	else {
		items.map(function(item){
			push(null, item);
		});
		next();
	}
};
var fetchDetails = function(user){
	return readFeed({
		url: user.url,
		parse: 'json'
	});
};
var fetchCoordinates = function(location, next){
	if ( !location ) {
		next(null, null);
	}
	else {
		feedr.readFeed({
			url: 'https://api.tiles.mapbox.com/v3/examples.map-zr0njcqy/geocode/'+location+'.json',
			parse: 'json'
		}, next);
	}
};
var injectCoordinates = function(err, user, push, next){
	if ( err ) {
		push(err);
		next();
	}
	else {
		fetchCoordinates(user.location, function(err, coordinates){
			user.coordinates = coordinates;
			push(err, user);
			next();
		});
	}
};

/*
var result = _(['./index.js', './package.json'])
	.map(readFile)
	.sequence()
	.invoke('toString', ['utf8'])
	.pipe(process.stdout)
	;
*/

// Fetch members
_(['bevry', 'browserstate', 'interconnectapp', 'docpad', 'ideashare'])
	.map(fetchMembers)
	.sequence()

	// Remove duplicates
	.collect()
	.map(uniqueFielder('id'))
	.consume(expandArray)

	// Fetch there complete details
	.map(fetchDetails)
	.sequence()

	// Inject the coordinates
	.consume(injectCoordinates)
	.map(function(user){
		if ( user.coordinates ) {
			var result = user.coordinates.results[0][0];
			user.coordinates = [result.lon, result.lat];
		}
		return user;
	})

	// Conect into a geojson file
	.filter(function(user){
		return user.coordinates !== null;
	})
	.map(function(user){
		return {
			type: "Feature",
			properties: {
				githubUsername: user.login
			},
			geometry: {
				type: "Point",
				coordinates: user.coordinates
			}
		};
	})
	.map(log)
	.collect()
	/* it stops working here due to the collect */
	.map(log)
	.map(function(features){
		console.log(features);
		return {
			type: "FeatureCollection",
			features: features
		};
	})
	.each(log)
	;