Create a gist now

Instantly share code, notes, and snippets.

@balupton /README.md
Last active Aug 29, 2015

What would you like to do?
Bevry member geolocation consolidation script

Bevry member geolocation consolidation script

This is an example script of doing some advanced data operations with high-level APIs

  • chainy.coffee is my upcoming way of doing this, using chainy
  • desired.js is my desired way of doing this, not yet possible
  • highland.js is a mostly-working implementation using highland

The goal of the script is to:

  1. Fetch the public members from multiple github organisations
  2. Fetch the members complete profiles
  3. Convert each member's location to lat long coordinates
  4. Convert the results into a single geojson file for all

If you know any better ways of doing this, or any libraries that could help accomplish this with super nice APIs, or whatever, then do leave your feedback, suggestions, and shoutouts in the comments.

CC0 Licensed. Attribution appreciated.

{Chainy} = require('chainy')
Chainy.create()
.add(['bevry','browserstate','ideashare','interconnectapp','docpad'])
.request (org) ->
return "https://api.github.com/orgs/#{org}/public_members"
.flatten().count()
.removeDuplicates('id').count()
.request (user) ->
return user.url
.hasField('location').count()
.map (user, complete) ->
Chainy.create()
.add([user])
.request (user) ->
return "https://api.tiles.mapbox.com/v3/examples.map-zr0njcqy/geocode/#{user.location}.json"
.map (geo) ->
result = geo.results[0][0]
user.coordinates = [result.lon, result.lat] if result
return geo
.fn -> complete(null, user)
.hasField('coordinates').count()
.map (user) ->
return {
type: 'Feature'
properties:
githubUsername: user.login
geometry:
type: 'Point'
coordinates: user.coordinates
}
.replace (data) ->
return {
type: 'FeatureCollection'
features: data
}
.log()
.replace (data) ->
JSON.stringify(data, null, '\t')
.pipe(
require('fs').createWriteStream('./out.geojson')
)
// This is my wishful way of doing it
var _ = require('whatever');
_()
// Add the oranisations
.add(['bevry', 'browserstate', 'interconnectapp', 'docpad', 'ideashare'])
// Fetch the members of the organisations
.request(function(org){
return 'https://api.github.com/orgs/'+org+'/public_members';
})
// Remove duplicate members as they could be in multiple orgs
.removeDuplicates('id')
// Fetch their details
.request(function(user){
return user.url;
})
// Reject those with no location
.hasField('location')
// Inject the coordinates field
.map(function(user, next){
return _()
.add(user)
.request(function(user){
return 'https://api.tiles.mapbox.com/v3/examples.map-zr0njcqy/geocode/'+user.location+'.json';
})
.map(function(geocode){
var result = geocode.results[0][0];
user.coordinates = [result.lon, result.lat];
})
.done(next);
})
// Reject those with no coordinates
.hasField('coordinates')
// Convert into a geojson cormat
.map(function(user){
return {
type: "Feature",
properties: {
githubUsername: user.login
},
geometry: {
type: "Point",
coordinates: user.coordinates
}
};
})
.collect()
.map(function(features){
return {
type: "FeatureCollection",
features: features
};
})
.writeFile('./members.geojson')
;
// This is how you do it in highland
// npm install highland feedr
var _ = require('highland');
var feedr = require('feedr').create({cache:'preferred'});
var readFeed = _.wrapCallback(feedr.readFeed.bind(feedr));
var log = function(i){
console.log(i);
return i;
};
var fetchMembers = function(org){
return readFeed({
url: 'https://api.github.com/orgs/'+org+'/public_members',
parse: 'json'
});
};
var uniqueFielder = function(field){
return function(items){
var counts = {};
return items.filter(function(item){
var id = item[field];
counts[id] = counts[id] || 0;
++counts[id];
return counts[id] === 1;
});
};
};
var expandArray = function(err, items, push, next){
if ( err ) {
push(err);
next();
}
else {
items.map(function(item){
push(null, item);
});
next();
}
};
var fetchDetails = function(user){
return readFeed({
url: user.url,
parse: 'json'
});
};
var fetchCoordinates = function(location, next){
if ( !location ) {
next(null, null);
}
else {
feedr.readFeed({
url: 'https://api.tiles.mapbox.com/v3/examples.map-zr0njcqy/geocode/'+location+'.json',
parse: 'json'
}, next);
}
};
var injectCoordinates = function(err, user, push, next){
if ( err ) {
push(err);
next();
}
else {
fetchCoordinates(user.location, function(err, coordinates){
user.coordinates = coordinates;
push(err, user);
next();
});
}
};
/*
var result = _(['./index.js', './package.json'])
.map(readFile)
.sequence()
.invoke('toString', ['utf8'])
.pipe(process.stdout)
;
*/
// Fetch members
_(['bevry', 'browserstate', 'interconnectapp', 'docpad', 'ideashare'])
.map(fetchMembers)
.sequence()
// Remove duplicates
.collect()
.map(uniqueFielder('id'))
.consume(expandArray)
// Fetch there complete details
.map(fetchDetails)
.sequence()
// Inject the coordinates
.consume(injectCoordinates)
.map(function(user){
if ( user.coordinates ) {
var result = user.coordinates.results[0][0];
user.coordinates = [result.lon, result.lat];
}
return user;
})
// Conect into a geojson file
.filter(function(user){
return user.coordinates !== null;
})
.map(function(user){
return {
type: "Feature",
properties: {
githubUsername: user.login
},
geometry: {
type: "Point",
coordinates: user.coordinates
}
};
})
.map(log)
.collect()
/* it stops working here due to the collect */
.map(log)
.map(function(features){
console.log(features);
return {
type: "FeatureCollection",
features: features
};
})
.each(log)
;
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment