Skip to content

Instantly share code, notes, and snippets.

@jenningsanderson
Created November 29, 2018 02:40
Show Gist options
  • Save jenningsanderson/b04ac2cc68a2cf79a4c4cb2c1b16c96d to your computer and use it in GitHub Desktop.
Save jenningsanderson/b04ac2cc68a2cf79a4c4cb2c1b16c96d to your computer and use it in GitHub Desktop.
var osmium = require('osmium');
var _ = require("lodash")
var count = 0;
var restrictions=0;
var relHandler = new osmium.Handler();
var geomHandler = new osmium.Handler();
var nodes = []
var nodeLocations = {}
var ways = []
var wayLocations = {}
var relations = {} //Will just store this in memory after the first pass
relHandler.on('relation', function(relation) {
count++;
if (count%100==0){process.stderr.write("\r"+count)}
if (relation.tags("type")==='restriction'){
restrictions++;
try{
var members = relation.members();
if (members.length ){
var via = members.filter((m)=>{return m.role=='via'})[0]
var from = members.filter((m)=>{return m.role=='from'})[0]
var to = members.filter((m)=>{return m.role=='to'})[0]
if(via){
if(via.type=='n'){
nodes.push(via.ref)
var ref = {'type':'n','ref':via.ref}
}else if(via.type=='w'){
ways.push(via.ref)
var ref = {'type':'w','ref':via.ref}
}
}else if(from){
if(from.type=='n'){
nodes.push(from.ref)
var ref = {'type':'n','ref':from.ref}
}else if(from.type=='w'){
ways.push(from.ref)
var ref = {'type':'w','ref':from.ref}
}
}else if(to){
if(to.type=='n'){
nodes.push(to.ref)
var ref = {'type':'n','ref':to.ref}
}else if(to.type=='w'){
ways.push(to.ref)
var ref = {'type':'w','ref':to.ref}
}
}else{
if(members[0].type=='n'){
nodes.push(members[0].ref)
var ref = {'type':'n','ref':members[0].ref}
}else if(members[0].type=='w'){
ways.push(members[0].ref)
var ref = {'type':'w','ref':members[0].ref}
}
}
relations[relation.id] = {
i: relation.id,
u: relation.uid,
h: relation.user,
t: relation.timestamp_seconds_since_epoch,
c: relation.changeset,
v: relation.version,
tags: relation.tags(),
ref: ref
}
}
}catch(e){
console.warn(e)
}
}
});
var nodeCount = 0;
geomHandler.on('node',function(node){
++nodeCount;
if (nodeCount%1000000==0){process.stderr.write("\rread "+nodeCount/1000000+"M nodes, remaining geometries: "+nodes.length+" ")}
if (node.id > curNode){
while (curNode < node.id){
console.warn("Node" + curNode + " not in file")
curNode = nodes.pop()
}
}
if (node.id==curNode){
nodeLocations[node.id] = node.coordinates
//last step
curNode = nodes.pop();
}
})
var wayCount = 0;
geomHandler.on('way',function(way){
++wayCount;
if (wayCount%1000==0){process.stderr.write("\r"+wayCount/1000+"K ways")}
if (way.id > curWay){
while (curWay < node.id){
console.warn("Way" + curWay + " not in file")
curWay = nodes.pop()
}
}
if (way.id==curWay){
var repNode = way.node_refs(0)
nodes.push( repNode ) //push that node, mark it.
wayLocations[curWay] = repNode
//last step
curWay = ways.pop()
}
})
geomHandler.on('done', function() {
//Now matching geometries and creating geojson
Object.keys(relations).forEach(function(relId){
try{
var thisRel = relations[relId]
var geom;
if (thisRel.ref.type=='n'){
geom = {'type':"Point",'coordinates':[nodeLocations[thisRel.ref.ref].lon,nodeLocations[thisRel.ref.ref].lat]}
}
if (thisRel.ref.type=='w'){
repNode = wayLocations[thisRel.ref.ref]
geom = {'type':"Point",'coordinates':[nodeLocations[repNode].lon, nodeLocations[repNode].lat]}
}
var props = thisRel.tags;
delete props.ref
props['@id'] = thisRel.i
props['@user'] = thisRel.h
props['@uid'] = thisRel.u
props['@timestamp'] = thisRel.t
props['@changeset'] = thisRel.c
props['@version'] = thisRel.v
props['@tr'] = true
console.log(JSON.stringify(
{'type':"Feature",
"geometry":geom,
"properties":props}))
}catch(e){
console.warn(e)
}
})
console.warn("\nFinished, nodes left in nodes array: " + nodes.length)
})
relHandler.on('done', function() {
//Sort them... and hope that the pbf file is ordered the way we hope it is.
console.warn(`\rProcessed ${count} relations, found ${restrictions} restrictions`)
console.warn(`Nodes Involved: ${nodes.length}`)
console.warn(`Ways Involved: ${ways.length}`)
console.warn(`Missing points: ${restrictions - nodes.length - ways.length}\n`);
nodes = _.uniq(_.sortBy(nodes,function(x){return -Number(x)}));
ways = _.uniq(_.sortBy(ways,function(x){return -Number(x)}));
console.warn(`Nodes Needed: ${nodes.length}`)
console.warn(`Ways Needed: ${ways.length}`)
curNode = nodes.pop();
curWay = ways.pop();
});
/*
Runtime
*/
if (!process.argv[2]) {
console.error('Usage: \n');
console.error('\tnode extract-geojson-relations.js [OSM FILE] > [GEOJSONSEQ FILE]');
console.error("\n\n")
process.exit(1);
}
var input_filename = process.argv[2]
console.warn("----------------------------------------------------------------------")
console.warn("Stage 1: Getting ways / nodes lists from restriction relations\n")
var reader = new osmium.Reader(input_filename, {node: false, way: false, relation:true});
osmium.apply(reader, relHandler);
relHandler.end();
reader.close();
console.warn("----------------------------------------------------------------------")
console.warn("Stage 2: Getting representative NODES for " + ways.length + " ways")
reader = new osmium.Reader(input_filename, {node: false, way: true, relation:false});
osmium.apply(reader, geomHandler);
reader.close();
nodes = _.uniq(_.sortBy(nodes,function(x){return -Number(x)}));
console.warn(`\nNodes Now Needed: ${nodes.length}`)
console.warn("---------------------------------------------------------------------")
console.warn("Stage 3: Getting Node geometries")
reader = new osmium.Reader(input_filename, {node: true, way: false, relation:false});
osmium.apply(reader, geomHandler);
geomHandler.end();
reader.close();
@jenningsanderson
Copy link
Author

jenningsanderson commented Nov 29, 2018

Putting Turn Restrictions in OSM-QA-Tiles?

This is a prototype of using node-osmium to process the latest planet file and extract all turn restrictions.

Turn restrictions are then represented by a single point; this is useful for seeing where and when turn restrictions are edited on the map...

Briefly, it does the following (without using a location handler to improve performance because it's so sparse):

  1. Identifies all of the turn restrictions in an OSM file and represent it as a single point: preferably the point of the via node if it is a node, otherwise pull the first node out of one of the ways from another member to represent the point.

  2. Create a geojson feature for each point with the properties matching the osm-qa-tile schema

  3. Pipe it out to a geojsonseq file or tippecanoe!

    $ tippecanoe -l osm -Z12 -z12 -d20 -pf -pk -ps --no-duplication --no-tile-stats -b0 -pd -Pf -o turn-restrictions.mbtiles turn-restrictions.geojsonseq
    
  4. Should be able to use tile-join to add these features to osm-qa-tiles.... and now osm-qa-tiles can have turn restrictions 👍

    $ tile-join -pg -pk -f -o osm-qa-with-tr.mbtiles latest.planet.mbtiles turn-restrictions.mbtiles
    

Implementation

Takes 1-2 hours to run on full planet file, just over 1M turn restrictions. Requires parsing the planet file 3 times, but each type of OSM element only once, and doesn't use any location_handlers, so it's pretty fast.

Ultimately, it could be faster in C++, porting this node proof-of-concept shouldn't be too complicated.

Thoughts

Wouldn't this be much faster to lookup in a database? Yes, but this is meant to be a stand-alone utility with no uptime dependencies.

Next Steps

  • Can certainly expand to other types of relations
  • Could use different geometries depending on type of relation
  • Would be great to incorporate into something like osmium-export eventually

@joto
Copy link

joto commented Dec 8, 2018

Here is a suggestion: Use Osmium first to get all the turn restrictions filtered from the planet file: osmium tags-filter planet.osm.pbf r/type=restriction -o restrictions.osm.pbf. This runs in less than 10 minutes on my test system. Then run your script on the result. Should be much faster this way.

@jenningsanderson
Copy link
Author

Thanks @joto - this is a great suggestion, will test on my next run.... thinking about how the r/type=restriction filter likely works (and requires parsing the file 3x?), I imagine these are very similar processes...

Could probably be refactored to simply run the osmium tags-filter to get all of the necessary objects and then do a single run through the output converting to GeoJSON on the fly (just hold all 1M node / way locations in memory)... will test down the road.

Thinking more down the road, what are the big red flags that you see with such geometric representations of these objects, and do you see value in defining / standardizing such objects as GeoJSON for an eventual --include-restrictions tag in something like osmium-export?

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment