Geosupport w/ JS and node-ffi

README.md

Geocoding 10,000 addresses a second with NYC's Geosupport library and Node FFI

Following on Chris Whong's excellent writeup of how to make calls directly to NYC's Geosupport client and this first attempt at generalizing it, here's a way that let me geocode about 10,000 addresses a second on Ubuntu using Node FFI.

Note: this assumes Ubuntu - other Linux is probably fine but may need adjustments.

First, install the basics:

# Update, install Node and unzip (if needed)
sudo apt-get update --yes && sudo apt-get upgrade --yes
sudo apt-get install unzip nodejs npm

# Fix legacy node naming nonsense on Ubuntu (if needed)
sudo ln -s `which nodejs` /usr/bin/node

# Start a package (optional)
npm init

# Install node-ffi and proj4
# proj4 is optional, only for reprojecting coordinates to lng/lat
npm install --save ffi proj4

# Download Geosupport Desktop for Linux
wget http://www1.nyc.gov/assets/planning/download/zip/data-maps/open-data/gdelx_16c.zip

# Unzip the contents
unzip gdelx_16c.zip

# Set env variables to the full paths to lib/ and fls/
export LD_LIBRARY_PATH="$(pwd)/version-16c_16.3/lib/"
export GEOFILES="$(pwd)/version-16c_16.3/fls/"

That's it! Now you're ready to geocode using a script like the attached geocode.js, which wraps libgeo.so so you can call it directly by passing two Buffers.

That example assumes you have a JSON file, addresses.json, with addresses like so:

[
  {
    "HouseNumber": "86-54",
    "StreetName": "WINCHESTER BOULEVARD",
    "ZipCode": "11427",
    "BoroughCode": 4
  },
  ...
]

but you could use the same logic in any context.

This script also reprojects the results, which are in NY Long Island State Plane, into latitude/longitude using proj4.

To handle various error codes, or to use other Geosupport functions besides address geocoding (there are lots), you would only need to update the working area inputs (lines 25-26) and how you parse them afterwards (lines 45 - 55) according to the fields in the User guide (p. 570+).

Note: this example assumes you have a borough code rather than a borough name. NYC borough codes are:

Manhattan: 1
Bronx: 2
Brooklyn: 3
Queens: 4
Staten Island: 5

geocode.js

var proj4 = require("proj4"),
    ffi = require("ffi");

var lib = ffi.Library("version-16c_16.3/lib/libgeo.so", {
  geo: [ "void", [ "char *", "char *" ] ]
});

// Faster to reuse the same buffers
var wa1Buffer = new Buffer(1200),
    wa2Buffer = new Buffer(1200);

var reproject = proj4('PROJCS["NAD_1983_StatePlane_New_York_Long_Island_FIPS_3104_Feet",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Lambert_Conformal_Conic"],PARAMETER["False_Easting",984250.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-74.0],PARAMETER["Standard_Parallel_1",40.66666666666666],PARAMETER["Standard_Parallel_2",41.03333333333333],PARAMETER["Latitude_Of_Origin",40.16666666666666],UNIT["Foot_US",0.3048006096012192]]').inverse;

// Load in some addresses
var addresses = require("./addresses.json"),
    
// Geocode each address
addresses.forEach(geocode);

// Optional: write the results to a file or something!

function geocode(address) {

  // Construct COW string for work area 1
  var wa1 = ("1 " + rightpad(address.HouseNumber, 16) + rightpad("", 38) + address.BoroughCode + rightpad("", 10) + rightpad(address.AltStreetName || address.StreetName, 32) + rightpad("", 113) + "C" + rightpad(address.ZipCode, 5)).toUpperCase(),
      wa2, returnCode, x, y;

  // Reset work areas
  wa1Buffer.fill(" ");
  wa2Buffer.fill(" ");

  // Write to work area 1
  wa1Buffer.write(wa1, "utf8");

  // Geocode
  lib.geo(wa1Buffer, wa2Buffer);

  // Update with results
  wa1 = wa1Buffer.toString();
  wa2 = wa2Buffer.toString();

  returnCode = wa1.substring(716, 718);

  // Success
  if (returnCode === "00") {

    // State Plane coords
    x = +(wa2.substring(125, 132));
    y = +(wa2.substring(132, 139));

    address.tract = +(wa2.substring(223, 229));
    address.block = +(wa2.substring(229, 233));
    address.lngLat = reproject([x, y]);

  }
  // Optional: handle errors, return code "EE", etc.

  return address;

}

function rightpad(str, width) {

  str = str.toString();

  while (str.length < width) {
    str += " ";
  }

  if (str.length > width) {
    return str.slice(0, width);
  }

  return str;

}