Stream Features from ArcGIS

arcgis-feature-stream.js

var request = require('request');
var debug = require('debug')('arcgis-feature-stream');
var noms =  require('noms').obj;

function arcgisFeatureStream(options) {
  var baseUrl = options.baseUrl;
  var idField = options.idField;
  var start = options.start || 0;
  var step = options.step || 1000;

  return createGetFeatureStream(options.baesUrl, options.idField, start, step);
}

function getFeaturesBetween(baseUrl, idField, start, end, callback) {
  var query = '&where=' + idField + ' >= ' + start + ' and ' + idField + ' < ' + end;
  var url = baseUrl + query;

  debug('executing query', url);

  request({ url: url, json: true }, function (body, res) {
    callback(res.body.features);
  });
}

function createGetFeatureStream(baseUrl, idField, start, step) {
  return noms(function (done) {
    var stream = this;

    debug('getting features from %s to %s', start, start + step);

    getFeaturesBetween(baseUrl, idField, start, start + step, function (features) {
      if (!features) {
        debug('done writing features');
        stream.push(null);
        return done();
      }

      debug('writing %s features', features.length);

      // Write individual features to stream.
      features.forEach(function (feature) {
        stream.push(geoJsonFeature);
      });

      start += step;
      done();
    });
  });
}

module.exports = arcgisFeatureStream;

example.js

var fs = require('fs');
var parser = require('terraformer-arcgis-parser');
var geojsonStream = require('geojson-stream');
var arcgisFeatureStream = require('./arcgis-feature-stream');

var options = {
  baseUrl: 'https://www.sciencebase.gov/arcgis/rest/services/Catalog/55785759e4b032353cbff779/MapServer/0/query?geometryType=esriGeometryEnvelope&spatialRel=esriSpatialRelIntersects&returnGeometry=true&returnIdsOnly=false&returnCountOnly=false&returnZ=false&returnM=false&returnDistinctValues=false&f=json',
  idField: 'FID'
};

// Start by creating a stream of features from ArcGIS
arcgisFeatureStream(options)
  // Convert to GeoJSON
  .pipe(esriToGeoJson())
  // Stringify as a FeatureCollection, with start and end text
  .pipe(geojsonStream.stringify()))
  // Write output to a file
  .pipe(fs.createWriteStream('output.geojson'));

function esriToGeoJson() {
  return through2.obj(function (feature, enc, callback) {
    var geojson = parser.parse(feature);
    this.push(geojson);
    return callback();
  });
}