Load data from ArcGIS Server Rest API into geojson-p2p-db

index.js

var hyperlog = require('hyperlog')
var level = require('level-browserify')
var gjdb = require('geojson-p2p-db')
var FeatureService = require('featureservice')
var request = require('xhr-request')
var series = require('run-series')
var waterfall = require('run-waterfall')
var Terraformer = require('terraformer')
Terraformer.ArcGIS = require('terraformer-arcgis-parser')

var db = {
  log: level('/tmp/geojson-p2p/log'),
  index: level('/tmp/geojson-p2p/index')
}
var fdstore = require('fd-chunk-store')
var storefile = '/tmp/geojson-p2p/kdb'

var gj = gjdb({
  log: hyperlog(db.log, { valueEncoding: 'json' }),
  db: db.index,
  store: fdstore(4096, storefile)
})

var service = new FeatureService('https://sjcgis.org/arcgis/rest/services/Polaris/Buildings/MapServer/0')

gj.ready(function () {
  var tasks = []
  service.pages(function (err, urls) {
    if (err) throw err
    urls.forEach(function (url) {
      tasks.push(createExtractTask(url.req))
    })
    series(tasks, function (err, data) {
      if (err) throw err
      var rows = data.reduce(function (acc, i) {
        return acc.concat(i).map(function (f) {
          return { type: 'put', value: f }
        })
      })
      gj.batch(rows, function (err) {
        if (err) throw err
      })
    })
  })
})

function transformData (json, cb) {
  var gj = json.features.map(function (f) {
    return Terraformer.ArcGIS.parse(f)
  })
  cb(null, gj)
}

function createExtractTask (url) {
  return function (cb) {
    waterfall([
      function (cb) {
        request(url, {
          json: true
        }, cb)
      },
      function (body, res, cb) {
        transformData(body, cb)
      }
    ], function (err, res) {
      if (err) cb(err)
      cb(null, res)
    })
  }
}