dmfenton

//clean shutdown
process.on('SIGINT', () => process.exit(0))
process.on('SIGTERM', () => process.exit(0))

// Initialize Koop
const Koop = require('koop')
const koop = new Koop()

// Install the Yelp Provider
const yelp = require('yelp')

dmfenton

const express = require('express')
const app = express()
const Koop = new Koop()

app.use(koop.server)

app.listen(80)

dmfenton

const Koop = require('koop')
const koop = new Koop()
koop.server.listen(80)

dmfenton

module.exports = {
  name: 'agol', // Required, the name of this provider and the start of all its URLS
  type: 'provider', // Required, the type of Koop Plugin this is
  version: require('./package.json').version, // Required, the version of this provider
  Model: require('./agol'), // Required contains getData and other functions called by controller
  hosts: true, // Optional, whether or not `getData` should receive a `host` parameter
  disableIdParam: false, // Optional, whether or not `getData` should receive an `id` parameter
  routes: require('./routes'), // Optional, any additional routes that should be handled by this provider
  Controller: require('./controller'), // Optional, a controller to support unique routes
}

dmfenton

'use strict'
const ttl = 60 * 60
const request = require('request').defaults({gzip: true})
const types = require('./mappings/types.js')
module.exports = function (koop) {
  // This is our one public function it's job its to fetch data from craigslist and return as a feature collection
  this.getData = function (req, callback) {
    const key = `craigslist::${req.params.host}::${req.params.id}`
    koop.cache.retrieve(key, req.query, (err, geojson) => {
      if (geojson) {

dmfenton

module.exports = function (model) {
  this.model = model
  /**
   * returns a list of the registered hosts and their ids
   */
  this.get = function (req, res) {
    this.model.log.debug({route: 'dataset:get', params: req.params, query: req.query})
    if (req.params.dataset) this._getDataset(req, res)
    else this._getDatasets(req, res)
  }

dmfenton

module.exports = [
{
    path: '/agol/:id/datasets',
    methods: ['get'],
    handler: 'get'
  },
  {
    path: '/agol/:id/datasets/:dataset.:format',
    methods: ['get'],
    handler: 'get'

dmfenton

/* @flow */
'use strict';

var path = require('path');
var config = require('config');
var Winnow = require('winnow');
var FeatureParser = require('feature-parser');

var createKoop = require('koop');
var koop = createKoop(config);

dmfenton

Lessons learned AWS GeoSpatial Big Data System

A team working under a DARPA contract moved a system for geolocating iamges from a local cluster to AWS.

• aws s3 => always sync thrice
• do your own key store in the root of your bucket?
• map high volume IO to local ephemeral disks
• if you are using a NAT within a VPC, know that all traffic moves through that instance so size it accordingly
• snapshots are cheap, compressed, differential
• https://libcloud.apache.org/ is a library that abstracts different cloud providers into the same API

dmfenton

{
  "filesystem": {
    "s3": {
      "bucket": $S3_BUCKET,
      "endpoint" $S3-ENDPOINT //optional https://forums.aws.amazon.com/ann.jspa?annID=3112
    }
  }
}