Skip to content

Instantly share code, notes, and snippets.

View dmfenton's full-sized avatar

Daniel Fenton dmfenton

104 followers · 131 following
  • JLL Technologies
  • Washington, DC
  • X @Dmfenton
View GitHub Profile
@dmfenton
dmfenton / model.js
Created February 28, 2017 20:00
'use strict'
const ttl = 60 * 60
const request = require('request').defaults({gzip: true})
const types = require('./mappings/types.js')
module.exports = function (koop) {
// This is our one public function it's job its to fetch data from craigslist and return as a feature collection
this.getData = function (req, callback) {
const key = `craigslist::${req.params.host}::${req.params.id}`
koop.cache.retrieve(key, req.query, (err, geojson) => {
if (geojson) {
@dmfenton
dmfenton / controller.js
Last active February 28, 2017 19:44
module.exports = function (model) {
this.model = model
/**
* returns a list of the registered hosts and their ids
*/
this.get = function (req, res) {
this.model.log.debug({route: 'dataset:get', params: req.params, query: req.query})
if (req.params.dataset) this._getDataset(req, res)
else this._getDatasets(req, res)
}
@dmfenton
dmfenton / routes.js
Last active February 23, 2017 22:33
module.exports = [
{
path: '/agol/:id/datasets',
methods: ['get'],
handler: 'get'
},
{
path: '/agol/:id/datasets/:dataset.:format',
methods: ['get'],
handler: 'get'
@dmfenton
dmfenton / getData.js
Created February 23, 2017 21:28
/*
Example request: http://koop.com/craigslist/washingtondc/apartments/FeatureServer/0/query?where=price>20&f=geojson
Req is the express request object: https://expressjs.com/en/4x/api.html#req
req.params = {
host: 'washingtondc',
id: 'apartments'
}
req.query = {
where: 'price > 20',
f: geojson
@dmfenton
dmfenton / index.js
Last active March 2, 2017 15:22
Example Koop Index.js
module.exports = {
name: 'agol', // Required, the name of this provider and the start of all its URLS
type: 'provider', // Required, the type of Koop Plugin this is
version: require('./package.json').version, // Required, the version of this provider
Model: require('./agol'), // Required contains getData and other functions called by controller
hosts: true, // Optional, whether or not `getData` should receive a `host` parameter
disableIdParam: false, // Optional, whether or not `getData` should receive an `id` parameter
routes: require('./routes'), // Optional, any additional routes that should be handled by this provider
Controller: require('./controller'), // Optional, a controller to support unique routes
}
@dmfenton
dmfenton / exportFile.js
Created August 10, 2016 18:27
/* @flow */
'use strict';
var path = require('path');
var config = require('config');
var Winnow = require('winnow');
var FeatureParser = require('feature-parser');
var createKoop = require('koop');
var koop = createKoop(config);
@dmfenton
dmfenton / FOSS4GNA 2016 Notes.md
Created May 5, 2016 19:01

Lessons learned AWS GeoSpatial Big Data System

A team working under a DARPA contract moved a system for geolocating iamges from a local cluster to AWS.

  • aws s3 => always sync thrice
  • do your own key store in the root of your bucket?
  • map high volume IO to local ephemeral disks
  • if you are using a NAT within a VPC, know that all traffic moves through that instance so size it accordingly
  • snapshots are cheap, compressed, differential
  • https://libcloud.apache.org/ is a library that abstracts different cloud providers into the same API
@dmfenton
dmfenton / config.md
Last active April 13, 2016 15:07
Koop S3 Config 
{
  "filesystem": {
    "s3": {
      "bucket": $S3_BUCKET,
      "endpoint" $S3-ENDPOINT //optional https://forums.aws.amazon.com/ann.jspa?annID=3112
    }
  }
}
@dmfenton
dmfenton / image.html
Last active August 29, 2015 14:24
<html><body><img src="http://cdn.citylab.com/media/img/citylab/2013/08/30/2011/lead_large.png"></img></body></html>
@dmfenton
dmfenton / translate.js
Created July 2, 2015 21:13
This file has been truncated, but you can view the full file.
module['exports'] = function echoHttp (hook) {
var geographicCodes = {"3819":{"name":"GCS_HD1909","wkt":"GEOGCS[\"GCS_HD1909\",DATUM[\"D_Hungarian_Datum_1909\",SPHEROID[\"Bessel_1841\",6377397.155,299.1528128]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]]"},"3821":{"name":"GCS_TWD_1967","wkt":"GEOGCS[\"GCS_TWD_1967\",DATUM[\"D_TWD_1967\",SPHEROID[\"GRS_1967_Truncated\",6378160.0,298.25]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]]"},"3824":{"name":"GCS_TWD_1997","wkt":"GEOGCS[\"GCS_TWD_1997\",DATUM[\"D_TWD_1997\",SPHEROID[\"GRS_1980\",6378137.0,298.257222101]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]]"},"3889":{"name":"GCS_IGRS","wkt":"GEOGCS[\"GCS_IGRS\",DATUM[\"D_Iraqi_Geospatial_Reference_System\",SPHEROID[\"GRS_1980\",6378137.0,298.257222101]],PRIMEM[\"Greenwich\",0.0],UNIT[\"Degree\",0.0174532925199433]]"},"3906":{"name":"GCS_MGI_1901","wkt":"GEOGCS[\"GCS_MGI_1901\",DATUM[\"D_MGI_1901\",SPHEROID[\"Bessel_1841\",6377397.155,299.1528128]],PRIMEM[\"