Transform json with pipeline filters

README.md

Quick demonstration of how to filter/transform newline-delimited JSON in unix-pipeline fashion.

Run npm install to install dependencies.

Try filter.coffee < in.json or ...

cat in.json | filter.coffee | pretty.coffee 

... or ...

URL='http://registry.npmjs.org/fence-filter'
get.coffee $URL | filter.coffee

... or ...

curl -s $URL | filter.coffee

filter.coffee

#!/usr/bin/env coffee 
thru = require 'through'
split = require 'split'

filter = ->

  write = (data) -> 
    if data
      d = JSON.parse(data).author       # queue attributes of interest
      @queue JSON.stringify(d) + "\n"

  end = -> @queue null                  # append EOF string if desired

  thru(write, end)

process.stdin
  .pipe(split())
  .pipe(filter())
  .pipe(process.stdout)

get.coffee

#!/usr/bin/env coffee
request = require 'request'
 
url = process.argv[2] # try 'http://registry.npmjs.org/event-stream'

request(url).pipe(process.stdout)

in.json

{ "name": "split", "version": "0.2.10", "description": "split a Text Stream into a Line Stream", "homepage": "http://github.com/dominictarr/split", "repository": { "type": "git", "url": "git://github.com/dominictarr/split.git" }, "dependencies": { "through": "2" }, "devDependencies": { "asynct": "*", "it-is": "1", "ubelt": "~2.9", "stream-spec": "~0.2", "event-stream": "~3.0.2" }, "scripts": { "test": "asynct test/" }, "author": { "name": "Dominic Tarr", "email": "dominic.tarr@gmail.com", "url": "http://bit.ly/dominictarr" }, "optionalDependencies": {}, "engines": { "node": "*" }, "readmeFilename": "readme.markdown", "bugs": { "url": "https://github.com/dominictarr/split/issues" }, "_id": "split@0.2.10", "_from": "split@" }
{ "name": "split", "version": "0.2.10", "description": "split a Text Stream into a Line Stream", "homepage": "http://github.com/dominictarr/split", "repository": { "type": "git", "url": "git://github.com/dominictarr/split.git" }, "dependencies": { "through": "2" }, "devDependencies": { "asynct": "*", "it-is": "1", "ubelt": "~2.9", "stream-spec": "~0.2", "event-stream": "~3.0.2" }, "scripts": { "test": "asynct test/" }, "author": { "name": "Dominic Tarr", "email": "dominic.tarr@gmail.com", "url": "http://bit.ly/dominictarr" }, "optionalDependencies": {}, "engines": { "node": "*" }, "readmeFilename": "readme.markdown", "bugs": { "url": "https://github.com/dominictarr/split/issues" }, "_id": "split@0.2.10", "_from": "split@" }
{ "name": "split", "version": "0.2.10", "description": "split a Text Stream into a Line Stream", "homepage": "http://github.com/dominictarr/split", "repository": { "type": "git", "url": "git://github.com/dominictarr/split.git" }, "dependencies": { "through": "2" }, "devDependencies": { "asynct": "*", "it-is": "1", "ubelt": "~2.9", "stream-spec": "~0.2", "event-stream": "~3.0.2" }, "scripts": { "test": "asynct test/" }, "author": { "name": "Dominic Tarr", "email": "dominic.tarr@gmail.com", "url": "http://bit.ly/dominictarr" }, "optionalDependencies": {}, "engines": { "node": "*" }, "readmeFilename": "readme.markdown", "bugs": { "url": "https://github.com/dominictarr/split/issues" }, "_id": "split@0.2.10", "_from": "split@" }

out.json

{"name":"Dominic Tarr","email":"dominic.tarr@gmail.com","url":"http://bit.ly/dominictarr"}
{"name":"Dominic Tarr","email":"dominic.tarr@gmail.com","url":"http://bit.ly/dominictarr"}
{"name":"Dominic Tarr","email":"dominic.tarr@gmail.com","url":"http://bit.ly/dominictarr"}

package.json

{
  "dependencies": {
    "split": "~0.2",
    "through": ">=2",
    "event-stream": ">=3",
    "request": "~2.28.0"
  }
}

pretty.coffee

#!/usr/bin/env coffee 
thru = require 'through'
split = require 'split'

filter = ->
  thru (data) -> console.log JSON.parse(data) if data

process.stdin
  .pipe(split())
  .pipe(filter())