Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
import { mapLimit, auto, constant } from 'async'
import request from 'request'
import analyzer from './analyze'
import logger from './log'
const data = [
'http://example.com/item/0'
'http://example.com/item/1'
'http://example.com/item/2'
'http://example.com/item/3'
]
function fetch (opts, callback) {
const { url } = opts
request(url, callback) // this will do a simple GET of the url
}
function parse (opts, callback) {
const { fetch } = opts
let data = null
try {
data = JSON.parse(fetch.body)
} catch (err) {
return callback(err)
}
callback(null, data)
}
function process (item, callback) {
const block = {
url: constant(item),
fetch: ['url', fetch],
data: ['fetch', parse],
analyze: ['data', (opts, cb) => analyze(opts.data, cb),
log: ['analyze', log]
}
auto(block, callback)
}
function done (err, results) {
if (err) return console.error(err)
results
.map(r => r.analyze) // log the analysis results
.map(r => console.log(r))
}
// This calls `process` for each item in data, maximum 2 at a time
// Then passes the results to done, as a `results` array
// If any item returns an error, then the first error is send to `err` and no results
mapLimit(data, 2, process, done)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.