Paging JSON API as a lazy Highland stream

pagination-stream.js

const _ = require('highland')
const JSONstream = require('JSONstream')

require('isomorphic-fetch')

const highlandFetch = require('highland-fetch')

Response.prototype.highland = highlandFetch.highlandResponse

// pages through next links and returns json objects as one continuous stream
function pageStream (url) {
  return _(fetch(url, { headers: new Headers({ Accept: 'application/json' }) }))
  .flatMap(res => {
    if (!res.ok) throw new Error(res.status)
    const next = _(res.headers.get('link').split(','))
    .filter(link => (/rel="next"/).test(link))
    .map(link => link.match(/<(.*)>/)[1])
    .flatMap(pageStream)

    return res.highland()
      .through(highlandFetch.textDecoder)
      .through(JSONstream.parse('*'))
      .concat(next)
  })
}

// lazily stream and parse on the fly
pageStream('https://example.com/paging-api')
.stopOnError(err => console.error(err.stack))
.each(_.log)
.done(() => console.log('DONE'))