How to crawl a rest API and save response to DB with Scramjet.

simple-rest-to-db.js

const scramjet = require("scramjet");
const stream = require("stream");
const request = require("request-promise");
const config = require(process.env.CONFIG);

let cnt = 0;
// build a stream of indexes
new scramjet.DataStream({
     read() {
       this.offset = (this.offset || 0);
       this.push(this.offset);
       this.offset += 1;
     }
  })
  // map indexes to API responses
  .map((i) => request.get(config.rest_api_url, i, i + config.range))
  // do the same as flatten in HighLand
  .flatMap((item) => item)
  // filter out the id's containing "3"
  .filter((item) => (""+item.id).indexOf('3') > -1)
  // add timestamp to id's containing "9"
  .map((item) => (""+item.id).indexOf('3') > -1 ? Object.assign(item, {timestamp: Date.now()) : item)
  // write to database
  .accumulate((insert, item) => insert(item), insert);