a demo of using stream json for big data.

data.json

{
    "a": {
      "b": [
        {
          "id": 301940,
          "slug": "ef4f2422125f",
          "nickname": "说",
          "avatar_source": "asdfsd",
          "total_likes_count": 28553,
          "total_wordage": 1373341,
          "is_following_user": false,
          "ob": {
            "a": 1,
            "b": 2
          }
        },
        {
          "id": 3950651,
          "slug": "ca5b9d6f94dc",
          "nickname": "三屿",
          "avatar_source": "asdf",
          "total_likes_count": 3625,
          "total_wordage": 169200,
          "is_following_user": false
        }
      ]
    },
    "total_count": 35932
  }

index.js

const { chain } = require("stream-chain");
const { streamValues } = require("stream-json/streamers/StreamValues");
const Pick = require("stream-json/filters/Pick");
const fs = require("fs");

const pipeline = chain([
  fs.createReadStream("./small.json"),
  Pick.withParser({ filter: /^a\.b\.\d+/ }),
  streamValues()
]);

pipeline.on("data", data => {
    console.log(data);
});

pipeline.on("end", () => {
  console.log(`the end`);
});