Reads results.json and sends a POST request for each row to be saved to the DB.

scraper.js

const axios = require('axios');
const fs = require("fs");

const promisesArray = [];

(async function main () {
  try {
    // Read data dump
    let scraperOutput = JSON.parse(fs.readFileSync('./input/results.json', 'utf8'));
    
    // Schedule operations
    scraperOutput.forEach(row => {
      promisesArray.push(
        axios.post('http://localhost:8001/api/scraper', row, {
          headers: { 'Content-Type': 'application/json' }
        })
      )
    });

    // Wait for all of them to end before ending script execution
    const responses = await axios.all(promisesArray);

    // Obtain the responses that got errors.
    const errors = responses
      .filter(i => i.data.error != null)
      .map(i => i.data)

    // Write the responses that got errors to a file.
    fs.writeFile('./output/errors.json', JSON.stringify(errors), 'utf8');

    // End
    console.log('Finished.')
  } catch (e) {
    console.error(e)
  }
})()