quick and dirty find duplicate urls in sample.json file

findup.js

const fs = require('fs');
const _ = require('lodash');

fs.readFile('sample.json', 'utf8', (err, data) => {
  if (err) {
    console.error(err);
    return;
  }
  
  const websiteLinks = JSON.parse(data);
  const allItems = _.flatten(websiteLinks.map((item) => item.links));
  const uniqueLinks = new Set();

  allItems.forEach((item, i) => {
    const website = item.url.match(/^(?:https?:\/\/)?(?:[^@\n]+@)?(?:www\.)?([^:\/\n?]*(\/.*)?)/mi)[1];

    if (uniqueLinks.has(website)) {
      console.log(`${website} is a duplicate`);
    } else {
      uniqueLinks.add(website);
    }
  })
});