stupidNewsScraper.js

var urls;
var writeStream;
var prettifier;
var specialCharRegExp = /[^a-zA-Z0-9 ]/g;
var fs = require('fs');
var Transform = require('stream').Transform;
var request = require('request');
var pretty = require('pretty');





urls = [
  'http://www.google.com',
  'https://timesofindia.indiatimes.com/world/middle-east/irans-gen-soleimani-killed-in-airstrike-at-baghdad-airport/articleshow/73077598.cms'
];
prettifier = new Transform({
  decodeStrings: false
});
prettifier._transform = (chunk, enconding, done) => {
  var textChunk = chunk.toString();
  var prettyTextChunk = pretty(textChunk);
  done(null, prettyTextChunk);
};



urls.forEach((url, index) => {
  writeStream = fs.createWriteStream(url.replace(specialCharRegExp, '') + '.txt');
  request
    .get(url)
    .on('response', function(response) {
      console.log('==========================================================');
      console.log(response.statusCode);
      console.log(response.headers['content-type']);
    })
    .pipe(prettifier)
    .pipe(writeStream);
});