Skip to content

Instantly share code, notes, and snippets.

@waltir
Last active April 30, 2019 04:38
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save waltir/6c331bd494992fbdda97df9c4bfebadf to your computer and use it in GitHub Desktop.
Save waltir/6c331bd494992fbdda97df9c4bfebadf to your computer and use it in GitHub Desktop.
require('events').EventEmitter.defaultMaxListeners = 100;
const cheerio = require('cheerio');
const request = require('request');
const fs = require('fs');
var interval = 500;
let data = new Array;
request({
method: 'GET',
url: 'https://www.npr.org/sections/national/'
}, (err, res, body) => {
if (err) return console.error(err);
let $ = cheerio.load(body);
$('div.imagewrap > a').each(function(index, elem) {
setTimeout(function () {
let link = $(elem).attr('href');
request({
method: 'GET',
url: link
}, (err, res, body) => {
if (err) return console.error(err);
let $ = cheerio.load(body);
let post = {
title: $('h1').text(),
url: link,
date: $('.date').text(),
slug: $('h3.slug > a').text(),
author: $('.byline__name > a:nth-child(1)').text(),
img: $('#storytext > div > div > img').attr('src'),
body:$('#storytext > p').map(function(i, el) { return $(this).text(); }).get().join(' ')
}
data.push(post)
})
fs.writeFile("log.json", JSON.stringify(data), function(err) {
if (err) return console.error(err);
console.log("Anoter post saved!");
});
}, index * interval);
});
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment