Skip to content

Instantly share code, notes, and snippets.

@waltir
Last active May 2, 2019 01:03
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save waltir/ddc49bfbeb82d23f197dc6b0647235d7 to your computer and use it in GitHub Desktop.
Save waltir/ddc49bfbeb82d23f197dc6b0647235d7 to your computer and use it in GitHub Desktop.
Getting meta tags from multiple posts on NPR using Cheerio.js
const cheerio = require('cheerio');
const request = require('request');
const fs = require('fs');
var interval = 500;
let data = new Array;
request({
method: 'GET',
url: 'https://www.npr.org/sections/national/'
}, (err, res, body) => {
if (err) return console.error(err);
let $ = cheerio.load(body);
$('div.imagewrap > a').each(function(index, elem) {
setTimeout(function () {
let link = $(elem).attr('href');
request({
method: 'GET',
url: link
}, (err, res, body) => {
if (err) return console.error(err);
let $ = cheerio.load(body);
let post = {
title: $('h1').text(),
url: link,
canonical: $('link[rel="canonical"]').attr('href'),
description: $('meta[name="description"]').attr('content'),
// Get OG Values
og_title: $('meta[property="og:title"]').attr('content'),
og_url: $('meta[property="og:url"]').attr('content'),
og_img: $('meta[property="og:image"]').attr('content'),
og_type: $('meta[property="og:type"]').attr('content'),
// Get Twitter Values
twitter_site: $('meta[name="twitter:site"]').attr('content'),
twitter_domain: $('meta[name="twitter:domain"]').attr('content'),
twitter_img_src: $('meta[name="twitter:image:src"]').attr('content'),
// Get Facebook Values
fb_appid: $('meta[property="fb:app_id"]').attr('content'),
fb_pages: $('meta[property="fb:pages"]').attr('content'),
}
data.push(post)
})
fs.writeFile("log.json", JSON.stringify(data), function(err) {
if (err) return console.error(err);
console.log("Anoter post saved!");
});
}, index * interval);
});
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment