Skip to content

Instantly share code, notes, and snippets.

@norbornen
Last active November 7, 2019 15:05
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save norbornen/f3877a7fdf87dded5199929f903e4870 to your computer and use it in GitHub Desktop.
Save norbornen/f3877a7fdf87dded5199929f903e4870 to your computer and use it in GitHub Desktop.
load rss feed titles with node.js streams
#!/usr/bin/env node
// @ts-check
const https = require('https');
const { finished } = require('stream');
load_rss_feed_titles('https://news.google.com/rss?hl=en-US&gl=US&ceid=US:en')
.then(console.log).catch(console.error);
load_rss_feed_titles('https://news.google.com/rss?hl=ru&gl=RU&ceid=RU:ru')
.then(console.log).catch(console.error);
function load_rss_feed_titles(url) {
const itemRegexp = /(?:(?<=<item>)(.+)(?=<\/item>)|(?<=<item>)(.+))/g;
const titleRegexp = /<title>(.+?)<\/title>/;
return new Promise((resolve, reject) => {
https.get(url, (res) => {
if (res.statusCode !== 200) {
reject(new Error(`statusCode=${res.statusCode}`));
}
const titles = [];
let buff;
res.setEncoding('utf8');
res.on('data', (d) => {
res.pause();
buff = buff !== undefined ? buff + d : d;
const str = buff.toString('utf8');
const matches = [...str.matchAll(itemRegexp)];
if (matches.length > 0) {
buff = undefined;
for (const [x, full_item, part_item] of matches) {
if (full_item !== null && full_item !== undefined && full_item !== '') {
const title = full_item.match(titleRegexp);
if (title) {
titles.push(title[1]);
}
} else if (part_item !== null && part_item !== undefined && part_item !== '') {
buff = `<item>${part_item}`;
}
}
}
res.resume();
});
finished(res, (err) => {
if (err) {
reject(err);
} else {
resolve(titles);
}
});
}).on('error', (err) => {
reject(err);
});
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment