Skip to content

Instantly share code, notes, and snippets.

@funteractive
Created April 19, 2016 23:06
Show Gist options
  • Star 1 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save funteractive/f6e2a686b5f21da755655821221ccd8a to your computer and use it in GitHub Desktop.
Save funteractive/f6e2a686b5f21da755655821221ccd8a to your computer and use it in GitHub Desktop.
var Request = require('request');
var Moment = require('moment');
var FeedParser = require('feedparser')
// config
var username = process.argv[2];
var baseUrl = 'http://b.hatena.ne.jp/' + username + '/';
var rssUrl = 'rss';
var baseYear = parseInt(process.argv[3]);
var baseMonth = parseInt(process.argv[4]) - 1;
var baseDate = parseInt(process.argv[5]);
var term = process.argv[6] * -1;
var i = term + 1;
function parseBookmarks(i) {
var dateUrl = Moment([baseYear, baseMonth, baseDate]).add(i, 'days').format('YYYYMMDD');
var url = baseUrl + rssUrl + '?date=' + dateUrl;
var options = {
url: url,
headers: {
'User-Agent': 'from_rss_application'
}
};
var req = Request(options);
var feedparser = new FeedParser();
var items = [];
req.on('response', function(res) {
this.pipe(feedparser);
});
req.on('end', function() {
i = i + 1;
if( i <= 0 ) {
parseBookmarks(i);
}
});
feedparser.on('error', function(error) {
console.log(error);
});
feedparser.on('readable', function() {
var item;
while(item = this.read()) {
items.push(item);
}
});
feedparser.on('end', function() {
items.forEach(function(item) {
console.log('- [' + item.title + ']' + '(' + item.link + ')');
});
});
}
parseBookmarks(i);
@funteractive
Copy link
Author

Requirement

$ npm install --save-dev request moment feedparser

Usage

$ node hatebu-crawl.js USERNAME YYYY MM DD DATES

Ex ) 
$ node hatebu-crawl.js imu16 2016 04 22 7
// 2016/4/16 - 2016/4/22 のimu16のブックマークを取得

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment