crawl_tweets.js

(function() {
    var tweets = [];
    $('div.content').map(function(_,d){
        d = $(d);
        var img = d.find('img.js-action-profile-avatar')[0];
        var screen_name = $(d.find('span.username > b')[0]).text();
        var detail_url = $(d.find('small > a')[0]).attr('href');
        detail_url = detail_url || "";
        detail_url = detail_url.replace('/' + screen_name + '/status/', '');
        var time = $(d.find('small > a > span')[0]).attr('data-time');
        var text = $(d.find('p')[0]).text();
        var hashtags = [];
        d.find('p > a.twitter-hashtag b').map(function(_,o) {
            hashtags.push($(o).text());
        });
        var urls = [];
        d.find('p > a.twitter-timeline-link').map(function(_,o) {
            urls.push({expanded_url : $(o).attr('data-expanded-url')});
        });
        tweets.push({
            id_str : detail_url,
            time : time,
            text : text,
            user : {
                screen_name : screen_name,
                profile_image_url : $(img).attr('src')
            },
            entities : {
                hashtags : hashtags,
                urls : urls
            }
        });
    });
    tweets = tweets.filter(function(t){return t.id_str;});

    $('#json_area').remove();
    $('body').append('textarea')
        .text(JSON.stringify(tweets))
        .css({
            position : "fixed"
        })
        .attr('id', 'json_area');
    
    return tweets;
})();