Pull images from twitter based on a hashtag and save them locally.

scrape.js

#!/usr/bin/env node
/*
This is a total hack, just needed to grab all the photos from twitter
so that we could import them into our Flickr group..
*/
const fs = require('fs');
const config = require('./config.json');
const Twitter = require('twitter');
const parse = require('querystring').parse;
const client = new Twitter({
    consumer_key: config.consumer.key,
    consumer_secret: config.consumer.secret,
    access_token_key: config.access.key,
    access_token_secret: config.access.secret
});
const path = require('path');
const http = require('http');
const async = require('async');
const images = {};
let counter = 0;

const fetch = (url, callback) => {
    counter++;
    const ext = path.extname(url);
    const image = path.join(__dirname, 'images', `hacksi_${counter}${ext}`);
    console.log(url, image);
    const writer = fs.createWriteStream(image);
    writer.on('end', () => {
        callback();
    });
    http.get(require('url').parse(url), (res) => {
        res.pipe(writer);
    });
};

const get = (args, callback) => {
    console.log(args);
    client.get('search/tweets', args, (error, tweets, response) => {
        tweets.statuses.forEach((tweet) => {
            const photos = tweet.entities && tweet.entities.media;
            if (photos) {
                photos.forEach((photo) => {
                    if (photo.type === 'photo') {
                        images[photo.media_url] = 1;
                    }
                });
            }
        });
        if (tweets.search_metadata.next_results) {
            return get(parse(tweets.search_metadata.next_results.replace('?', '')), callback);
        }
        callback();
    });
};

get({
    q: '#hacksi',
    count: 100,
    include_entities: true
}, () => {
    const items = Object.keys(images);
    console.log(`Fetching ${items.length} images from twitter..`);
    async.each(items, fetch, () => {
        console.log('Done..');
    });
});