twitter-media.js

#! /usr/bin/env casperjs

var casper = require('casper').create({
    logLevel: 'debug',
    verbose: true,
    viewportSize: {
        width: 1280,
        height: 800,
    },
    pageSettings: {
        loadImages: false,
    },
    waitTimeout: 50000,
});
var fs = require('fs');

var currentTweets, mediaUrls;

var scrollToBottomAndWaitForTweetsToLoad = function() {
    casper.evaluate(function scrollToBottom() {
        window.document.body.scrollTop = document.body.scrollHeight;
    });

    var noMoreTweets = casper.evaluate(function() {
        var localNoMoreTweets = document.querySelectorAll('.has-more-items') == 0;

        return localNoMoreTweets;
    });

    if (! noMoreTweets) {
        casper.then(function() {
            casper.waitForResource(/media_timeline/);
        });
    }
};

var doneCheck = function() {
    var noMoreTweets = casper.evaluate(function() {
        var localNoMoreTweets = document.querySelectorAll('.has-more-items') == 0;

        return localNoMoreTweets;
    });

    if (! noMoreTweets) {
        casper.log('Not done yet.');

        casper.then(scrollToBottomAndWaitForTweetsToLoad);

        casper.then(function() {
            currentTweets = casper.evaluate(function getCurrentTweets() {
                return document.querySelectorAll('.ProfileTweet').length;
            });

            casper.log(currentTweets);
        });

        if (currentTweets < 800) {
            casper.then(doneCheck);
        }
    }
};

(function() {
    casper.start('https://twitter.com/swolebeau/media', function getCurrentTweets() {
        currentTweets = casper.evaluate(function getCurrentTweets() {
            return document.querySelectorAll('.ProfileTweet').length;
        });

        casper.log(currentTweets);
    });

    // We need to repeat this until noMoreTweets
    casper.then(doneCheck);

    casper.then(function getMediaUrls() {
        mediaUrls = casper.evaluate(function getMediaElements() {
            var mediaElements = document.querySelectorAll('.TwitterPhoto-mediaSource');

            // We have to do this in the evaluate, doing it outside means we can only "see" the first element
            var localMediaUrls = [].map.call(mediaElements, function getSrc(element) {
                return element.src;
            });

            return localMediaUrls;
        });

        casper.log(mediaUrls);
    });

    casper.then(function writeUrls() {
        var serializedUrls = JSON.stringify(mediaUrls, null, 4);
        fs.write('media-urls.json', serializedUrls, 'w');

        casper.log('Writing to file.');
    });

    casper.run();
})();