Skip to content

Instantly share code, notes, and snippets.

@r
Created April 15, 2014 11:23
Show Gist options
  • Save r/10724342 to your computer and use it in GitHub Desktop.
Save r/10724342 to your computer and use it in GitHub Desktop.
// npm install twitter <- requirement
CONSUMER_KEY = '';
CONSUMER_SECRET = '';
ACCESS_TOKEN_KEY = '';
ACCESS_TOKEN_SECRET = '';
NUMBER_TWEETS_DOWNLOAD = 5000;
OUTPUT_FILE = 'twitter.tsv';
var util = require('util'),
twitter = require('twitter'),
fs = require('fs');
var twit = new twitter({
consumer_key: CONSUMER_KEY,
consumer_secret: CONSUMER_SECRET,
access_token_key: ACCESS_TOKEN_KEY,
access_token_secret: ACCESS_TOKEN_SECRET
});
String.prototype.dateToISOString = function() { return new Date(this).toISOString(); }
String.prototype.whiteSpaceCollapse = function() { return this.replace(/\s+/g, ' '); }
var lines = 0;
var dataStream = fs.createWriteStream(OUTPUT_FILE);
twit.stream('statuses/sample', function(stream) {
stream.on('data', function(data) {
// only react to tweet messages
if (data['delete'] === undefined) {
var tweet_str =
[
data['id_str'],
data['created_at'].dateToISOString(),
data['user']['id_str'],
data['text'].whiteSpaceCollapse(),
data['in_reply_to_status_id_str'],
];
if(data['retweeted_status']) {
tweet_str.push(data['retweeted_status']['id_str']);
tweet_str.push(data['retweeted_status']['created_at'].dateToISOString());
tweet_str.push(data['retweeted_status']['user']['id_str']);
tweet_str.push(data['retweeted_status']['text'].whiteSpaceCollapse());
} else {
tweet_str.push('');
tweet_str.push('');
tweet_str.push('');
tweet_str.push('');
}
if(data['coordinates'] && data['coordinates']['type'] === 'Point') {
tweet_str.push(data['coordinates']['coordinates'][1]);
tweet_str.push(data['coordinates']['coordinates'][0]);
} else {
tweet_str.push('');
tweet_str.push('');
}
dataStream.write(tweet_str.join('\t') + "\n");
lines = lines + 1;
console.log(lines + "=" + data['id_str']);
if(lines > NUMBER_TWEETS_DOWNLOAD) {
dataStream.end();
process.exit(code = 0);
}
}
});
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment