Skip to content

Instantly share code, notes, and snippets.

@lucydjo
Created December 3, 2018 11:54
Show Gist options
  • Save lucydjo/8b63245f77452b4eabab2b0d69924c83 to your computer and use it in GitHub Desktop.
Save lucydjo/8b63245f77452b4eabab2b0d69924c83 to your computer and use it in GitHub Desktop.
Get tweet and upload to amazon aws | Nodejs | AWS S3
const fs = require('fs');
const AWS = require('aws-sdk');
const path = require('path');
const Twit = require('twit');
var db = require('node-localdb');
var database = db('data_1.json');
//configuring Twitter api key
var T = new Twit({
consumer_key: '...',
consumer_secret: '...',
access_token: '...',
access_token_secret: '...',
timeout_ms: 60*1000, // optional HTTP request timeout to apply to all requests.
strictSSL: true, // optional - requires SSL certificates to be valid.
})
//configuring the AWS environment
AWS.config.update({
accessKeyId: "...",
secretAccessKey: ".../j"
});
var s3 = new AWS.S3({ signatureVersion: 'v4', region: 'eu-west-3' });
var datacount = 1;
/* EVENT TWEET */
var stream = T.stream('statuses/filter', { track: 'a'})
stream.on('tweet', function (tweet) {
database.insert({tweet: tweet}).then(function(u){
//console.log('[LOG] NEW TWEET IN BASE.');
});
})
/* CROP AND UPLOAD DATA.JSON EVERY 60 SECONDS */
setInterval(function(){
const stats = fs.statSync('data_'+datacount+'.json');
const fileSizeInBytes = stats.size
console.log('[LOG] DATABASE SIZE | ' + bytesToSize(fileSizeInBytes));
console.log('[LOG] UPLOAD FILE');
uploadData('data_'+datacount+'.json');
datacount++;
console.log('[LOG] INIT NEW DATA FILE | data_'+datacount+'.json' );
database = db('data_'+datacount+'.json');
}, 60000);
/* UPLOAD TO AWS S3 */
function uploadData(filename) {
var params = {
Bucket: 'MyBucket',
Body : fs.createReadStream(filename),
Key : "twitter/"+Date.now()+"_"+path.basename(filename)
};
s3.upload(params, function (err, data) {
if (err) {
console.log("Error", err);
}
if (data) {
console.log("[LOG] FILE UPLOADED");
console.log('[LOG] REMOVE DATA FILE | ' + filename);
var filePath = filename;
fs.unlinkSync(filePath);
}
});
}
function bytesToSize(bytes) {
var sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
if (bytes == 0) return '0 Byte';
var i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)));
return Math.round(bytes / Math.pow(1024, i), 2) + ' ' + sizes[i];
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment