Created
December 3, 2018 11:54
-
-
Save lucydjo/8b63245f77452b4eabab2b0d69924c83 to your computer and use it in GitHub Desktop.
Get tweet and upload to amazon aws | Nodejs | AWS S3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const fs = require('fs'); | |
const AWS = require('aws-sdk'); | |
const path = require('path'); | |
const Twit = require('twit'); | |
var db = require('node-localdb'); | |
var database = db('data_1.json'); | |
//configuring Twitter api key | |
var T = new Twit({ | |
consumer_key: '...', | |
consumer_secret: '...', | |
access_token: '...', | |
access_token_secret: '...', | |
timeout_ms: 60*1000, // optional HTTP request timeout to apply to all requests. | |
strictSSL: true, // optional - requires SSL certificates to be valid. | |
}) | |
//configuring the AWS environment | |
AWS.config.update({ | |
accessKeyId: "...", | |
secretAccessKey: ".../j" | |
}); | |
var s3 = new AWS.S3({ signatureVersion: 'v4', region: 'eu-west-3' }); | |
var datacount = 1; | |
/* EVENT TWEET */ | |
var stream = T.stream('statuses/filter', { track: 'a'}) | |
stream.on('tweet', function (tweet) { | |
database.insert({tweet: tweet}).then(function(u){ | |
//console.log('[LOG] NEW TWEET IN BASE.'); | |
}); | |
}) | |
/* CROP AND UPLOAD DATA.JSON EVERY 60 SECONDS */ | |
setInterval(function(){ | |
const stats = fs.statSync('data_'+datacount+'.json'); | |
const fileSizeInBytes = stats.size | |
console.log('[LOG] DATABASE SIZE | ' + bytesToSize(fileSizeInBytes)); | |
console.log('[LOG] UPLOAD FILE'); | |
uploadData('data_'+datacount+'.json'); | |
datacount++; | |
console.log('[LOG] INIT NEW DATA FILE | data_'+datacount+'.json' ); | |
database = db('data_'+datacount+'.json'); | |
}, 60000); | |
/* UPLOAD TO AWS S3 */ | |
function uploadData(filename) { | |
var params = { | |
Bucket: 'MyBucket', | |
Body : fs.createReadStream(filename), | |
Key : "twitter/"+Date.now()+"_"+path.basename(filename) | |
}; | |
s3.upload(params, function (err, data) { | |
if (err) { | |
console.log("Error", err); | |
} | |
if (data) { | |
console.log("[LOG] FILE UPLOADED"); | |
console.log('[LOG] REMOVE DATA FILE | ' + filename); | |
var filePath = filename; | |
fs.unlinkSync(filePath); | |
} | |
}); | |
} | |
function bytesToSize(bytes) { | |
var sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB']; | |
if (bytes == 0) return '0 Byte'; | |
var i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024))); | |
return Math.round(bytes / Math.pow(1024, i), 2) + ' ' + sizes[i]; | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment