Skip to content

Instantly share code, notes, and snippets.

@stephenplusplus
Created February 25, 2015 01:01
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save stephenplusplus/c3d38b7b9a3f20d2ff8c to your computer and use it in GitHub Desktop.
Save stephenplusplus/c3d38b7b9a3f20d2ff8c to your computer and use it in GitHub Desktop.
var fs = require('fs'),
gcloud = require('gcloud'),
knox = require('knox'),
path = require('path'),
ProgressBar = require('progress'),
RSVP = require('rsvp'),
tar = require('tar'),
through = require('through2'),
uuid = require('uuid'),
zlib = require('zlib');
var aws = knox.createClient({
key: '',
secret: '',
region: 'us-west-2',
bucket: '',
secure: true
});
var gcs = gcloud.storage({
keyFilename: '/Users/stephen/dev/keyfile.json',
projectId: 'nth-circlet-705'
});
var gcsBucket = gcs.bucket('gcsbuckettwo');
var directory = uuid.v4();
function getWriteStream(service, fileName, opts) {
var stream = through();
if (service === 'google') {
var serviceStream = gcsBucket.file(fileName).createWriteStream({
validation: false,
resumable: false
});
}
if (service === 's3') {
var serviceStream = aws.put(fileName, opts)
.on('response', function() {
stream.emit('finish');
});
}
return stream.pipe(serviceStream);
}
function upload(service) {
console.log('Uploading to', service);
var gunzip = zlib.createGunzip(),
untar = tar.Parse(),
outstandingCount = 1,
uploadedBytes = 0;
var bar = new ProgressBar(':bar', { total: 750 });
var start = new Date().getTime();
return new RSVP.Promise(function(resolve, reject) {
function checkComplete() {
if (outstandingCount === 0) {
var now = new Date().getTime(),
time = now - start;
console.log('Finished uploading to', service, 'in', time / 1000.0, ' seconds');
resolve({
totalBytes: uploadedBytes,
totalTime: time
});
}
}
var extraction = fs.createReadStream('files.tar.gz').pipe(gunzip).pipe(untar).on('entry', function(entry) {
if (entry.type === 'File') {
outstandingCount += 1;
var fileName = directory + '/' + entry.path.split(path.sep).slice(1).join('/');
var writeStream = getWriteStream(service, fileName, { 'Content-Length': entry.size });
entry.pipe(writeStream)
.on('error', reject)
.on('finish', function() {
outstandingCount -= 1;
bar.tick();
uploadedBytes += entry.size;
checkComplete();
});
}
}).on('end', function() {
outstandingCount -= 1;
checkComplete();
});
});
}
upload('s3').then(function() {
return upload('google');
}).catch(function(err) {
throw(error);
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment