Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
Quick S3 bulk downloader
<?php // requires aws/aws-php-dsk
// reads s3://bucket/key.ext lines from STDIN, saves them to ../files
require __DIR__ . '/vendor/autoload.php';
ini_set("memory_limit", "2G"); // impacted by concurrency + file sizes, since we aren't streaming
$s3 = new Aws\S3\S3Client(['credentials' => [
'key' => 'KEY_HERE', 'secret' => 'SECRET_HERE'
], 'version' => '2006-03-01', 'region' => 'us-tirefire-1']);
for ($i = 0, $promises = []; $line = trim(fgets(STDIN)); $i++) {
$line = str_replace('s3://', '', $line);
[$bucket, $key] = explode('/', $line, 2);
fputs(STDERR, "Starting to download " . $key . "\n");
$promises[] = $s3->getObjectAsync([
'Bucket' => $bucket,
'Key' => $key
])->then(function (\Aws\Result $result) use ($key) {
fputs(STDOUT, 'Downloaded: ' . $key . ', size: ' . $result->get('ContentLength') . "\n");
file_put_contents(__DIR__ . '/../files/' . $key, $result->get('Body'));
}, function() use ($key) {
fputs(STDOUT, 'Failed: ' . $key);
var_dump(func_get_args());
});
if (count($promises) == 100) /* concurrency level */ {
\GuzzleHttp\Promise\all($promises)->wait();
$promises = [];
}
}
\GuzzleHttp\Promise\all($promises)->wait();
fputs(STDOUT, "Downloaded $i files\n");
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
You can’t perform that action at this time.