Skip to content

Instantly share code, notes, and snippets.

@auxten
Created June 1, 2023 07:56
Show Gist options
  • Save auxten/2ae1807914a8ca969671bdc07d36e386 to your computer and use it in GitHub Desktop.
Save auxten/2ae1807914a8ca969671bdc07d36e386 to your computer and use it in GitHub Desktop.
aws s3 download pre allocate disk space
#include <iostream>
#include <fstream>
#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>
#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/HeadObjectRequest.h>
#include <aws/s3/model/GetObjectRequest.h>
int main(int argc, char** argv)
{
if (argc < 4) {
std::cerr << "Usage: s3_download <bucket> <key> <local_file>" << std::endl;
return 1;
}
const Aws::String bucket_name = argv[1];
const Aws::String key = argv[2];
const std::string local_file_path = argv[3];
// Initialize the SDK
Aws::SDKOptions options;
Aws::InitAPI(options);
{
// Create an S3 client
Aws::S3::S3Client s3_client;
// Head the object to get its size
Aws::S3::Model::HeadObjectRequest head_request;
head_request.SetBucket(bucket_name);
head_request.SetKey(key);
auto head_outcome = s3_client.HeadObject(head_request);
if (!head_outcome.IsSuccess()) {
std::cerr << "Failed to head object: " << head_outcome.GetError().GetMessage() << std::endl;
return 1;
}
const auto file_size = head_outcome.GetResult().GetSize();
// Open the local file for writing
std::ofstream ofs(local_file_path, std::ios::out | std::ios::binary);
if (!ofs.good()) {
std::cerr << "Failed to open file: " << local_file_path << std::endl;
return 1;
}
// Pre-allocate disk space for the file
const auto fd = ofs.rdbuf()->fd();
if (posix_fallocate(fd, 0, file_size) != 0) {
std::cerr << "Failed to pre-allocate disk space: " << strerror(errno) << std::endl;
}
// Download the object and write it to the local file
Aws::S3::Model::GetObjectRequest get_request;
get_request.SetBucket(bucket_name);
get_request.SetKey(key);
auto get_outcome = s3_client.GetObject(get_request);
if (!get_outcome.IsSuccess()) {
std::cerr << "Failed to get object: " << get_outcome.GetError().GetMessage() << std::endl;
return 1;
}
auto& body_stream = get_outcome.GetResultWithOwnership().GetBody();
char buffer[4096];
while (body_stream.good()) {
body_stream.read(buffer, sizeof(buffer));
ofs.write(buffer, body_stream.gcount());
}
}
// Shutdown the SDK
Aws::ShutdownAPI(options);
return 0;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment