Skip to content

Instantly share code, notes, and snippets.

@hackhat
Last active October 7, 2022 11:19
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save hackhat/cc0adf1317eeedcec52b1a4ff38f738b to your computer and use it in GitHub Desktop.
Save hackhat/cc0adf1317eeedcec52b1a4ff38f738b to your computer and use it in GitHub Desktop.
Upload folder to AWS.S3 with Node.js (works on windows, keeps the folder structure intact, correct mime type)
// Your config file
const s3Config = require('./s3Config');
const AWS = require("aws-sdk");
const fs = require("fs");
const path = require("path");
const mime = require("mime");
const config = {
s3BucketName: s3Config.bucketName,
// Absolute path
localFolder: s3Config.localFolder,
accessKeyId: s3Config.accessKeyId,
secretAccessKey: s3Config.secretAccessKeyId,
};
const start = async ({accessKeyId, secretAccessKeyId, s3BucketName, localFolder}) => {
AWS.config.setPromisesDependency(Promise);
const s3 = new AWS.S3({
signatureVersion: 'v4',
accessKeyId: accessKeyId,
secretAccessKey: secretAccessKeyId,
});
const filesPaths = await walkSync(localFolder);
for (let i = 0; i < filesPaths.length; i++) {
const statistics = `(${i + 1}/${filesPaths.length}, ${Math.round((i + 1) / filesPaths.length * 100)}%)`;
const filePath = filesPaths[i];
const fileContent = fs.readFileSync(filePath);
// If the slash is like this "/" s3 will create a new folder, otherwise will not work properly.
const relativeToBaseFilePath = path.normalize(path.relative(localFolder, filePath));
const relativeToBaseFilePathForS3 = relativeToBaseFilePath.split(path.sep).join('/');
const mimeType = mime.getType(filePath);
console.log(`Uploading`, statistics, relativeToBaseFilePathForS3);
// https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObject-property
await s3.putObject({
ACL: `public-read`,
Bucket: s3BucketName,
Key: relativeToBaseFilePathForS3,
Body: fileContent,
ContentType: mimeType,
}).promise();
console.log(`Uploaded `, statistics, relativeToBaseFilePathForS3);
}
};
start(config).then(() => {
console.log(`Completed!`);
});
async function walkSync(dir) {
const files = fs.readdirSync(dir);
const output = [];
for (const file of files) {
const pathToFile = path.join(dir, file);
const isDirectory = fs.statSync(pathToFile).isDirectory();
if (isDirectory) {
output.push(...await walkSync(pathToFile));
} else {
output.push(await pathToFile);
}
}
return output;
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment