Skip to content

Instantly share code, notes, and snippets.

@mihaiserban
Last active June 27, 2023 03:38
Show Gist options
  • Star 3 You must be signed in to star a gist
  • Fork 2 You must be signed in to fork a gist
  • Save mihaiserban/1f35d488405812f2bbd4b16e38e4afb5 to your computer and use it in GitHub Desktop.
Save mihaiserban/1f35d488405812f2bbd4b16e38e4afb5 to your computer and use it in GitHub Desktop.
aws s3.listObjects recursive including metadata
const aws = require('aws-sdk')
aws.config.update({
accessKeyId: process.env.aws_access_key_id,
secretAccessKey: process.env.aws_secret_access_key,
region: process.env.aws_region});
const options = {
Bucket: process.env.s3_bucket,
region: process.env.aws_region,
signatureVersion: 'v4',
ACL: 'public-read',
}
const s3 = new aws.S3(options)
function getS3ObjectHead(filename) {
const params = {
Bucket: process.env.s3_bucket,
Key: filename,
};
return s3.headObject(params).promise();
}
//recursive function to list s3 objects
async function s3ListObjects(params, s3DataContents, includeHead) {
try {
const data = await s3.listObjects(params).promise()
let contents = data.Contents;
if (includeHead) {
for (let i = 0; i < contents.length; i++) {
let element = contents[i];
const fileObject = await getS3ObjectHead(element.Key)
element.head = fileObject
}
}
s3DataContents = s3DataContents.concat(contents);
if (data.IsTruncated) {
// Set Marker to last returned key
params.Marker = contents[contents.length-1].Key;
await s3ListObjects(params, s3DataContents);
} else {
return s3DataContents;
}
} catch (error) {
throw error
}
}
// list all the files in a box
router.get('/list/:user/:box', async function (req, res) {
const pathToFolder = `${req.params.user}/${req.params.box}/`
const params = {
Bucket: process.env.s3_bucket,
Prefix: pathToFolder,
};
let s3DataContents = [];
try {
const includeHead = true
const data = await s3ListObjects(params, s3DataContents, includeHead)
res.status(200).json({
message: 'ok',
data: data,
});
} catch (error) {
res.status(500).json({
message: error.message
});
}
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment