Skip to content

Instantly share code, notes, and snippets.

@hadzimme
Last active Jan 29, 2021
Embed
What would you like to do?
Archiver and S3 Stream
import * as aws from "aws-sdk";
import * as archiver from "archiver";
import * as stream from "stream";
import * as uuid from "uuid";
import * as events from "events";
interface File {
fileName: string;
createReadStream: () => stream.Readable;
}
interface Upload {
writeStream: stream.Writable;
promise: Promise<aws.S3.ManagedUpload.SendData>;
}
interface ArchiveFilesOptions {
fileIds: string[];
destId: string;
}
interface S3ZipArchiverProps {
bucketName: string;
s3Options?: aws.S3.ClientConfiguration;
}
class S3ZipArchiver {
private readonly s3: aws.S3;
private readonly bucketName: string;
constructor(props: S3ZipArchiverProps) {
this.s3 = new aws.S3(props.s3Options || {});
this.bucketName = props.bucketName;
}
async archiveFiles(options: ArchiveFilesOptions): Promise<void> {
const process = archiver("zip");
const { writeStream, promise } = this.initUpload(options.destId);
const files = options.fileIds.map((fileId) => this.initFile(fileId));
writeStream.on("error", this.reject);
process.on("warning", this.warnOrReject);
process.on("error", this.reject);
process.pipe(writeStream);
const emitter = new events.EventEmitter();
emitter.on("append", (n) => {
if (n >= files.length) {
process.finalize();
return;
}
const file = files[n];
const readStream = file.createReadStream();
readStream.on("end", () => {
emitter.emit("append", n + 1);
});
readStream.on("error", this.reject);
process.append(readStream, {
name: file.fileName,
});
});
emitter.emit("append", 0);
await promise;
}
private reject(error: Error) {
throw error;
}
private warnOrReject(error: archiver.ArchiverError) {
if (error.code === "ENOENT") {
console.warn(error);
return;
}
this.reject(error);
}
private initUpload(destId: string): Upload {
const passThrough = new stream.PassThrough();
const upload = this.s3.upload({
Bucket: this.bucketName,
Key: `dest/${destId}.zip`,
Body: passThrough,
});
return {
writeStream: passThrough,
promise: upload.promise(),
};
}
private initFile(fileId: string): File {
const fileName = `${fileId}.zip`;
const request = this.s3.getObject({
Bucket: this.bucketName,
Key: `test1/${fileName}`,
});
return {
fileName,
createReadStream: () => {
return request.createReadStream();
},
};
}
}
const bucketName = process.env.BUCKET_NAME;
if (!bucketName) {
throw new Error("process.env.BUCKET_NAME not found.");
}
const zipArchiver = new S3ZipArchiver({ bucketName });
interface Event {
fileIds: string[];
}
export const handle = async (event: Event) => {
await zipArchiver.archiveFiles({
fileIds: event.fileIds,
destId: uuid.v4(),
});
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment