Skip to content

Instantly share code, notes, and snippets.

@mryhryki
Last active March 13, 2022 00:47
Show Gist options
  • Save mryhryki/83d071d428aa6b40fc7574a00083206c to your computer and use it in GitHub Desktop.
Save mryhryki/83d071d428aa6b40fc7574a00083206c to your computer and use it in GitHub Desktop.

s3-checksum

Setup

$ npm install

Execute

$ npm start

Development

$ npm run dev

Log sample

$ npm start

> start
> esbuild --platform=node --external:node:* --bundle --minify --outfile=./index.js ./index.ts && node ./index.js


  index.js  6.3mb ⚠️

⚡ Done in 121ms
--------------------------------------------------------------------------------
1. Upload with valid checksum
SUCCESS!
--------------------------------------------------------------------------------
2. Upload with INVALID checksum
ERROR: The SHA256 you specified did not match the calculated checksum.
--------------------------------------------------------------------------------
3-1. Create multipart upload
SUCCESS!
--------------------------------------------------------------------------------
3-2. Upload 1st part with valid checksum
SUCCESS!
--------------------------------------------------------------------------------
3-3. Upload 2nd part with INVALID checksum
ERROR: The SHA256 you specified did not match the calculated checksum.
--------------------------------------------------------------------------------
3-4. Upload 2nd part with valid checksum
SUCCESS!
--------------------------------------------------------------------------------
3-5. Complete multipart
SUCCESS!
--------------------------------------------------------------------------------
4. List Bucket
SUCCESS!
* Object1: multipart-upload.txt (10485760 Bytes)
* Object2: valid-checksum.txt (1024 Bytes)

Links

package-lock.json
node_modules/**
.editorconfig
.eslintrc.yaml
.prettierrc.json
data/**
*.js
import {
completeMultipartUpload,
createMultipartUpload,
getSHA256,
listObjects,
putObjectWithChecksum,
TestData1KB,
TestData5MB,
uploadMultipart,
} from "./utils";
const startSection = (title: string): void =>
console.log("--------------------------------------------------------------------------------\n%s", title);
const promiseHandler = async <T>(promise: Promise<T>): Promise<T | void> => {
return promise
.then((result: T) => {
console.log("SUCCESS!");
return result;
})
.catch((err) => {
if (err instanceof Error) {
console.error("ERROR: %s", err.message);
} else {
console.error(err);
}
});
};
const main = async () => {
// await cleanupBucket();
const invalidChecksum = getSHA256("");
const validChecksum1KB = getSHA256(TestData1KB);
const validChecksum5MB = getSHA256(TestData5MB);
const multipartKey = "multipart-upload.txt";
startSection("1. Upload with valid checksum");
await promiseHandler(putObjectWithChecksum("valid-checksum.txt", TestData1KB, validChecksum1KB));
startSection("2. Upload with INVALID checksum");
await promiseHandler(putObjectWithChecksum("invalid-checksum.txt", TestData1KB, invalidChecksum));
startSection("3-1. Create multipart upload");
const multipart = await promiseHandler(createMultipartUpload(multipartKey));
const { UploadId } = multipart ?? {};
if (UploadId == null) throw new Error("UploadId is null");
startSection("3-2. Upload 1st part with valid checksum");
const part1 = await promiseHandler(uploadMultipart(multipartKey, TestData5MB, 1, validChecksum5MB, UploadId));
startSection("3-3. Upload 2nd part with INVALID checksum");
await promiseHandler(uploadMultipart(multipartKey, TestData5MB, 2, invalidChecksum, UploadId));
startSection("3-4. Upload 2nd part with valid checksum");
const part2 = await promiseHandler(uploadMultipart(multipartKey, TestData5MB, 2, validChecksum5MB, UploadId));
startSection("3-5. Complete multipart");
await promiseHandler(
completeMultipartUpload(multipartKey, UploadId, [
{ ETag: part1?.ETag, PartNumber: part1?.partNumber, ChecksumSHA256: validChecksum5MB },
{ ETag: part2?.ETag, PartNumber: part2?.partNumber, ChecksumSHA256: validChecksum5MB },
])
);
startSection("4. List Bucket");
const objects = await promiseHandler(listObjects());
objects?.Contents?.forEach(({ Key, Size }, i) => {
console.log(`* Object${i + 1}: ${Key} (${Size} Bytes)`);
});
};
main().catch(console.error);
{
"name": "s3-checksum",
"author": "mryhryki",
"private": true,
"license": "MIT",
"engines": {
"node": "16.x",
"npm": "8.x"
},
"scripts": {
"start": "esbuild --platform=node --external:node:* --bundle --minify --outfile=./index.js ./index.ts && node ./index.js",
"dev": "nodemon --watch ./ --ext ts --exec 'npm start'",
"lint": "mryhryki-lint",
"lint:fix": "mryhryki-lint-fix",
"type": "tsc",
"type:watch": "tsc --watch"
},
"dependencies": {
"@mryhryki/lint": "^0.0.5",
"@types/aws-sdk": "^2.7.0",
"@types/node": "^17.0.8",
"aws-sdk": "^2.1091.0",
"esbuild": "^0.14.11",
"nodemon": "^2.0.15",
"typescript": "^4.5.4"
},
"nodemonConfig": {
"delay": 1000
}
}
{
"compilerOptions": {
"allowJs": false,
"allowSyntheticDefaultImports": true,
"esModuleInterop": true,
"jsx": "preserve",
"lib": ["esnext", "DOM.Iterable", "dom"],
"noEmit": true,
"noImplicitAny": true,
"skipLibCheck": true,
"strict": true
},
"include": [
"*.ts"
],
"exclude": [
"**/node_modules/**",
"**/*.test.ts"
]
}
import { S3 } from "aws-sdk";
import Crypto from "node:crypto";
export const TestData1KB = [
"7798762c1ae1b9e17546e572870ba5c7214f10cb0d4aa8170fe290036138dea5a993ae3de8b1d3ae28de337ead0bc9b89da",
"2a6f3b66d33f238b41e53c6e7b93d0ac66f959d3f42fba9c71a7f9597c6c4e7dffa28ad0bd86e0be28386c1e6db7eeb7ac9",
"720e6c398919ede3aaadff485991a38c62f169ec2bec7e7833ec0233f043d773a16cd369b5772a9501d6e0d9aa30f644c74",
"149e8c7c216e3a879ee995b1b611b57bcd3badb56f29b090925384a74bb3d997c6cad0d1530a50f93fdf1dfd7c6d8fca168",
"637d9210a0c132e95c078888317b60d187f98b7bb4e2c150337966a4cfb949d06f0fa3d58b150cc27f5c8a6ef53824722d8",
"29c877c7870df3fdad1092d5737464f3aab0101c910b4c4ae01f230be1611836c69ae385bfc2e07bb14f8b350dd0019956a",
"6530bc2fc9365d4ca9a10878ba6d9271078c4a3b93a8a03ee169f0a9480d679b333c6d80fd46e8dd849ab5cf521df924f09",
"8c6cd75509c640eb1af6b648d5e15c5efcc4f61a0233899392166ac748043b9fad77a595390b055d95e58be6943bba761e1",
"7bfe04ef07163877c179365b11cfde8b9c748bded7007d357205fd9383da45ea5e178745539ed35839a0dd51f7d3496877c",
"c8e1e3987cc13775efe4945484b8d8231f3f9527176325418e4a93e4e04fcbdfe8d879cb2575948820a08f792985e2cbbe9",
"212ffd062577d9aa829ab88",
"",
].join("\n");
if (TestData1KB.length !== 1024) throw new Error("TestData1KB is not 1KB.");
export const TestData5MB = Array.from({ length: 5 * 1024 })
.map(() => TestData1KB)
.join("");
export const getSHA256 = (data: string): string => Crypto.createHash("sha256").update(data, "utf8").digest("base64");
// ----- S3 -----
const TestBucket = "mryhryki-test";
const s3 = new S3({ apiVersion: "2006-03-01" });
export const listObjects = () => s3.listObjectsV2({ Bucket: TestBucket, Prefix: "" }).promise();
export const putObjectWithChecksum = (key: string, data: string, checksum: string) => {
return s3
.putObject({
Bucket: TestBucket,
Key: key,
Body: data,
ContentType: "text/plain; charset=utf-8",
ChecksumAlgorithm: "SHA256",
ChecksumSHA256: checksum,
})
.promise();
};
export const createMultipartUpload = async (key: string): Promise<{ UploadId: string }> => {
const { UploadId } = await s3
.createMultipartUpload({
Bucket: TestBucket,
Key: key,
ChecksumAlgorithm: "SHA256",
})
.promise();
if (UploadId == null) throw new Error("UploadId is null");
return { UploadId };
};
export const uploadMultipart = async (
key: string,
data: string,
partNumber: number,
checksum: string,
uploadId: string
): Promise<{ ETag: string; partNumber: number }> => {
const { ETag } = await s3
.uploadPart({
Bucket: TestBucket,
Key: key,
Body: data,
PartNumber: partNumber,
ChecksumAlgorithm: "SHA256",
ChecksumSHA256: checksum,
UploadId: uploadId,
})
.promise();
if (ETag == null) {
throw new Error("ETag null");
}
return { ETag, partNumber };
};
export const completeMultipartUpload = (key: string, UploadId: string, Parts: S3.Types.CompletedPart[]) => {
return s3
.completeMultipartUpload({
Bucket: TestBucket,
Key: key,
MultipartUpload: { Parts },
UploadId,
})
.promise();
};
export const cleanupBucket = async () => {
const objects = await listObjects();
await Promise.all(
(objects.Contents ?? []).map(async ({ Key }) => {
if (Key == null) return;
console.log("Delete object: %s", Key);
await s3.deleteObject({ Bucket: TestBucket, Key });
})
);
const multiparts = await s3.listMultipartUploads({ Bucket: TestBucket }).promise();
await Promise.all(
(multiparts.Uploads ?? []).map(async ({ Key, UploadId }) => {
if (Key == null || UploadId == null) return;
console.debug("Delete Multipart Upload:", Key, UploadId);
await s3.abortMultipartUpload({ Bucket: TestBucket, Key, UploadId }).promise();
})
);
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment