!README.md

s3-checksum

Setup

$ npm install

Execute

$ npm start

Development

$ npm run dev

Log sample

$ npm start

> start
> esbuild --platform=node --external:node:* --bundle --minify --outfile=./index.js ./index.ts && node ./index.js


  index.js  6.3mb ⚠️

⚡ Done in 121ms
--------------------------------------------------------------------------------
1. Upload with valid checksum
SUCCESS!
--------------------------------------------------------------------------------
2. Upload with INVALID checksum
ERROR: The SHA256 you specified did not match the calculated checksum.
--------------------------------------------------------------------------------
3-1. Create multipart upload
SUCCESS!
--------------------------------------------------------------------------------
3-2. Upload 1st part with valid checksum
SUCCESS!
--------------------------------------------------------------------------------
3-3. Upload 2nd part with INVALID checksum
ERROR: The SHA256 you specified did not match the calculated checksum.
--------------------------------------------------------------------------------
3-4. Upload 2nd part with valid checksum
SUCCESS!
--------------------------------------------------------------------------------
3-5. Complete multipart
SUCCESS!
--------------------------------------------------------------------------------
4. List Bucket
SUCCESS!
* Object1: multipart-upload.txt (10485760 Bytes)
* Object2: valid-checksum.txt (1024 Bytes)

Links

• Amazon S3、データの整合性確認を最大90％高速化。新たなオブジェクトのチェックサムアルゴリズム追加で － Publickey
• New – Additional Checksum Algorithms for Amazon S3 | AWS News Blog

.gitignore

package-lock.json
node_modules/**
.editorconfig
.eslintrc.yaml
.prettierrc.json

data/**
*.js

index.ts

import {
  completeMultipartUpload,
  createMultipartUpload,
  getSHA256,
  listObjects,
  putObjectWithChecksum,
  TestData1KB,
  TestData5MB,
  uploadMultipart,
} from "./utils";

const startSection = (title: string): void =>
  console.log("--------------------------------------------------------------------------------\n%s", title);

const promiseHandler = async <T>(promise: Promise<T>): Promise<T | void> => {
  return promise
    .then((result: T) => {
      console.log("SUCCESS!");
      return result;
    })
    .catch((err) => {
      if (err instanceof Error) {
        console.error("ERROR: %s", err.message);
      } else {
        console.error(err);
      }
    });
};

const main = async () => {
  // await cleanupBucket();

  const invalidChecksum = getSHA256("");
  const validChecksum1KB = getSHA256(TestData1KB);
  const validChecksum5MB = getSHA256(TestData5MB);
  const multipartKey = "multipart-upload.txt";

  startSection("1. Upload with valid checksum");
  await promiseHandler(putObjectWithChecksum("valid-checksum.txt", TestData1KB, validChecksum1KB));

  startSection("2. Upload with INVALID checksum");
  await promiseHandler(putObjectWithChecksum("invalid-checksum.txt", TestData1KB, invalidChecksum));

  startSection("3-1. Create multipart upload");
  const multipart = await promiseHandler(createMultipartUpload(multipartKey));
  const { UploadId } = multipart ?? {};
  if (UploadId == null) throw new Error("UploadId is null");

  startSection("3-2. Upload 1st part with valid checksum");
  const part1 = await promiseHandler(uploadMultipart(multipartKey, TestData5MB, 1, validChecksum5MB, UploadId));

  startSection("3-3. Upload 2nd part with INVALID checksum");
  await promiseHandler(uploadMultipart(multipartKey, TestData5MB, 2, invalidChecksum, UploadId));

  startSection("3-4. Upload 2nd part with valid checksum");
  const part2 = await promiseHandler(uploadMultipart(multipartKey, TestData5MB, 2, validChecksum5MB, UploadId));

  startSection("3-5. Complete multipart");
  await promiseHandler(
    completeMultipartUpload(multipartKey, UploadId, [
      { ETag: part1?.ETag, PartNumber: part1?.partNumber, ChecksumSHA256: validChecksum5MB },
      { ETag: part2?.ETag, PartNumber: part2?.partNumber, ChecksumSHA256: validChecksum5MB },
    ])
  );

  startSection("4. List Bucket");
  const objects = await promiseHandler(listObjects());
  objects?.Contents?.forEach(({ Key, Size }, i) => {
    console.log(`* Object${i + 1}: ${Key} (${Size} Bytes)`);
  });
};

main().catch(console.error);

package.json

{
  "name": "s3-checksum",
  "author": "mryhryki",
  "private": true,
  "license": "MIT",
  "engines": {
    "node": "16.x",
    "npm": "8.x"
  },
  "scripts": {
    "start": "esbuild --platform=node --external:node:* --bundle --minify --outfile=./index.js ./index.ts && node ./index.js",
    "dev": "nodemon --watch ./ --ext ts --exec 'npm start'",
    "lint": "mryhryki-lint",
    "lint:fix": "mryhryki-lint-fix",
    "type": "tsc",
    "type:watch": "tsc --watch"
  },
  "dependencies": {
    "@mryhryki/lint": "^0.0.5",
    "@types/aws-sdk": "^2.7.0",
    "@types/node": "^17.0.8",
    "aws-sdk": "^2.1091.0",
    "esbuild": "^0.14.11",
    "nodemon": "^2.0.15",
    "typescript": "^4.5.4"
  },
  "nodemonConfig": {
    "delay": 1000
  }
}

tsconfig.json

{
  "compilerOptions": {
    "allowJs": false,
    "allowSyntheticDefaultImports": true,
    "esModuleInterop": true,
    "jsx": "preserve",
    "lib": ["esnext", "DOM.Iterable", "dom"],
    "noEmit": true,
    "noImplicitAny": true,
    "skipLibCheck": true,
    "strict": true
  },
  "include": [
    "*.ts"
  ],
  "exclude": [
    "**/node_modules/**",
    "**/*.test.ts"
  ]
}

utils.ts

import { S3 } from "aws-sdk";
import Crypto from "node:crypto";

export const TestData1KB = [
  "7798762c1ae1b9e17546e572870ba5c7214f10cb0d4aa8170fe290036138dea5a993ae3de8b1d3ae28de337ead0bc9b89da",
  "2a6f3b66d33f238b41e53c6e7b93d0ac66f959d3f42fba9c71a7f9597c6c4e7dffa28ad0bd86e0be28386c1e6db7eeb7ac9",
  "720e6c398919ede3aaadff485991a38c62f169ec2bec7e7833ec0233f043d773a16cd369b5772a9501d6e0d9aa30f644c74",
  "149e8c7c216e3a879ee995b1b611b57bcd3badb56f29b090925384a74bb3d997c6cad0d1530a50f93fdf1dfd7c6d8fca168",
  "637d9210a0c132e95c078888317b60d187f98b7bb4e2c150337966a4cfb949d06f0fa3d58b150cc27f5c8a6ef53824722d8",
  "29c877c7870df3fdad1092d5737464f3aab0101c910b4c4ae01f230be1611836c69ae385bfc2e07bb14f8b350dd0019956a",
  "6530bc2fc9365d4ca9a10878ba6d9271078c4a3b93a8a03ee169f0a9480d679b333c6d80fd46e8dd849ab5cf521df924f09",
  "8c6cd75509c640eb1af6b648d5e15c5efcc4f61a0233899392166ac748043b9fad77a595390b055d95e58be6943bba761e1",
  "7bfe04ef07163877c179365b11cfde8b9c748bded7007d357205fd9383da45ea5e178745539ed35839a0dd51f7d3496877c",
  "c8e1e3987cc13775efe4945484b8d8231f3f9527176325418e4a93e4e04fcbdfe8d879cb2575948820a08f792985e2cbbe9",
  "212ffd062577d9aa829ab88",
  "",
].join("\n");
if (TestData1KB.length !== 1024) throw new Error("TestData1KB is not 1KB.");
export const TestData5MB = Array.from({ length: 5 * 1024 })
  .map(() => TestData1KB)
  .join("");
export const getSHA256 = (data: string): string => Crypto.createHash("sha256").update(data, "utf8").digest("base64");

// ----- S3 -----

const TestBucket = "mryhryki-test";
const s3 = new S3({ apiVersion: "2006-03-01" });
export const listObjects = () => s3.listObjectsV2({ Bucket: TestBucket, Prefix: "" }).promise();

export const putObjectWithChecksum = (key: string, data: string, checksum: string) => {
  return s3
    .putObject({
      Bucket: TestBucket,
      Key: key,
      Body: data,
      ContentType: "text/plain; charset=utf-8",
      ChecksumAlgorithm: "SHA256",
      ChecksumSHA256: checksum,
    })
    .promise();
};

export const createMultipartUpload = async (key: string): Promise<{ UploadId: string }> => {
  const { UploadId } = await s3
    .createMultipartUpload({
      Bucket: TestBucket,
      Key: key,
      ChecksumAlgorithm: "SHA256",
    })
    .promise();
  if (UploadId == null) throw new Error("UploadId is null");
  return { UploadId };
};

export const uploadMultipart = async (
  key: string,
  data: string,
  partNumber: number,
  checksum: string,
  uploadId: string
): Promise<{ ETag: string; partNumber: number }> => {
  const { ETag } = await s3
    .uploadPart({
      Bucket: TestBucket,
      Key: key,
      Body: data,
      PartNumber: partNumber,
      ChecksumAlgorithm: "SHA256",
      ChecksumSHA256: checksum,
      UploadId: uploadId,
    })
    .promise();
  if (ETag == null) {
    throw new Error("ETag null");
  }
  return { ETag, partNumber };
};

export const completeMultipartUpload = (key: string, UploadId: string, Parts: S3.Types.CompletedPart[]) => {
  return s3
    .completeMultipartUpload({
      Bucket: TestBucket,
      Key: key,
      MultipartUpload: { Parts },
      UploadId,
    })
    .promise();
};

export const cleanupBucket = async () => {
  const objects = await listObjects();
  await Promise.all(
    (objects.Contents ?? []).map(async ({ Key }) => {
      if (Key == null) return;
      console.log("Delete object: %s", Key);
      await s3.deleteObject({ Bucket: TestBucket, Key });
    })
  );
  const multiparts = await s3.listMultipartUploads({ Bucket: TestBucket }).promise();
  await Promise.all(
    (multiparts.Uploads ?? []).map(async ({ Key, UploadId }) => {
      if (Key == null || UploadId == null) return;
      console.debug("Delete Multipart Upload:", Key, UploadId);
      await s3.abortMultipartUpload({ Bucket: TestBucket, Key, UploadId }).promise();
    })
  );
};