This is a sample to copy a file between two S3 buckets on four different ways with AWS SDK for JavaScript.

!README.md

s3-copy

This is a sample to copy a file between two S3 buckets on four different ways with AWS SDK for JavaScript.

Setup

$ npm install

$ npm run setup
# and Edit .env file

Execute

# Use CopyObject
$ npm run CopyObject

# Use UploadPartCopy
$ npm run UploadPartCopy

# Use GetObject and PutObject
$ npm run GetAndPutObject

# Use GetObject and UploadPart
$ npm run GetObjectAndUploadPart

AWS Official Documents

• Class: AWS.S3 — AWS SDK for JavaScript

REST API

• GetObject - Amazon Simple Storage Service
• HeadObject - Amazon Simple Storage Service
• PutObject - Amazon Simple Storage Service
• CopyObject - Amazon Simple Storage Service
• CreateMultipartUpload - Amazon Simple Storage Service
• UploadPartCopy - Amazon Simple Storage Service
• CompleteMultipartUpload - Amazon Simple Storage Service

.env.example

SOURCE_AWS_ACCESS_KEY_ID="(REPLACE-YOUR-TOKEN)"
SOURCE_AWS_SECRET_ACCESS_KEY="(REPLACE-YOUR-TOKEN)"
# SOURCE_AWS_SESSION_TOKEN="(REPLACE-YOUR-TOKEN)" # OPTIONAL
SOURCE_AWS_S3_BUCKET_NAME="(REPLACE-YOUR-BUCKET-NAME)"
SOURCE_AWS_S3_OBJECT_KEY="(REPLACE-YOUR-OBJECT-KEY)"
SOURCE_AWS_REGION="(REPLACE-YOUR-BUCKET-REGION)"
# SOURCE_AWS_ENDPOINT="(REPLACE-YOUR-BUCKET-ENDPOINT)" # OPTIONAL

DEST_AWS_ACCESS_KEY_ID="(REPLACE-YOUR-TOKEN)"
DEST_AWS_SECRET_ACCESS_KEY="(REPLACE-YOUR-TOKEN)"
# DEST_AWS_SESSION_TOKEN="(REPLACE-YOUR-TOKEN)" # OPTIONAL
DEST_AWS_S3_BUCKET_NAME="(REPLACE-YOUR-BUCKET-NAME)"
DEST_AWS_S3_OBJECT_KEY="(REPLACE-YOUR-OBJECT-KEY)"
DEST_AWS_REGION="(REPLACE-YOUR-BUCKET-REGION)"
# DEST_AWS_ENDPOINT="(REPLACE-YOUR-BUCKET-ENDPOINT)" # OPTIONAL

.gitignore

.env
node_modules/**
package-lock.json

CopyObject.ts

import { execute } from "./utils";

execute(async (source, dest): Promise<void> => {
  await dest.instance
    .copyObject({
      Bucket: dest.bucket,
      Key: dest.key,
      CopySource: encodeURI(`${source.bucket}/${source.key}`),
    })
    .promise();
  console.log("[CopyObject]");
});

GetAndPutObject.ts

import { execute } from "./utils";

execute(async (source, dest): Promise<void> => {
  const getObjectResponse = await source.instance.getObject({ Bucket: source.bucket, Key: source.key }).promise();
  const { Body, ContentType } = getObjectResponse;
  console.log(`GetObject: ${getObjectResponse.ContentLength} Bytes`);

  await dest.instance
    .putObject({ Bucket: dest.bucket, Key: dest.key, Body, ContentType })
    .promise();
  console.log(`PutObject: ${getObjectResponse.ContentLength} Bytes`);
});

GetObjectAndUploadPart.ts

import { execute } from "./utils";
import { CompletedPartList } from "aws-sdk/clients/s3";

const MinMultipartSize = 5 * 1024 ** 2; // 5MiB
interface PartInfo {
  partNumber: number;
  rangeStart: number;
  rangeEnd: number;
}

execute(async (source, dest): Promise<void> => {
  const headObjectResponse = await source.instance
    .headObject({
      Bucket: source.bucket,
      Key: source.key,
    })
    .promise();
  const { ContentType, ContentLength } = headObjectResponse;

  const createMultipartUploadResponse = await dest.instance
    .createMultipartUpload({
      Bucket: dest.bucket,
      Key: dest.key,
      ContentType,
    })
    .promise();
  const UploadId = createMultipartUploadResponse?.UploadId ?? "";
  console.log(`[CreateMultipartUpload] UploadId: ${UploadId}`);

  const objectSize: number = ContentLength ?? 0;
  const parts: PartInfo[] = Array.from({ length: Math.ceil(objectSize / MinMultipartSize) }).map(
    (_, i): PartInfo => ({
      partNumber: i + 1,
      rangeStart: i * MinMultipartSize,
      rangeEnd: ((i + 1) * MinMultipartSize < objectSize ? (i + 1) * MinMultipartSize : objectSize) - 1,
    }),
  );

  const Parts: CompletedPartList = [];
  for await (const { partNumber, rangeStart, rangeEnd } of parts) {
    const getObjectResponse = await source.instance
      .getObject({
        Bucket: source.bucket,
        Key: source.key,
        Range: `bytes=${rangeStart}-${rangeEnd}`,
      })
      .promise();
    const { Body } = getObjectResponse;
    console.log(`[GetObject #${partNumber}] ${rangeStart}-${rangeEnd} Bytes`);

    const uploadPartResponse = await dest.instance
      .uploadPart({
        Bucket: dest.bucket,
        Key: dest.key,
        PartNumber: partNumber,
        UploadId,
        Body,
      })
      .promise();
    Parts.push({
      ETag: uploadPartResponse.ETag,
      PartNumber: partNumber,
    });
    console.log(`[UploadPart #${partNumber}] ${rangeStart}-${rangeEnd} Bytes, ETag: ${uploadPartResponse.ETag}`);
  }

  await dest.instance
    .completeMultipartUpload({
      Bucket: dest.bucket,
      Key: dest.key,
      MultipartUpload: { Parts },
      UploadId,
    })
    .promise();
  console.log(`[CompleteMultipartUpload] UploadId: ${UploadId}`);
});

package.json

{
  "name": "s3-copy",
  "author": "mryhryki",
  "private": true,
  "license": "MIT",
  "engines": {
    "node": "16.x",
    "npm": "8.x"
  },
  "scripts": {
    "CopyObject": "esbuild --platform=node --external:node:* --bundle ./CopyObject.ts | node",
    "UploadPartCopy": "esbuild --platform=node --external:node:* --bundle ./UploadPartCopy.ts | node",
    "GetAndPutObject": "esbuild --platform=node --external:node:* --bundle ./GetAndPutObject.ts | node",
    "GetObjectAndUploadPart": "esbuild --platform=node --external:node:* --bundle ./GetObjectAndUploadPart.ts | node",
    "setup": "test -f .env || cp .env{.example,}"
  },
  "dependencies": {
    "@types/aws-sdk": "^2.7.0",
    "@types/dotenv": "^8.2.0",
    "@types/node": "^18.11.9",
    "aws-sdk": "^2.1258.0",
    "dotenv": "^16.0.3",
    "esbuild": "^0.15.15",
    "typescript": "^4.9.3"
  }
}

UploadPartCopy.ts

import { execute } from "./utils";
import { CompletedPartList } from "aws-sdk/clients/s3";

const MinMultipartSize = 5 * 1024 ** 2; // 5MiB
interface PartInfo {
  partNumber: number;
  rangeStart: number;
  rangeEnd: number;
}

execute(async (source, dest): Promise<void> => {
  const headObjectResponse = await dest.instance
    .headObject({
      Bucket: source.bucket,
      Key: source.key,
    })
    .promise();
  const { ContentType, ContentLength } = headObjectResponse;

  const createMultipartUploadResponse = await dest.instance
    .createMultipartUpload({
      Bucket: dest.bucket,
      Key: dest.key,
      ContentType,
    })
    .promise();
  const UploadId = createMultipartUploadResponse?.UploadId ?? "";
  console.log(`[CreateMultipartUpload] UploadId: ${UploadId}`);

  const objectSize: number = ContentLength ?? 0;
  const parts: PartInfo[] = Array.from({ length: Math.ceil(objectSize / MinMultipartSize) }).map(
    (_, i): PartInfo => ({
      partNumber: i + 1,
      rangeStart: i * MinMultipartSize,
      rangeEnd: ((i + 1) * MinMultipartSize < objectSize ? (i + 1) * MinMultipartSize : objectSize) - 1,
    })
  );

  const Parts: CompletedPartList = [];
  for await (const { partNumber, rangeStart, rangeEnd } of parts) {
    const uploadPartCopyResponse = await dest.instance
      .uploadPartCopy({
        Bucket: dest.bucket,
        Key: dest.key,
        CopySource: encodeURI(`${source.bucket}/${source.key}`),
        CopySourceRange: `bytes=${rangeStart}-${rangeEnd}`,
        PartNumber: partNumber,
        UploadId,
      })
      .promise();
    console.log(`[UploadPartCopy #${partNumber}] ${rangeStart}-${rangeEnd} Bytes`);
    Parts.push({
      ETag: uploadPartCopyResponse.CopyPartResult?.ETag ?? "",
      PartNumber: partNumber,
    });
  }

  await dest.instance
    .completeMultipartUpload({
      Bucket: dest.bucket,
      Key: dest.key,
      MultipartUpload: { Parts },
      UploadId,
    })
    .promise();
  console.log(`[CompleteMultipartUpload] UploadId: ${UploadId}`);
});

utils.ts

import * as dotenv from "dotenv"; // https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
import { Endpoint, S3 } from "aws-sdk";

dotenv.config();

export const execute = (callback: (source: AwsConfig, dest: AwsConfig) => Promise<void>) => {
  console.log("##### START #####");
  const source = getAwsConfig("SOURCE");
  const dest = getAwsConfig("DEST");
  callback(source, dest)
    .then(() => {
      console.log("##### END #####");
    })
    .catch((err) => {
      console.error("##### ERROR #####");
      console.error(err);
    });
};

interface AwsConfig {
  instance: S3;
  bucket: string;
  key: string;
}

const getAwsConfig = (type: "SOURCE" | "DEST"): AwsConfig => {
  const endpoint = getOptionalEnvVar(`${type}_AWS_ENDPOINT`);
  const instance = new S3({
    apiVersion: "2006-03-01",
    region: getRequiredEnvVar(`${type}_AWS_REGION`),
    endpoint: endpoint == null ? undefined : new Endpoint(endpoint),
    accessKeyId: getRequiredEnvVar(`${type}_AWS_ACCESS_KEY_ID`),
    secretAccessKey: getRequiredEnvVar(`${type}_AWS_SECRET_ACCESS_KEY`),
    sessionToken: getOptionalEnvVar(`${type}_AWS_SESSION_TOKEN`),
  });
  return {
    instance,
    bucket: getRequiredEnvVar(`${type}_AWS_S3_BUCKET_NAME`),
    key: getRequiredEnvVar(`${type}_AWS_S3_OBJECT_KEY`),
  };
};

const getOptionalEnvVar = (name: string): string | undefined => process.env[name];
const getRequiredEnvVar = (name: string): string => {
  const value = getOptionalEnvVar(name);
  if (!value) throw new Error(`Missing environment variable ${name}`);
  return value;
};