Skip to content

Instantly share code, notes, and snippets.

@mryhryki
Last active August 12, 2023 03:43
Show Gist options
  • Save mryhryki/99d6821b7b05587ed1c3b688bd76c96d to your computer and use it in GitHub Desktop.
Save mryhryki/99d6821b7b05587ed1c3b688bd76c96d to your computer and use it in GitHub Desktop.
This is a sample to copy a file between two S3 buckets on four different ways with AWS SDK for JavaScript.

s3-copy

This is a sample to copy a file between two S3 buckets on four different ways with AWS SDK for JavaScript.

Setup

$ npm install

$ npm run setup
# and Edit .env file

Execute

# Use CopyObject
$ npm run CopyObject

# Use UploadPartCopy
$ npm run UploadPartCopy

# Use GetObject and PutObject
$ npm run GetAndPutObject

# Use GetObject and UploadPart
$ npm run GetObjectAndUploadPart

AWS Official Documents

REST API

SOURCE_AWS_ACCESS_KEY_ID="(REPLACE-YOUR-TOKEN)"
SOURCE_AWS_SECRET_ACCESS_KEY="(REPLACE-YOUR-TOKEN)"
# SOURCE_AWS_SESSION_TOKEN="(REPLACE-YOUR-TOKEN)" # OPTIONAL
SOURCE_AWS_S3_BUCKET_NAME="(REPLACE-YOUR-BUCKET-NAME)"
SOURCE_AWS_S3_OBJECT_KEY="(REPLACE-YOUR-OBJECT-KEY)"
SOURCE_AWS_REGION="(REPLACE-YOUR-BUCKET-REGION)"
# SOURCE_AWS_ENDPOINT="(REPLACE-YOUR-BUCKET-ENDPOINT)" # OPTIONAL
DEST_AWS_ACCESS_KEY_ID="(REPLACE-YOUR-TOKEN)"
DEST_AWS_SECRET_ACCESS_KEY="(REPLACE-YOUR-TOKEN)"
# DEST_AWS_SESSION_TOKEN="(REPLACE-YOUR-TOKEN)" # OPTIONAL
DEST_AWS_S3_BUCKET_NAME="(REPLACE-YOUR-BUCKET-NAME)"
DEST_AWS_S3_OBJECT_KEY="(REPLACE-YOUR-OBJECT-KEY)"
DEST_AWS_REGION="(REPLACE-YOUR-BUCKET-REGION)"
# DEST_AWS_ENDPOINT="(REPLACE-YOUR-BUCKET-ENDPOINT)" # OPTIONAL
.env
node_modules/**
package-lock.json
import { execute } from "./utils";
execute(async (source, dest): Promise<void> => {
await dest.instance
.copyObject({
Bucket: dest.bucket,
Key: dest.key,
CopySource: encodeURI(`${source.bucket}/${source.key}`),
})
.promise();
console.log("[CopyObject]");
});
import { execute } from "./utils";
execute(async (source, dest): Promise<void> => {
const getObjectResponse = await source.instance.getObject({ Bucket: source.bucket, Key: source.key }).promise();
const { Body, ContentType } = getObjectResponse;
console.log(`GetObject: ${getObjectResponse.ContentLength} Bytes`);
await dest.instance
.putObject({ Bucket: dest.bucket, Key: dest.key, Body, ContentType })
.promise();
console.log(`PutObject: ${getObjectResponse.ContentLength} Bytes`);
});
import { execute } from "./utils";
import { CompletedPartList } from "aws-sdk/clients/s3";
const MinMultipartSize = 5 * 1024 ** 2; // 5MiB
interface PartInfo {
partNumber: number;
rangeStart: number;
rangeEnd: number;
}
execute(async (source, dest): Promise<void> => {
const headObjectResponse = await source.instance
.headObject({
Bucket: source.bucket,
Key: source.key,
})
.promise();
const { ContentType, ContentLength } = headObjectResponse;
const createMultipartUploadResponse = await dest.instance
.createMultipartUpload({
Bucket: dest.bucket,
Key: dest.key,
ContentType,
})
.promise();
const UploadId = createMultipartUploadResponse?.UploadId ?? "";
console.log(`[CreateMultipartUpload] UploadId: ${UploadId}`);
const objectSize: number = ContentLength ?? 0;
const parts: PartInfo[] = Array.from({ length: Math.ceil(objectSize / MinMultipartSize) }).map(
(_, i): PartInfo => ({
partNumber: i + 1,
rangeStart: i * MinMultipartSize,
rangeEnd: ((i + 1) * MinMultipartSize < objectSize ? (i + 1) * MinMultipartSize : objectSize) - 1,
}),
);
const Parts: CompletedPartList = [];
for await (const { partNumber, rangeStart, rangeEnd } of parts) {
const getObjectResponse = await source.instance
.getObject({
Bucket: source.bucket,
Key: source.key,
Range: `bytes=${rangeStart}-${rangeEnd}`,
})
.promise();
const { Body } = getObjectResponse;
console.log(`[GetObject #${partNumber}] ${rangeStart}-${rangeEnd} Bytes`);
const uploadPartResponse = await dest.instance
.uploadPart({
Bucket: dest.bucket,
Key: dest.key,
PartNumber: partNumber,
UploadId,
Body,
})
.promise();
Parts.push({
ETag: uploadPartResponse.ETag,
PartNumber: partNumber,
});
console.log(`[UploadPart #${partNumber}] ${rangeStart}-${rangeEnd} Bytes, ETag: ${uploadPartResponse.ETag}`);
}
await dest.instance
.completeMultipartUpload({
Bucket: dest.bucket,
Key: dest.key,
MultipartUpload: { Parts },
UploadId,
})
.promise();
console.log(`[CompleteMultipartUpload] UploadId: ${UploadId}`);
});
{
"name": "s3-copy",
"author": "mryhryki",
"private": true,
"license": "MIT",
"engines": {
"node": "16.x",
"npm": "8.x"
},
"scripts": {
"CopyObject": "esbuild --platform=node --external:node:* --bundle ./CopyObject.ts | node",
"UploadPartCopy": "esbuild --platform=node --external:node:* --bundle ./UploadPartCopy.ts | node",
"GetAndPutObject": "esbuild --platform=node --external:node:* --bundle ./GetAndPutObject.ts | node",
"GetObjectAndUploadPart": "esbuild --platform=node --external:node:* --bundle ./GetObjectAndUploadPart.ts | node",
"setup": "test -f .env || cp .env{.example,}"
},
"dependencies": {
"@types/aws-sdk": "^2.7.0",
"@types/dotenv": "^8.2.0",
"@types/node": "^18.11.9",
"aws-sdk": "^2.1258.0",
"dotenv": "^16.0.3",
"esbuild": "^0.15.15",
"typescript": "^4.9.3"
}
}
import { execute } from "./utils";
import { CompletedPartList } from "aws-sdk/clients/s3";
const MinMultipartSize = 5 * 1024 ** 2; // 5MiB
interface PartInfo {
partNumber: number;
rangeStart: number;
rangeEnd: number;
}
execute(async (source, dest): Promise<void> => {
const headObjectResponse = await dest.instance
.headObject({
Bucket: source.bucket,
Key: source.key,
})
.promise();
const { ContentType, ContentLength } = headObjectResponse;
const createMultipartUploadResponse = await dest.instance
.createMultipartUpload({
Bucket: dest.bucket,
Key: dest.key,
ContentType,
})
.promise();
const UploadId = createMultipartUploadResponse?.UploadId ?? "";
console.log(`[CreateMultipartUpload] UploadId: ${UploadId}`);
const objectSize: number = ContentLength ?? 0;
const parts: PartInfo[] = Array.from({ length: Math.ceil(objectSize / MinMultipartSize) }).map(
(_, i): PartInfo => ({
partNumber: i + 1,
rangeStart: i * MinMultipartSize,
rangeEnd: ((i + 1) * MinMultipartSize < objectSize ? (i + 1) * MinMultipartSize : objectSize) - 1,
})
);
const Parts: CompletedPartList = [];
for await (const { partNumber, rangeStart, rangeEnd } of parts) {
const uploadPartCopyResponse = await dest.instance
.uploadPartCopy({
Bucket: dest.bucket,
Key: dest.key,
CopySource: encodeURI(`${source.bucket}/${source.key}`),
CopySourceRange: `bytes=${rangeStart}-${rangeEnd}`,
PartNumber: partNumber,
UploadId,
})
.promise();
console.log(`[UploadPartCopy #${partNumber}] ${rangeStart}-${rangeEnd} Bytes`);
Parts.push({
ETag: uploadPartCopyResponse.CopyPartResult?.ETag ?? "",
PartNumber: partNumber,
});
}
await dest.instance
.completeMultipartUpload({
Bucket: dest.bucket,
Key: dest.key,
MultipartUpload: { Parts },
UploadId,
})
.promise();
console.log(`[CompleteMultipartUpload] UploadId: ${UploadId}`);
});
import * as dotenv from "dotenv"; // https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import
import { Endpoint, S3 } from "aws-sdk";
dotenv.config();
export const execute = (callback: (source: AwsConfig, dest: AwsConfig) => Promise<void>) => {
console.log("##### START #####");
const source = getAwsConfig("SOURCE");
const dest = getAwsConfig("DEST");
callback(source, dest)
.then(() => {
console.log("##### END #####");
})
.catch((err) => {
console.error("##### ERROR #####");
console.error(err);
});
};
interface AwsConfig {
instance: S3;
bucket: string;
key: string;
}
const getAwsConfig = (type: "SOURCE" | "DEST"): AwsConfig => {
const endpoint = getOptionalEnvVar(`${type}_AWS_ENDPOINT`);
const instance = new S3({
apiVersion: "2006-03-01",
region: getRequiredEnvVar(`${type}_AWS_REGION`),
endpoint: endpoint == null ? undefined : new Endpoint(endpoint),
accessKeyId: getRequiredEnvVar(`${type}_AWS_ACCESS_KEY_ID`),
secretAccessKey: getRequiredEnvVar(`${type}_AWS_SECRET_ACCESS_KEY`),
sessionToken: getOptionalEnvVar(`${type}_AWS_SESSION_TOKEN`),
});
return {
instance,
bucket: getRequiredEnvVar(`${type}_AWS_S3_BUCKET_NAME`),
key: getRequiredEnvVar(`${type}_AWS_S3_OBJECT_KEY`),
};
};
const getOptionalEnvVar = (name: string): string | undefined => process.env[name];
const getRequiredEnvVar = (name: string): string => {
const value = getOptionalEnvVar(name);
if (!value) throw new Error(`Missing environment variable ${name}`);
return value;
};
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment