Simple Node.js script to transfer data from one AWS DynamoDB table to another table or account

dynamodb-transfer.js

const { spawnSync } = require('child_process');

const options = {
	awsCliBinary: '/usr/local/bin/aws',
	awsProfileSource: 'profile1',
	awsRegionSource: 'eu-west-1',
	awsTableSource: 'table1',
	awsProfileTarget: 'profile2',
	awsRegionTarget: 'eu-west-2',
	awsTableTarget: 'table2',
};



/* Retrieve items from source table
 * Warning: As per default this will download no more than 1 MB of data!
 * See: https://docs.aws.amazon.com/cli/latest/reference/dynamodb/scan.html
 */

function downloadDump() {
	const result = spawnSync(
		options.awsCliBinary,
		[
			'dynamodb',
			'scan',
			'--profile',
			options.awsProfileSource,
			'--region',
			options.awsRegionSource,
			'--table-name',
			options.awsTableSource,
			'--output',
			'json',
			// '--max-items',
			// '100',
		],
		{
			maxBuffer: 2 * 1024 * 1024 * 1024, // 2 MB
			encoding: 'utf8',
		}
	);

	if (result.status !== 0) {
		const err = result.error || new Error(`Table scan failed with exit code ${result.status}. ${result.stderr}`); 
		throw err;
	}

	const dump = JSON.parse(result.stdout);
	return dump;
}

const dump = downloadDump();
const items = dump.Items;
console.log('Number of items in source table:', dump.Count);
console.log('Number of items downloaded:', items.length);



/* Generate put requests for all items */

const requests = items.map(item => ({
	PutRequest: {
		Item: item
	}
}));



/* Batch-process items until all are processed */

function syncSleep(time) {
	// Note: This a very quick and dirty way of doing this!
	var end = new Date().getTime() + time;
	while (new Date().getTime() < end) { /* nothing */ }
}

function batchUpload() {
	const itemsForCurrentRun = [];
	while (requests.length > 0 && itemsForCurrentRun.length < 25) {
		const item = requests.pop();
		itemsForCurrentRun.push(item);
	}
	const requestItemsString = JSON.stringify({
		[options.awsTableTarget]: itemsForCurrentRun
	});
	console.log(`Processing ${itemsForCurrentRun.length} items...`);

	const result = spawnSync(
		options.awsCliBinary,
		[
			'dynamodb',
			'batch-write-item',
			'--profile',
			options.awsProfileTarget,
			'--region',
			options.awsRegionTarget,
			'--output',
			'json',
			'--request-items',
			requestItemsString,
		],
		{
			maxBuffer: 2 * 1024 * 1024 * 1024, // 2 MB
			encoding: 'utf8',
		}
	);

	if (result.status !== 0) {
		const err = result.error || new Error(`Batch writing failed with exit code ${result.status}. ${result.stderr}`);
		throw err;
	}

	if (result.stdout) {
		const json = JSON.parse(result.stdout);
		const unprocessedItems = (json.UnprocessedItems && json.UnprocessedItems[options.awsTableTarget]) || [];
		if (unprocessedItems.length) {
			console.log(`Unprocessed items: ${unprocessedItems.length} (sleep for 2s)`);
			unprocessedItems.forEach(item => requests.push(item));
			syncSleep(2000);
		}
	}
}

while (requests.length > 0) {
	batchUpload();
}

console.log('Done.');