Skip to content

Instantly share code, notes, and snippets.

@lcherone

lcherone/script.js

Last active Dec 12, 2020
Embed
What would you like to do?
Minio Migrate All - Script to migrate all buckets and files from one S3 server to another.
/**
* S3 migration script
*
* Use this script to migrate all buckets and files from one S3 server to another.
*
* Author: Lawrence Cherone
*/
const HOSTS = {
from: {
ip: 'old-server.example.com',
port: 443,
useSSL: true,
access_key: 'minio-access_key',
access_secret: 'minio-access_secret',
region: 'eu-west-2'
},
to: {
ip: 'new-server.example.com',
port: 443,
useSSL: true,
access_key: 'minio-access_key',
access_secret: 'minio-access_secret',
region: 'eu-west-2'
}
}
//
const {
Client
} = require('minio')
//
const clients = {
from: new Client({
endPoint: HOSTS.from.ip,
port: parseInt(HOSTS.from.port, 10),
useSSL: HOSTS.from.useSSL,
accessKey: HOSTS.from.access_key,
secretKey: HOSTS.from.access_secret,
region: HOSTS.from.region
}),
to: new Client({
endPoint: HOSTS.to.ip,
port: parseInt(HOSTS.to.port, 10),
useSSL: HOSTS.to.useSSL,
accessKey: HOSTS.to.access_key,
secretKey: HOSTS.to.access_secret,
region: HOSTS.to.region
})
}
//
async function listBuckets(host = '') {
let buckets = await clients[host].listBuckets()
return buckets.length ? buckets.map(i => i.name) : []
}
//
async function listObjects(host = '', bucket = '', prefix = '', recursive = false, startAfter = '') {
return new Promise(async (resolve, reject) => {
try {
let objects = []
await clients[host].listObjectsV2(bucket, prefix, recursive, startAfter)
.on('data', obj => (obj && (obj.name || obj.prefix)) ? objects.push(obj) : undefined)
.on('error', e => reject(e))
.on('end', () => resolve(objects))
} catch (e) {
reject(e)
}
})
}
// recursive list to work around <= 1000 file list limit
async function listObjectsAll(host = '', bucket = '', prefix = '', recursive = false, startAfter = '') {
let files = await listObjects(host, bucket, prefix, recursive, startAfter)
if (files.length === 1000) {
let startAfter = files[files.length - 1].name
return files.concat(await listObjectsAll(host, bucket, prefix, recursive, startAfter))
} else {
return files
}
}
//
async function stream(bucket = '', prefix = '') {
return new Promise(async (resolve, reject) => {
try {
await clients.to.putObject(bucket, prefix, await clients.from.getObject(bucket, prefix))
console.log('File copied: [%s]: %s', bucket, prefix)
resolve(true)
} catch (e) {
console.log('File not copied: [%s]: %s - %s', bucket, prefix, e.message)
reject(e)
}
})
}
//
;
(async function () {
try {
// get buckets
for (bucket of await listBuckets('from')) {
// make bucket
try {
await clients.to.makeBucket(bucket, 'eu-west-2')
console.log('Bucket created: [%s]', bucket)
} catch {
console.log('Bucket exists: [%s]', bucket)
}
// migrate files
for (file of await listObjectsAll('from', bucket, '', true))
await stream(bucket, file.name)
}
} catch (e) {
console.error(e)
}
})()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment