Minio Migrate All - Script to migrate all buckets and files from one S3 server to another.

script.js

/**
 * S3 migration script
 *
 * Use this script to migrate all buckets and files from one S3 server to another.
 *
 * Author: Lawrence Cherone
 */

const HOSTS = {
    from: {
        ip: 'old-server.example.com',
        port: 443,
        useSSL: true,
        access_key: 'minio-access_key',
        access_secret: 'minio-access_secret',
        region: 'eu-west-2'
    },
    to: {
        ip: 'new-server.example.com',
        port: 443,
        useSSL: true,
        access_key: 'minio-access_key',
        access_secret: 'minio-access_secret',
        region: 'eu-west-2'
    }
}

//
const {
    Client
} = require('minio')

//
const clients = {
    from: new Client({
        endPoint: HOSTS.from.ip,
        port: parseInt(HOSTS.from.port, 10),
        useSSL: HOSTS.from.useSSL,
        accessKey: HOSTS.from.access_key,
        secretKey: HOSTS.from.access_secret,
        region: HOSTS.from.region
    }),
    to: new Client({
        endPoint: HOSTS.to.ip,
        port: parseInt(HOSTS.to.port, 10),
        useSSL: HOSTS.to.useSSL,
        accessKey: HOSTS.to.access_key,
        secretKey: HOSTS.to.access_secret,
        region: HOSTS.to.region
    })
}

//
async function listBuckets(host = '') {
    let buckets = await clients[host].listBuckets()
    return buckets.length ? buckets.map(i => i.name) : []
}

//
async function listObjects(host = '', bucket = '', prefix = '', recursive = false, startAfter = '') {
    return new Promise(async (resolve, reject) => {
        try {
            let objects = []

            await clients[host].listObjectsV2(bucket, prefix, recursive, startAfter)
                .on('data', obj => (obj && (obj.name || obj.prefix)) ? objects.push(obj) : undefined)
                .on('error', e => reject(e))
                .on('end', () => resolve(objects))
        } catch (e) {
            reject(e)
        }
    })
}

// recursive list to work around <= 1000 file list limit
async function listObjectsAll(host = '', bucket = '', prefix = '', recursive = false, startAfter = '') {
    let files = await listObjects(host, bucket, prefix, recursive, startAfter)
    if (files.length === 1000) {
        let startAfter = files[files.length - 1].name
        return files.concat(await listObjectsAll(host, bucket, prefix, recursive, startAfter))
    } else {
        return files
    }
}

//
async function stream(bucket = '', prefix = '') {
    return new Promise(async (resolve, reject) => {
        try {
            await clients.to.putObject(bucket, prefix, await clients.from.getObject(bucket, prefix))
            console.log('File copied: [%s]: %s', bucket, prefix)
            resolve(true)
        } catch (e) {
            console.log('File not copied: [%s]: %s - %s', bucket, prefix, e.message)
            reject(e)
        }
    })
}

//
;
(async function () {
    try {
        // get buckets
        for (bucket of await listBuckets('from')) {
            // make bucket
            try {
                await clients.to.makeBucket(bucket, 'eu-west-2')
                console.log('Bucket created: [%s]', bucket)
            } catch {
                console.log('Bucket exists: [%s]', bucket)
            }

            // migrate files
            for (file of await listObjectsAll('from', bucket, '', true))
                await stream(bucket, file.name)
        }
    } catch (e) {
        console.error(e)
    }
})()