Mongodb Index exporter importer

readme.md

MongoDB Index Exporter and Importer

• Scripts for exporting and importing MongoDB indexes
• Just execute export/import script with mongo shell command without install!

Prerequisite

• MongoShell(mongo command) shoule be required

How to

1. Export mongodb indexes (create index.js)

mongo dbname --quiet mongodbIndexExporter.js > index.js

2. Import index.js to your db

index.js is required

There are 2 modes executing mongodbIndexImporter.js

• TEST MODE (This will not change anything. Just see the result preview)

mongo dbname --quiet --eval "let mode='DRYRUN';" mongodbIndexImporter.js

• LIVE MODE (This will change db index)

mongo dbname --quiet --eval "let mode='LIVE';" mongodbIndexImporter.js

mongodbIndexExporter.js

/*
How to export
* mongo is mongoshell command

mongo dbname --quiet mongodbIndexExporter.js > index.js
*/
let collectionNames = db.getCollectionNames();
let index_data = {'collections': []};

for (let i in collectionNames) {
    let collection = collectionNames[i];
    let indexes;
    try {
        indexes = db[collection].getIndexes();
    } catch (err) { continue; }
    
    let json = { 'name': collection, 'indexes': [] };

    for (let j in indexes) {
        let idx = indexes[j];
        let doc = { 'name': idx['name'], 'key': idx['key'] };
        if ('expireAfterSeconds' in idx) {
            doc['expireAfterSeconds'] = idx['expireAfterSeconds'];
        }

        if ('background' in idx) {
            doc['background'] = idx['background'];
        }

        if ('unique' in idx) {
            doc['unique'] = idx['unique'];
        }

        if('collation' in idx) {
            doc['collation'] = {
                'locale': idx['collation']['locale'],
                'strength': idx['collation']['strength'],
            };
        }

        json['indexes'].push(doc)
    }
    index_data['collections'].push(json);
};
print('let indexes = ');
printjson(index_data);
print(';');

mongodbIndexImporter.js

/*
How to (required index.js exported from exporter.js)
* mongo is mongoshell command

mongo dbname --quiet --eval "let mode='DRYRUN';" mongodbIndexImporter.js

 */

// available modes
let LIVE = 'LIVE';
let DRYRUN = 'DRYRUN'; // actually all other words except 'LIVE' are DRYRUN mode.

load('index.js');
if(typeof mode == 'undefined') {
    print(red('mode variable required!')+' '+cyan('--eval "let mode=\'DRYRUN\';" or "let mode=\'LIVE\';'));
}
else {
    print('*** Index importer: started with ' + yellow(mode) + ' mode ***');

    let collections = indexes['collections'];
    let collectionNames = db.getCollectionNames();

    for (let i in collections) {
        let collection = collections[i];
        let indexItems = collection["indexes"];
        let collectionName = collection['name'];
        print('='.repeat(55));
        if (collectionNames.indexOf(collectionName) >=0) {
            print('collection '+ green(collectionName) + ' already '+green('exists'));
        } else {
            print('will '+yellow('create')+' collection '+ green(collectionName));
            if (mode == LIVE) {
                // do create collection
                db.createCollection(collectionName)
            }
        }

        let dbCollection = db.getCollection(collectionName);
        let existingIndexes = dbCollection.getIndexes();

        for(let itemIdx in indexItems) {
            let indexItem = indexItems[itemIdx];
            let indexName = indexItem['name'];
            let indexKey = indexItem['key'];
            let checked = false;
            for(let eIdx in existingIndexes) {
                let existingIndex = existingIndexes[eIdx];
                if (existingIndex['name'] == indexName) {
                    if(!isSameKey(existingIndex, indexItem)) {
                        print('collection '+green(collectionName)+' index '+green(indexName)+' will be '+red('dropped'));
                        print('collection '+green(collectionName)+' index '+green(indexName)+' will be '+yellow('re-created'));
                        if(mode == LIVE) {
                            dbCollection.dropIndex(indexName);
                            createIndex(dbCollection, indexName, indexItem);
                        }
                    }
                    else {
                        print('collection '+green(collectionName)+' index '+green(indexName)+' is same, skip.');
                    }
                    checked = true;
                    break
                }
            }

            if (!checked) {
                print('collection '+green(collectionName)+' index '+green(indexName)+' will be '+yellow('created'));
                if(mode == LIVE) {
                    createIndex(dbCollection, indexName, indexItem);
                }
            }
        }
        
        for(let eIdx in existingIndexes) {
            let checked = false;
            let existingIndex = existingIndexes[eIdx];
            for(let itemIdx in indexItems) {
                let indexItem = indexItems[itemIdx];
                let indexName = indexItem['name'];
                if (existingIndex['name'] == indexName) {
                    checked = true
                    break
                }
            }

            if (!checked) {
                print('collection '+green(collectionName)+' index '+green(existingIndex["name"])+' will be '+red('dropped'));
                if (mode == LIVE) {
                    dbCollection.dropIndex(existingIndex["name"]);
                }
            }
        }
    }

    if (mode == LIVE) {
        print('Completed! '+magenta('LIVE MODE. All Operatoins are applied on database!'));
    }
    else {
        print('Completed! '+cyan('TEST MODE. Nothing is applied.'));
    }
}

function createIndex(db, name, indexItem) {
    let options = {'name': name};
    let key = indexItem['key'];
    if('expireAfterSeconds' in indexItem) {
        options['expireAfterSeconds'] = indexItem['expireAfterSeconds'];
    }

    if('background' in indexItem) {
        options['background'] = indexItem['background'];
    }

    if('unique' in indexItem) {
        options['unique'] = indexItem['unique'];
    }
    
    if('collation' in indexItem) {
        options['collation'] = indexItem['collation'];
    }
    
    db.createIndex(key, options);
}

function isSameKey(existingIndex, newIndex) {
    let key = existingIndex['key'];
    let newKey = newIndex['key'];

    // -2: v, ns
    let excepted = 0;
    for (let keyName in existingIndex) {
        if (keyName === "v") {
            excepted++
            continue
        }

        if (keyName === "ns") {
            excepted++
        }
    }
    
    if (Object.keys(existingIndex).length -excepted != Object.keys(newIndex).length) {
        return false;
    }

    if (Object.keys(key).length != Object.keys(newKey).length) {
        return false;
    }

    for(let i in key) {
        let val = key[i];
        if(i in newKey) {
            if(newKey[i] != val) {
                return false;
            }
        }
        else {
            return false;
        }
    }
    let except = ['v', 'ns', 'key'];
    for (let element in existingIndex) {
        if (except.indexOf(element) >= 0) {
            continue;
        }

        if (existingIndex[element] != newIndex[element]) {
            let a = existingIndex[element];
            let b = newIndex[element];
            if (isObject(a) && isObject(b)) {
                print(element)
                for (let el in b) {
                    if (a[el] != b[el]) {
                        return false;
                    }
                }
                return true;

            }
            return false;
        }
    }
    return true;
}

function isObject(val) {
    if (val === null) { return false;}
    return ( (typeof val === 'function') || (typeof val === 'object') );
}

// colors
function red (text) { return colorize(text, 'red') }
function green (text) { return colorize(text, 'green') }
function yellow (text) { return colorize(text, 'yellow') }
function blue (text) { return colorize(text, 'blue') }
function magenta (text) { return colorize(text, 'magenta') }
function cyan (text) { return colorize(text, 'cyan') }
function gray (text) { return colorize(text, 'gray') }

function colorize(text, color, style) {
    // color: 'red', 'green', 'blue'... (see below)
    // styles: 'normal' or undefined, 'bright', 'highlight'

    if (!style) {
        style = 'normal';
    }

    let _ansi = {
        csi: String.fromCharCode(0x1B) + '[',
        reset:      '0',
        text_prop:  'm',

        styles: {
            normal: '3',
            bright:     '9',
            highlight:  '4'
        },

        colors: {
            black:   '0',
            red:     '1',
            green:   '2',
            yellow:  '3',
            blue:    '4',
            magenta: '5',
            cyan:    '6',
            gray:    '7'
        }
    };

    let beginColor = _ansi.csi + _ansi.styles[style] + _ansi.colors[color] + _ansi.text_prop;
    let endColor = _ansi.csi + _ansi.reset + _ansi.text_prop;

    return beginColor + text + endColor;
}

z0.png

z1.png

Work well! Nice job!
One improvement, if there are view collections, the exporter will crash with error Namespace xx.xxx is a view, not a collection, changing line #12 let indexes = db[collection].getIndexes(); can fix this:

    // let indexes = db[collection].getIndexes();
    let indexes;
    try {
      indexes = db[collection].getIndexes();
    } catch (err) { continue; }

Work well! Nice job!
One improvement, if there are view collections, the exporter will crash with error Namespace xx.xxx is a view, not a collection, changing line #12 let indexes = db[collection].getIndexes(); can fix this:

    // let indexes = db[collection].getIndexes();
    let indexes;
    try {
      indexes = db[collection].getIndexes();
    } catch (err) { continue; }

Thank you for feedback. This is applied :)

hi,

What version of mongo shell to use this script?

@tungtnv Hi, What version do you use? Can you give me error message with version?
Currently, I'm using this on mongodb 4.4.
But If this script is not working on new version, I'll consider updating or creating another file on my gist.

Hi, my mongodb 's version is 5.x

Can you create another script for new mongodb version?

Tks

for executing this script on 6x version, add following code in first 'for' loop in Export script next to existing 'if' conditions-
if ('v' in idx) {
doc['v'] = idx['v'];
}
if ('weights' in idx) {
doc['weights'] = idx['weights'];
}
if ('ns' in idx) {
doc['ns'] = idx['ns'];
}
if ('default_language' in idx) {
doc['default_language'] = idx['default_language'];
}
if ('language_override' in idx) {
doc['language_override'] = idx['language_override'];
}
if ('textIndexVersion' in idx) {
doc['textIndexVersion'] = idx['textIndexVersion'];
}

And add following code in createIndex() of Import script next to existing 'if' conditions -
if('v' in indexItem) {
options['v'] = indexItem['v'];
}
if('expireAfterSeconds' in indexItem) {
options['expireAfterSeconds'] = indexItem['expireAfterSeconds'];
}
if('weights' in indexItem) {
options['weights'] = indexItem['weights'];
}
if('ns' in indexItem) {
options['ns'] = indexItem['ns'];
}
if('default_language' in indexItem) {
options['default_language'] = indexItem['default_language'];
}
if('language_override' in indexItem) {
options['language_override'] = indexItem['language_override'];
}
if('textIndexVersion' in indexItem) {
options['textIndexVersion'] = indexItem['textIndexVersion'];
}

The advance version of mongodb needs some additional properties for creating the indexes, so by adding above properties, it worked for me.

Thank you,
Sandeep