Skip to content

Instantly share code, notes, and snippets.

@dayitv89
Last active May 14, 2022 01:06
Show Gist options
  • Save dayitv89/fcd7cac4762fe28f1d719800a2f86015 to your computer and use it in GitHub Desktop.
Save dayitv89/fcd7cac4762fe28f1d719800a2f86015 to your computer and use it in GitHub Desktop.
Transform mysql 8 data into mongodb;
{
"dependencies": {
"mongodb": "^4.6.0",
"mysql2": "^2.3.3"
}
}
var mysql = require('mysql2');
var MongoClient = require('mongodb').MongoClient;
var mysql_config = {
host: 'localhost',
user: 'root',
password: 'root',
port: 3306,
database: 'mysql_db_name',
};
var mongo_uri = 'mongodb://root:root@localhost:27017/?authSource=admin';
var mongo_db_name = 'mongo_db_name';
function getMysqlTables(mysqlConnection, callback) {
mysqlConnection.query(
`SELECT table_name FROM information_schema.tables WHERE table_schema = '${mysql_config.database}' ORDER BY table_name`,
function (error, results, fields) {
if (error) {
callback(error);
} else {
var tables = results.map((r) => r.TABLE_NAME);
console.log('total mysql tables:', tables.length);
callback(null, tables);
}
},
);
}
function tableToCollection(mysqlConnection, tableName, mongoCollection, callback) {
var sql = 'SELECT * FROM `' + tableName + '`;';
mysqlConnection.query(sql, function (error, results, fields) {
console.log('loading data from mysql', tableName, 'result:', !error);
if (error) {
callback(error);
} else {
console.log('inserting:', tableName, 'records:', results.length);
if (results.length == 0) {
results = [
fields.reduce((acc, f) => {
acc[f.name] = 'BLANK';
return acc;
}, {}),
];
}
mongoCollection.insertMany(results, {}, function (error) {
if (error) {
callback(error);
} else {
callback(null);
}
});
}
});
}
MongoClient.connect(mongo_uri, async function (error, client) {
if (error) throw error;
var db = client.db(mongo_db_name);
var MysqlCon = mysql.createConnection(mysql_config);
MysqlCon.connect();
var jobs = 0;
getMysqlTables(MysqlCon, function (error, tables) {
tables.forEach(function (table) {
var collection = db.collection(table);
++jobs;
tableToCollection(MysqlCon, table, collection, function (error) {
if (error) throw error;
--jobs;
});
});
});
console.log('remaining:', jobs);
var interval = setInterval(function () {
console.log('remaining:', jobs);
if (jobs <= 0) {
clearInterval(interval);
client.close();
MysqlCon.end();
console.log('done!');
console.log('tap `cmd/ctrl+c` to exit');
}
}, 300);
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment