Export and import data from a Kuzzle index

dump-kuzzle-index.js

const
  fs = require('fs'),
  mkdirp = require('mkdirp'),
  {
    Kuzzle,
    WebSocket
  } = require('kuzzle-sdk');

const kuzzle = new Kuzzle(
  new WebSocket('localhost', {
    port: 7512
  }));

async function dump_collection(index, collection, filename, directoryName) {
  console.log(`Dumping collection: ${collection}`);
  const
    documents = [],
    searchOptions = {
      scroll: '1m',
      size: 100
    };

  let searchResult = await kuzzle.document.search(index, collection, {}, searchOptions);
  while (searchResult) {
    documents.push(...searchResult.hits)
    try {
      searchResult = await searchResult.next();
    } catch (e) {
      searchResult = undefined;
    }
  }

  const bulkDocs = [];

  documents.forEach(doc => {
    bulkDocs.push({
      create: {
        _id: doc._id,
        _index: doc._index,
        _type: doc._type
      }
    })

    delete doc._source._kuzzle_info;

    bulkDocs.push(doc._source)
  });

  mkdirp.sync(directoryName);
  fs.writeFileSync(`${directoryName}/${filename}`, JSON.stringify(bulkDocs, undefined, 2))
}



async function dump_index(index, directoryName) {
  console.log(`Dumping index: ${index}`);

  const collections = await kuzzle.collection.list(index);
  console.log('Collections to dump: ', collections);

  await Promise.all(
    collections.collections.map(collection => {
      return dump_collection(index, collection.name, `${index}-${collection.name}-dump.json`, directoryName);
    })
  );
}

async function main(index, collection, directoryName) {
  try {
    await kuzzle.connect();
    // await kuzzle.auth.login('local', { username: 'admin-kuzzle-team', password: 'password'});

    if (collection !== undefined) {
      await dump_collection(index, collection, `${index}-${collection}-dump.json`, directoryName);
    } else {
      await dump_index(index, directoryName);
    }
  } catch (e) {
    console.log(e);
  }

  kuzzle.disconnect();
}

try {
  const index = process.argv[2];
  const collection = process.argv[3];

  if (!index) {
    console.log('Usage: node dump-kuzzle-index.js <indexName> [collectionName]');
    process.exit(1);
  }

  const directoryName = `./dump/${index}`;

  main(index, collection, directoryName);

  console.log(`Dump files are available in directory ${directoryName}/.`);
} catch (e) {
  console.log(e);
  process.exit(1);
}

import-bulk-data.js

const
  fs = require('fs'),
  {
    Kuzzle,
    WebSocket
  } = require('kuzzle-sdk');


const dumpDirectory = process.argv[2];
if (!dumpDirectory) {
  console.log('Usage: node import-bulk-data.js <dumpDirectory> [host] [port]');
  process.exit(1);
}

const kuzzle = new Kuzzle(
  new WebSocket('localhost', { port: 7512 })
);

kuzzle.on('networkError', error => {
  console.error(`Network Error: ${error.message}`);
});

(async () => {
  try {
    await kuzzle.connect();
    // await kuzzle.auth.login('local', { username: 'admin-kuzzle-team', password: 'password'});

    fs.readdirSync(dumpDirectory).forEach(async dumpFile => {
      console.log(`Importing dump file ${dumpFile}.`);
      const bulkData = JSON.parse(fs.readFileSync(`${dumpDirectory}/${dumpFile}`));
      await kuzzle.bulk.import(bulkData)
    });
  } catch (error) {
    console.log(error);
  } finally {
    kuzzle.disconnect();
  }
})();