Copy Google Cloud Datastore entities to a new project and/or namespace

copyDatastoreEntities.js

/*
Copy Google Cloud Datastore entities to a new project and/or namespace.

This script compares the contents of the new datastore against the original
datastore to verify the copy was successful.

inspired by https://gist.github.com/tarunbhardwaj/f17b2451869a1ee588679c8464b40755
*/

const { Datastore } = require('@google-cloud/datastore');

/* == User config == */
const Kinds = ['Claims'];

// leave as blank string for [default] namespace
const origNamespace = '';
const newNamespace = '';

const origProjectId = 'your-project-id';
// leave as blank string to keep same project id
const newProjectId = '';

/* == Script code- don't change == */

const OrigDatabase = new Datastore({
  projectId: origProjectId,
});

const NewDatabase = new Datastore({
  projectId: newProjectId || origProjectId,
});

function getNewKey(kind, origKey) {
  const path = [kind, origKey.name];
  const key = NewDatabase.key({
    namespace: newNamespace,
    path,
  });
  return key;
}

async function getDatastoreContents(datastore, namespace, kind) {
  return datastore
    .createQuery(namespace, kind)
    .order('createdAt', { descending: true })
    .run()
    .then((data) => {
      let entities = data[0];
      return entities;
    });
}

async function insertEntities(entities, kind) {
  return Promise.resolve(entities)
    .then(entities => entities.map((entity) => {
      const origKey = entity[OrigDatabase.KEY];
      const newKey = getNewKey(kind, origKey);

      const row = {
        key: newKey,
        data: entity,
      }
      return row;
    }))
    .then(async (rows) => {
      for (const row of rows) {
        await NewDatabase.insert(row)
          .catch(err => console.error(err));
      }
    });
}

function sortObjects(objects, keys) {
  return objects.map((object) => {
    const sortedObject = {};
    for (const key of keys) {
      if (object[key] !== undefined) {
        sortedObject[key] = object[key];
      }
    }
    return sortedObject;
  });
}

function compareJson(origJson, newJson) {
  const origObjects = JSON.parse(origJson);
  const newObjects = JSON.parse(newJson);

  const origKeys = new Set();
  const newKeys = new Set();

  for (const object of origObjects) {
    Object.keys(object).map(key => origKeys.add(key));
  }
  for (const object of newObjects) {
    Object.keys(object).map(key => newKeys.add(key));
  }

  if (origKeys.size !== newKeys.size) {
    return false;
  }

  for (const key of origKeys) {
    if (!newKeys.has(key)) {
      return false;
    }
  }

  const sortedKeys = Array.from(origKeys).sort();
  const sortedOrigObjects = sortObjects(origObjects, sortedKeys);
  const sortedNewObjects = sortObjects(newObjects, sortedKeys);

  if (JSON.stringify(sortedOrigObjects) !== JSON.stringify(sortedNewObjects)) {
    return false;
  }

  return true;
}

async function copyEntities() {
  for (const Kind of Kinds) {
    const entities = await getDatastoreContents(OrigDatabase, origNamespace, Kind);
    const origDbContents = JSON.stringify(entities);

    console.log(`Copying ${entities.length} entities with Kind "${Kind}"`);
    await insertEntities(entities, Kind);

    const newDbContents = await getDatastoreContents(NewDatabase, newNamespace, Kind)
      .then(contents => JSON.stringify(contents));

    if (compareJson(origDbContents, newDbContents)) {
      console.log('Copy was successful!');
    }
    else {
      console.log('Copy was unsuccesssful.');
    }
  }
}

copyEntities();