High Performance Batched updates in Firestore

README.md

Batched updates in Firestore

This short script shows how to do a batched update of a very large data set in firestore.

The scenario is I have a large set of products, and a 'pool' of product codes that need to be assigned to them. This script loads the list of UPC codes from a firebase database, then updates a firestore collection, updating every doc with a UPC code. For each product, the next code docID is assigned (so it's reall a pointer to the code, not the code itself), then the code is updated to signify that is has been assigned.

The script pages through the product database 100 docs at a time, and assignes the update to a Promise array, then launches the array before recursing.

Keywords: recursion, batch updates, firebase, firestore

batch-update.js

const admin = require('firebase-admin');
const serviceAccount = require('./firebase-service-account.json');

admin.initializeApp({
  credential: admin.credential.cert(serviceAccount),
  databaseURL: 'https://demo.firebaseio.com',
});

const fsdb = admin.firestore();
const rtdb = admin.database();


let srcRef = rtdb.ref('upc-codes'); // a simple k/v store of UPC codes
let dstRef = fsdb.collection('products'); // a collection of products that need a UPC code asigned

(async function main() {
  const upcSnap = await srcRef.once('value'); //grab all the UPC doc IDs
  let upcIDs = []; //then stick them in an arry for easy access/iteration
  upcSnap.forEach(id => {
    upcIDs.push(id.key);
  });
  
  let nextStart = 0; // track what page we're on
  const PAGE_SIZE = 100; //fetch 100 products at a time
  let docCounter = 0; // track what index we need out of upc array
  const recurse = nextStart => {
    dstRef
      .orderBy('id')
      .startAfter(nextStart)
      .limit(PAGE_SIZE)
      .get()
      .then(nextSnap => {
        const promAry = []; //an array to hold the batch update promises
        nextSnap.docs.forEach(docSnap => { // loop through page of products
          let id = upcIDs[docCounter]; // grab a UPC doc id
          
          // assign upc to product
          let prom = docSnap.ref.update({ upc: id }).then(() => { 
            // then set the upc ID value to FALSE, signifying it is assigned (for future products assignments)
            srcRef.child(id).set(false);
          });
          promAry.push(prom); // add batch update to array of promises
          docCounter += 1; //increment doc counter
          console.log(docCounter);
        }); // finished looping through this page
        Promise.all(promAry).then(res => { //fire off all the updates
          if (nextSnap.docs.length === PAGE_SIZE) { // if the docs.lenth < PAGE_SIZE, we're on the last page!
            nextStart = nextSnap.docs[nextSnap.docs.length - 1].data().id; // set the next page start
            setTimeout(() => {
              recurse(nextStart); //do it all again
            }, 10); // do in a set timeout so as to not run out of stack space if the promise array is too big
          } else {
            return; // were done!
          }
        });
      })
      .catch(err => {
        console.log('error outer!', err);
      });
  };
  recurse(0); // kick it all off
})();