Skip to content

Instantly share code, notes, and snippets.

@katowulf

katowulf/pseudo_code.js

Last active Dec 4, 2020
Embed
What would you like to do?
Convert Cloud Storage json file to event stream and store output in Firestore
JSONStream = require('JSONStream');
es = require('event-stream');
fileStream = storage.bucket('your-bucket').file('your-JSON-file').createReadStream();
db = admin.firestore();
return new Promise( (resolve, reject) => {
batchPromises = [];
batchSize = 0;
batch = db.batch();
fileStream.pipe(JSONStream.parse(...)
.pipe(es.map(function (data) {
// transform the data here
})
.pipe(menuItem => {
ref = <get a ref>;
batch.set( ref, menuItem );
batchSize++;
if( batchSize === 500 ) {
batchPromises.push( batch.commit() );
batch = db.batch();
batchSize = 0;
}
});
.on("end", () => {
if( batchSize > 0 ) {
batchPromises.push( batch.commit() );
}
Promise.all(batchPromises).then(resolve).catch(reject)
});
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment