Convert Cloud Storage json file to event stream and store output in Firestore
JSONStream = require('JSONStream'); | |
es = require('event-stream'); | |
fileStream = storage.bucket('your-bucket').file('your-JSON-file').createReadStream(); | |
db = admin.firestore(); | |
return new Promise( (resolve, reject) => { | |
batchPromises = []; | |
batchSize = 0; | |
batch = db.batch(); | |
fileStream.pipe(JSONStream.parse(...) | |
.pipe(es.map(function (data) { | |
// transform the data here | |
}) | |
.pipe(menuItem => { | |
ref = <get a ref>; | |
batch.set( ref, menuItem ); | |
batchSize++; | |
if( batchSize === 500 ) { | |
batchPromises.push( batch.commit() ); | |
batch = db.batch(); | |
batchSize = 0; | |
} | |
}); | |
.on("end", () => { | |
if( batchSize > 0 ) { | |
batchPromises.push( batch.commit() ); | |
} | |
Promise.all(batchPromises).then(resolve).catch(reject) | |
}); | |
}); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment