Skip to content

Instantly share code, notes, and snippets.

@fasiha
Last active July 21, 2017 03:29
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save fasiha/7a2b64473ad5ba4d7c435b2ce78e2e46 to your computer and use it in GitHub Desktop.
Save fasiha/7a2b64473ad5ba4d7c435b2ce78e2e46 to your computer and use it in GitHub Desktop.
Chunked reduce for PouchDB’s allDocs: instead of storing the entire output of allDocs in memory and then calling reduce (fold), repeatedly run the reduction on small chunks of allDocs. This should reduce memory requirements at a (slight) CPU cost.
/* For this example, npm install pouchdb pouchdb-adapter-node-websql sqlite3 */
import PouchDB = require('pouchdb');
PouchDB.plugin(require('pouchdb-adapter-node-websql'));
export type Db = PouchDB.Database<{}>;
export let db: Db = new PouchDB('mypouchdb.db', { adapter: 'websql' });
async function allDocsReduce(db: Db, opts, func, init, limit: number = 25, verbose: boolean = false) {
opts = Object.assign({}, opts); // copy
opts.limit = limit;
while (1) {
let res = await db.allDocs(opts);
if (res.rows.length === 0) {
return init;
}
init = res.rows.reduce(func, init);
opts.startkey = res.rows[res.rows.length - 1].key;
opts.skip = 1;
if (verbose) {
console.log(init);
}
}
}
async function init() {
await db.bulkDocs(Array.from(Array(20), (_, n) => ({ _id: '' + n, n: Math.random() })));
console.log((await db.allDocs({ include_docs: true })).rows.map(o => o.doc))
}
// init()
async function test() {
// console.log((await db.allDocs({include_docs:true})).rows.map(o=>o.doc));
console.log('max', await allDocsReduce(db, { include_docs: true }, (prev, next) => Math.max(next.doc.n, prev), 0))
console.log('max', await allDocsReduce(db, { include_docs: true }, (prev, next) => Math.max(next.doc.n, prev), 0, 1))
console.log('max', await allDocsReduce(db, { include_docs: true }, (prev, next) => Math.max(next.doc.n, prev), 0, 10))
console.log('min', await allDocsReduce(db, { include_docs: true }, (prev, next) => Math.min(next.doc.n, prev), Infinity, 2))
console.log('min', await allDocsReduce(db, { include_docs: true, descending: true }, (prev, next) => Math.min(next.doc.n, prev), Infinity, 2, true))
}
test()
/* Outputs:
max 0.9801551321128437
max 0.9801551321128437
max 0.9801551321128437
min 0.023215878756427166
0.09281023315135406
0.07004790542727712
0.07004790542727712
0.07004790542727712
0.07004790542727712
0.07004790542727712
0.07004790542727712
0.07004790542727712
0.023215878756427166
0.023215878756427166
min 0.023215878756427166
*/
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment