Last active
July 21, 2017 03:29
-
-
Save fasiha/7a2b64473ad5ba4d7c435b2ce78e2e46 to your computer and use it in GitHub Desktop.
Chunked reduce for PouchDB’s allDocs: instead of storing the entire output of allDocs in memory and then calling reduce (fold), repeatedly run the reduction on small chunks of allDocs. This should reduce memory requirements at a (slight) CPU cost.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/* For this example, npm install pouchdb pouchdb-adapter-node-websql sqlite3 */ | |
import PouchDB = require('pouchdb'); | |
PouchDB.plugin(require('pouchdb-adapter-node-websql')); | |
export type Db = PouchDB.Database<{}>; | |
export let db: Db = new PouchDB('mypouchdb.db', { adapter: 'websql' }); | |
async function allDocsReduce(db: Db, opts, func, init, limit: number = 25, verbose: boolean = false) { | |
opts = Object.assign({}, opts); // copy | |
opts.limit = limit; | |
while (1) { | |
let res = await db.allDocs(opts); | |
if (res.rows.length === 0) { | |
return init; | |
} | |
init = res.rows.reduce(func, init); | |
opts.startkey = res.rows[res.rows.length - 1].key; | |
opts.skip = 1; | |
if (verbose) { | |
console.log(init); | |
} | |
} | |
} | |
async function init() { | |
await db.bulkDocs(Array.from(Array(20), (_, n) => ({ _id: '' + n, n: Math.random() }))); | |
console.log((await db.allDocs({ include_docs: true })).rows.map(o => o.doc)) | |
} | |
// init() | |
async function test() { | |
// console.log((await db.allDocs({include_docs:true})).rows.map(o=>o.doc)); | |
console.log('max', await allDocsReduce(db, { include_docs: true }, (prev, next) => Math.max(next.doc.n, prev), 0)) | |
console.log('max', await allDocsReduce(db, { include_docs: true }, (prev, next) => Math.max(next.doc.n, prev), 0, 1)) | |
console.log('max', await allDocsReduce(db, { include_docs: true }, (prev, next) => Math.max(next.doc.n, prev), 0, 10)) | |
console.log('min', await allDocsReduce(db, { include_docs: true }, (prev, next) => Math.min(next.doc.n, prev), Infinity, 2)) | |
console.log('min', await allDocsReduce(db, { include_docs: true, descending: true }, (prev, next) => Math.min(next.doc.n, prev), Infinity, 2, true)) | |
} | |
test() | |
/* Outputs: | |
max 0.9801551321128437 | |
max 0.9801551321128437 | |
max 0.9801551321128437 | |
min 0.023215878756427166 | |
0.09281023315135406 | |
0.07004790542727712 | |
0.07004790542727712 | |
0.07004790542727712 | |
0.07004790542727712 | |
0.07004790542727712 | |
0.07004790542727712 | |
0.07004790542727712 | |
0.023215878756427166 | |
0.023215878756427166 | |
min 0.023215878756427166 | |
*/ |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment