Created
October 16, 2021 10:18
-
-
Save Stwissel/00d1a065096ab342a7509cfa1e97ba39 to your computer and use it in GitHub Desktop.
Streaming couchDB using NodeJS stream API and nano
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const Nano = require("nano"); | |
const { Writable, Transform } = require("stream"); | |
const exportOneDb = (couchDBURL, resultCallback) => { | |
const nano = Nano(couchDBURL); | |
nano | |
.listAsStream({ include_docs: true }) | |
.on("error", (e) => console.error("error", e)) | |
.pipe(lineSplitter()) | |
.pipe(jsonMaker()) | |
.pipe(documentWriter(resultCallback)); | |
}; | |
const lineSplitter = () => | |
new Transform({ | |
objectMode: true, | |
transform(chunk, encoding, callback) { | |
let raw = Buffer.from(chunk, encoding).toString(); | |
if (this._leftOver) { | |
raw = this._leftOver + raw; | |
} | |
let lines = raw.split("\n"); | |
this._leftOver = lines.splice(lines.length - 1, 1)[0]; | |
for (var i in lines) { | |
this.push(lines[i]); | |
} | |
callback(); | |
}, | |
flush(callback) { | |
if (this._leftOver) { | |
this.push(this._leftOver); | |
} | |
this._leftOver = null; | |
callback(); | |
}, | |
}); | |
const jsonMaker = () => | |
new Transform({ | |
objectMode: true, | |
transform(rawLine, encoding, callback) { | |
// remove the comma at the end of the line - CouchDB sent an array | |
let line = rawLine.toString().replace(/,$/m, "").trim(); | |
if (line.startsWith('{"id":') && line.endsWith("}")) { | |
try { | |
let j = JSON.parse(line); | |
// We only want the document | |
if (j.doc) { | |
this.push(JSON.stringify(j.doc)); | |
} | |
} catch (e) { | |
console.error(e.message); | |
} | |
} | |
callback(); | |
}, | |
}); | |
const documentWriter = (resultCallback) => | |
new Writable({ | |
write(chunk, encoding, callback) { | |
let json = JSON.parse(Buffer.from(chunk, encoding).toString()); | |
// Process the code | |
resultCallback(json); | |
// Tell that we are done | |
callback(); | |
}, | |
}); | |
module.exports = { | |
streamOneDb | |
}; |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
this._leftOver
is not compatible with TSmight be an issue with TS not understanding the underlying context
but there is no resolution at the current moment for TypeScript implementation
I also looked into stream-json as an alternative but unsure what method would be ideal for nano since its not an array of documents