Last active
December 5, 2023 19:35
-
-
Save nestarz/1fa7ae93fb83f1eafb1b88c3a84f2e02 to your computer and use it in GitHub Desktop.
NDJSON File Streaming using browser Stream API + Loading Progress
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const parseJSON = () => | |
new TransformStream({ | |
transform(chunk, controller) { | |
controller.enqueue(JSON.parse(chunk)); | |
} | |
}); | |
const splitStream = splitOn => { | |
let buffer = ""; | |
return new TransformStream({ | |
transform(chunk, controller) { | |
buffer += chunk; | |
const parts = buffer.split(splitOn); | |
parts.slice(0, -1).forEach(part => controller.enqueue(part)); | |
buffer = parts[parts.length - 1]; | |
}, | |
flush(controller) { | |
if (buffer) controller.enqueue(buffer); | |
} | |
}); | |
}; | |
const fetchJSONLD = url => | |
fetch(url).then(response => ({ | |
response, | |
reader: response.body | |
.pipeThrough(new TextDecoderStream()) | |
// Needed to stream by line and then JSON parse the line | |
.pipeThrough(splitStream("\n")) | |
.pipeThrough(parseJSON()) | |
.getReader() | |
})); | |
export { fetchJSONLD }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(async () => { | |
const { fetchNDJSON } = await import("/fetchNDJSON.js"); | |
const results = []; | |
let progress = 0; | |
fetchJSONLD(file).then(({ response, reader }) => { | |
const length = response.headers.get("Content-Length"); | |
let received = 0; | |
const onReadChunk = async chunk => { | |
if (chunk.done) { | |
return; | |
} | |
// await timeout(1000); | |
received += chunk.value.length; | |
results.push(chunk.value); | |
progress = received / length; | |
reader.read().then(onReadChunk); | |
}; | |
reader.read().then(onReadChunk); | |
}); | |
})(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Here is a version using TextDecoder only :