Last active
February 10, 2019 04:04
-
-
Save BananaAcid/327c701b5147459a6a3b93cc240f7618 to your computer and use it in GitHub Desktop.
super fast file hashing, as ESM, all awaitable (parallel, no thread blocking)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// npm install murmurhash-native fs-extra -P | |
import path from 'path'; | |
/* experimental (node v10, v11): */ | |
import fsOrig from 'fs'; const fs = fsOrig.promises; | |
// import fs from 'fs-extra'; // awaitable fs for node < v10 | |
import murmur from 'murmurhash-native/stream'; // super fast HASHING ! https://github.com/royaltm/node-murmurhash-native | |
const sPath = process.pwd(); | |
(async () => { | |
console.log(`getting hash for all files in ${sPath}:`); | |
// asynchronously finishing, no waiting [fire and forget] | |
//(await fs.readdir(sPath)).forEach( async file => { | |
// async start, but collect the promises (to wait for) | |
let files = await fs.readdir(sPath); | |
let tasks = files.map( async file => { | |
// filter `.` files (hidden) | |
if (file[0] === '.') return; | |
// reconstruct the full file path | |
let fullPath = path.resolve(sPath, file); | |
// prepare a new hasher | |
let hash = murmur.createHash('murmurhash32', {seed: 0, encoding: 'hex', endianness: 'platform'}); | |
// pipe through the hasher (little memory foot print) | |
let val = await new Promise( resolve => { | |
let data=''; | |
fs.createReadStream(fullPath) | |
.pipe(hash) // create hash string | |
.on('data', d => data += d.toString()) // collect stream text from pipe->hash | |
.on('error', () => resolve(null)) // do not throw an error on error (so no reject) | |
.on('end', () => resolve(data)); // return info | |
}); | |
console.log('FILE: %s -> %s == %s', sPath, file, val); | |
}); | |
// wait for all parallel tasks to have finished (checks their resolved state) | |
await Promise.all(tasks); | |
console.log('done.'); | |
})().catch(console.error); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
since
let p = new Promise()
,let p = async function () {}
, andlet p = async () => {}
-> all return an ES6 Promise,.catch()
can be used to collect any errors, triggered by any code within.the
let hash
andlet val
lines are the one with the important parts.