Skip to content

Instantly share code, notes, and snippets.

Embed
What would you like to do?
const fs = require('fs')
const gs = require('glob-stream')
const File = require('vinyl')
const { Transform } = require('stream')
const map = (transform) => new Transform({ objectMode: true, transform })
const ospath = require('path')
//const isUTF8 = require('is-utf8')
const { pipeline } = require('stream')
function smartStat (path_, callback) {
fs.lstat(path_, (lstatErr, lstat) => {
if (lstatErr) {
callback(lstatErr) // NOTE extremely rare
} else if (lstat.isSymbolicLink()) {
fs.stat(path_, (statErr, stat) => {
//statErr ? callback(Object.assign(statErr, { link: true })) : callback(null, stat)
if (statErr) {
fs.readlink(path_, (_, link) => {
callback(Object.assign(statErr, { link }))
})
} else {
callback(null, stat)
}
})
} else {
callback(null, lstat)
}
})
}
;(async () => {
await new Promise((resolve, reject) => {
console.time('measure')
const accum = []
const cache = {}
const cwd = ospath.resolve('../docs')
pipeline(
gs('**/*', {
cwd,
//cwd: ospath.resolve('more-docs'),
follow: true,
nomount: true,
nosort: true,
uniqueBy: ((entries) => entries),
strict: false, // silently ignore files that cannot be read
cache, // we could reuse cache, but it's very low level and not likely to provide all the info we need
//nodir: true, // don't return directories, though it seems to come with a slight cost; but why?
}),
map(({ path: abspath }, _, next) => {
//to skip directories eagerly, map cache and enable this line
//if (Array.isArray(cache[abspath])) return next()
smartStat(abspath, (statErr, stat) => {
if (statErr) {
console.warn('skipping file: ' + abspath + (statErr.link ? ` (${statErr.code === 'ELOOP' ? 'symlink cycle' : 'broken symlink'}: ${statErr.link})` : ''))
return next()
}
if (stat.isDirectory()) return next()
fs.readFile(abspath, (readErr, contents) => {
if (readErr) {
console.warn('skipping file: ' + abspath)
return next()
}
//if (contents[0] === 0xEF && contents[1] === 0xBB && contents[2] === 0xBF && isUTF8(contents)) {
// contents = contents.slice(3)
//}
accum.push(new File({ base: cwd, cwd, path: abspath, contents, stat }))
next()
})
})
}),
(err) => err ? reject(err) : resolve(accum)
)
})
.then((files) => {
console.timeEnd('measure')
console.log(files.length)
console.log(files[1].relative)
//files.forEach((file) => console.log(file.relative))
})
.catch((err) => {
console.warn('error: ' + err.message)
})
})()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment