-
-
Save kethinov/6658166 to your computer and use it in GitHub Desktop.
// List all files in a directory in Node.js recursively in a synchronous fashion | |
var walkSync = function(dir, filelist) { | |
var fs = fs || require('fs'), | |
files = fs.readdirSync(dir); | |
filelist = filelist || []; | |
files.forEach(function(file) { | |
if (fs.statSync(dir + file).isDirectory()) { | |
filelist = walkSync(dir + file + '/', filelist); | |
} | |
else { | |
filelist.push(file); | |
} | |
}); | |
return filelist; | |
}; |
FerreiraRaphael
commented
Mar 22, 2018
Isn't there a race condition between fs.statSync(...).isDirectory() and the recursion?
Maybe instead:
const fs = require('fs');
const path = require('path');
function walkSync (dir, filelist = []) {
fs.readdirSync(dir).forEach(file => {
const dirFile = path.join(dir, file);
try {
filelist = walkSync(dirFile, filelist);
}
catch (err) {
if (err.code === 'ENOTDIR' || err.code === 'EBUSY') filelist = [...filelist, dirFile];
else throw err;
}
});
return filelist;
}
Forked it to make it a generator function so that space complexity goes from O(n) to O(1) (link to fork with full implementation here).
/**
* List all files in a directory recursively in a synchronous fashion.
*
* @param {String} dir
* @returns {IterableIterator<String>}
*/
function *walkSync(dir) {
const files = fs.readdirSync(dir);
for (const file of files) {
const pathToFile = path.join(dir, file);
const isDirectory = fs.statSync(pathToFile).isDirectory();
if (isDirectory) {
yield *walkSync(pathToFile);
} else {
yield pathToFile;
}
}
}
Hi, here's a modern version:
const fs = require('fs').promises;
const path = require('path');
const walk = async (dir, filelist = []) => {
const files = await fs.readdir(dir);
for (file of files) {
const filepath = path.join(dir, file);
const stat = await fs.stat(filepath);
if (stat.isDirectory()) {
filelist = await walk(filepath, filelist);
} else {
filelist.push(file);
}
}
return filelist;
}
Read as hierarchy tree
const walkSync = (dir, filelist = []) => {
const files = fs.readdirSync(dir);
for (const file of files) {
const dirFile = path.join(dir, file);
const dirent = fs.statSync(dirFile);
if (dirent.isDirectory()) {
console.log('directory', path.join(dir, file));
var odir = {
file: dirFile,
files: []
}
odir.files = walkSync(dirFile, dir.files);
filelist.push(odir);
} else {
filelist.push({
file: dirFile
});
}
}
return filelist;
};
Hi,
this is an excerpt from a project of mine to make a database of all the MP3 I have, using recursive-readdir. It takes around 40 mins for 18641 files. I was wondering how to shorten this time using multiple processes to scan different parts of the file system's portion.
const promises = [];
promises.push(readdir(thePath, [function ignoreFiles(file, stats) {
if (stats.isDirectory()) {
return false;
}
return !isMp3(file);
}]));
Throwing another iterative (non recursive) one in here. It won't follow symlinks:
const fs = require('fs');
const path = require('path');
module.exports = (dir) => {
const result = [];
const files = [dir];
do {
const filepath = files.pop();
const stat = fs.lstatSync(filepath);
if (stat.isDirectory()) {
fs
.readdirSync(filepath)
.forEach(f => files.push(path.join(filepath, f)));
} else if (stat.isFile()) {
result.push(path.relative(dir, filepath));
}
} while (files.length !== 0);
return result;
};
Shameless self promotion. I now use smart-fs
an FP approach
const { join } = require('path');
const { readdirSync, statSync } = require('fs');
const {
concat, difference, filter, map, reduce,
} = require('lodash/fp');
const getFilePaths = (path, encoding) => {
const entries = readdirSync(path, { encoding });
const paths = map(entry => join(path, entry), entries);
const filePaths = filter(entryPath => statSync(entryPath).isFile(), paths);
const dirPaths = difference(paths, filePaths);
const dirFiles = reduce((prev, curr) => concat(prev, getFilePaths(curr)), [], dirPaths);
return [...filePaths, ...dirFiles];
};
module.exports = getFilePaths;
Async, EJS6, and showing full path.
As few packages as possible.
Tested in Node 10.16.0
const fs = require('fs').promises
const path = require('path')
async function walk(dir, fileList = []) {
const files = await fs.readdir(dir)
for (const file of files) {
const stat = await fs.stat(path.join(dir, file))
if (stat.isDirectory()) fileList = await walk(path.join(dir, file), fileList)
else fileList.push(path.join(dir, file))
}
return fileList
}
walk('/home/').then((res) => {
console.log(res)
})
@SamMaxwell approach without using lodash
const fs = require('fs');
const path = require('path');
/** Retrieve file paths from a given folder and its subfolders. */
const getFilePaths = (folderPath) => {
const entryPaths = fs.readdirSync(folderPath).map(entry => path.join(folderPath, entry));
const filePaths = entryPaths.filter(entryPath => fs.statSync(entryPath).isFile());
const dirPaths = entryPaths.filter(entryPath => !filePaths.includes(entryPath));
const dirFiles = dirPaths.reduce((prev, curr) => prev.concat(getFilePaths(curr)), []);
return [...filePaths, ...dirFiles];
};
module.exports = getFilePaths;
rxjs 6.5, Typescript 3.5, node 10.6, modifying @quantumsheep's solution above, including both files and directories in a single pass, streaming results into an Observable of a custom type:
import { Stats, promises as fs } from 'fs';
import { Observable } from 'rxjs';
const dirsAndFiles = (dir: string): Observable<FileObject> =>
new Observable(
(observer): void => {
// Create an async recursive function for walking the directory tree:
const walk = async (dir: string): Promise<void> => {
const files = await fs.readdir(join(__dirname, dir));
for (const file of files) {
const filepath = join(dir, file);
const stat: Stats = await fs.stat(join(__dirname, filepath));
if (stat.isFile() || stat.isDirectory()) {
observer.next({
isFile: stat.isFile(),
filename: file,
size: stat.size,
path: dir
});
}
if (stat.isDirectory()) {
await walk(filepath);
}
}
};
// now call that async function and complete the Observable when it resolves
walk(dir)
.then((): void => observer.complete())
.catch((err): void => observer.error(err));
}
);
// later in code:
dirsAndFiles(dir).subscribe(
(file: FileObject) => doSomething(file),
(err): void => processError(err)
);
All fine, but how do you create an object than simulates the structure of the actual directory structure?
Like, from this:
Fig 1
LEVEL_1
LEVEL_2
| LEVEL_3_1
| | FILE_3_1_1
| | FILE_3_1_2
| LEVEL_3_2
| | FILE_3_2_1
| | FILE_3_2_2
| | LEVEL_4
| | | FILE_4_1
| | | FILE_4_2
| | | ... this could go on forever ...
| FILE_2_1
| FILE_2_2
FILE_1_1
FILE_1_2
to this:
Fig 2
{
LEVEL_2 : {
LEVEL_3_1 : {
FILE_3_1_1 : "FILE CONTENT",
FILE_3_1_2 : "FILE CONTENT"
},
LEVEL_3_2 : {
FILE_3_2_1 : "FILE CONTENT",
FILE_3_2_2 : "FILE CONTENT"
LEVEL_4 : {
FILE_4_1 : "FILE CONTENT",
FILE_4_2 : "FILE CONTENT"
}
},
FILE_1_1 : "FILE CONTENT",
FILE_2_1 : "FILE CONTENT"
}
}
good question, same here... anyone with ES5 old school simple solution?
ouuu hell yea... I love google and I love open source comunity... works just like that:
https://www.npmjs.com/package/directory-tree
const dirTree = require("directory-tree");
const tree = dirTree("/Users/admin/Music");
console.log(tree)
thats all... the rest is up to Angular-tree-control xDDD good luck
*edit - solution #2 https://www.npmjs.com/package/dree NOT tested, just another option
These return the structure alright in some 'weird' way, with all sorts of properties. My own solution to the problem with help from guys over at stackoverflow is more the kind I was looking for: https://stackoverflow.com/a/57626051/8082874
It return a straight object with folders as objects, filenames as keys – and in this case file contents as precompiled Handlebars templates ... like this:
{
admin: {
admin_subviews: {
index: [Function]
},
header: {
[Function: ret] isTop: true,
_setup: [Function],
_child: [Function]
},
index: {
[Function: ret] isTop: true,
_setup: [Function],
_child: [Function]
},
layout: {
[Function: ret] isTop: true,
_setup: [Function],
_child: [Function]
}
},
index:{
[Function: ret] isTop: true,
_setup: [Function],
_child: [Function]
},
layout: {
[Function: ret] isTop: true,
_setup: [Function],
_child: [Function]
},
login: {
index: {
[Function: ret] isTop: true,
_setup: [Function],
_child: [Function]
},
layout: {
[Function: ret] isTop: true,
_setup: [Function],
_child: [Function]
}
}
}
The root level is the object itself and you can manipulate the file content in any way you want and inject it as the value.
I have no idea what you mean but for me it just rendered JSON on server of all files/folders in tree -> send that tree to front-end-> and https://github.com/wix/angular-tree-control just directly render all of that at a glance 👍 but you look like that you need more than that...
Probably yes. And whenever something contains 'angular' I'm usually out anyway, so ; )
yea xD I'm locked in the 2015 year development and ES5 but I'm so grateful for that but depends on everybody's needs ;) good luck anyway
Here's an ES6 refactor. One that applies RegExp filtering:
const fs = require('fs');
const path = require('path');
function findInDir (dir, filter, fileList = []) {
const files = fs.readdirSync(dir);
files.forEach((file) => {
const filePath = path.join(dir, file);
const fileStat = fs.lstatSync(filePath);
if (fileStat.isDirectory()) {
findInDir(filePath, filter, fileList);
} else if (filter.test(filePath)) {
fileList.push(filePath);
}
});
return fileList;
}
// Usage
findInDir('./public/audio/', /\.mp3$/);
@meowsus Your solution works great, thanks.
Just another implementation
const path = require('path');
const fs = require('fs');
/* Prepend the given path segment */
const prependPathSegment = pathSegment => location => path.join(pathSegment, location);
/* fs.readdir but with relative paths */
const readdirPreserveRelativePath = location => fs.readdirSync(location).map(prependPathSegment(location));
/* Recursive fs.readdir but with relative paths */
const readdirRecursive = location => readdirPreserveRelativePath(location)
.reduce((result, currentValue) => fs.statSync(currentValue).isDirectory()
? result.concat(readdirRecursive(currentValue))
: result.concat(currentValue), []);
I think the most elegant way to list all files and/or directories recursively is using globs. You can use globby.
const globby = require('globby');
const listAllFilesAndDirs = dir => globby(`${dir}/**/*`);
(async () => {
const result = await listAllFilesAndDirs(process.cwd());
console.log(result);
})();
@gemyero Great solution, thanks!
mmh Carefull with globby ;)
Here is my way (with typescript):
import fs from "fs/promises";
import path from "path";
export default async function walk(directory: string) {
let fileList: string[] = [];
const files = await fs.readdir(directory);
for (const file of files) {
const p = path.join(directory, file);
if ((await fs.stat(p)).isDirectory()) {
fileList = [...fileList, ...(await walk(p))];
} else {
fileList.push(p);
}
}
return fileList;
}
Alternative solution in Just two lines, Just for fun and learning
const path = ''; // 👈 path to your location
const list = require("child_process").execSync(`cd ${path} && ls -R`).toString().split(`\n`);
Be careful, this solution use a bash script and it works fine in MACOS, for windows or linux could be different
What I use is based on @vidul-nikolaev-petrov version, with a callback argument to do something with files:
const walkSync = (dir, callback) => fs.lstatSync(dir).isDirectory()
? fs.readdirSync(dir).map(f => walkSync(path.join(dir, f), callback))
: callback(dir);
// Note: call to flat at the end to have one array with every paths
const svgs = walkSync(path, fileToBase64).flat()
Alternative solution in Just two lines, Just for fun and learning
const path = ''; // 👈 path to your location const list = require("child_process").execSync(`cd ${path} && ls -R`).toString().split(`\n`);
⚠️ IMPORTANT
Be careful, this solution use a bash script and it works fine in MACOS, for windows or linux could be different
Please, never do that
@FerreiraRaphael kudos
Hi, here's a modern version:
const fs = require('fs').promises; const path = require('path'); const walk = async (dir, filelist = []) => { const files = await fs.readdir(dir); for (file of files) { const filepath = path.join(dir, file); const stat = await fs.stat(filepath); if (stat.isDirectory()) { filelist = await walk(filepath, filelist); } else { filelist.push(file); } } return filelist; }
Here's what I used (needed import
)
import * as fs from 'fs/promises';
import path from 'path';
export async function directory_read(dir, filelist = []) {
const files = await fs.readdir(dir);
for (let file of files) {
const filepath = path.join(dir, file);
const stat = await fs.stat(filepath);
if (stat.isDirectory()) {
filelist = await directory_read(filepath, filelist);
} else {
filelist.push(file);
}
}
return filelist;
}
Node v20.1+
const filelist = await fs.readdir(dir, { recursive: true });