Skip to content

Instantly share code, notes, and snippets.

@kethinov
Created September 22, 2013 09:04
Show Gist options
  • Save kethinov/6658166 to your computer and use it in GitHub Desktop.
Save kethinov/6658166 to your computer and use it in GitHub Desktop.
List all files in a directory in Node.js recursively in a synchronous fashion
// List all files in a directory in Node.js recursively in a synchronous fashion
var walkSync = function(dir, filelist) {
var fs = fs || require('fs'),
files = fs.readdirSync(dir);
filelist = filelist || [];
files.forEach(function(file) {
if (fs.statSync(dir + file).isDirectory()) {
filelist = walkSync(dir + file + '/', filelist);
}
else {
filelist.push(file);
}
});
return filelist;
};
@noodlecollie
Copy link

Given the above script already makes use of path, wouldn't it be better to use path.extname(lib) !== ".js", instead of lib.indexOf(".js") === -1? The latter would be fooled by not-a-lib.js.txt.

@FerreiraRaphael
Copy link

/**
 * Find all files inside a dir, recursively.
 * @function getAllFiles
 * @param  {string} dir Dir path string.
 * @return {string[]} Array with all file names that are inside the directory.
 */
const getAllFiles = dir =>
  fs.readdirSync(dir).reduce((files, file) => {
    const name = path.join(dir, file);
    const isDirectory = fs.statSync(name).isDirectory();
    return isDirectory ? [...files, ...getAllFiles(name)] : [...files, name];
  }, []);

@Skhmt
Copy link

Skhmt commented May 21, 2018

Isn't there a race condition between fs.statSync(...).isDirectory() and the recursion?

Maybe instead:

const fs = require('fs');
const path = require('path');

function walkSync (dir, filelist = []) {
    fs.readdirSync(dir).forEach(file => {
        const dirFile = path.join(dir, file);
        try {
            filelist = walkSync(dirFile, filelist);
        }
        catch (err) {
            if (err.code === 'ENOTDIR' || err.code === 'EBUSY') filelist = [...filelist, dirFile];
            else throw err;
        }
    });
    return filelist;
}

@luciopaiva
Copy link

luciopaiva commented Aug 29, 2018

Forked it to make it a generator function so that space complexity goes from O(n) to O(1) (link to fork with full implementation here).

/**
 * List all files in a directory recursively in a synchronous fashion.
 *
 * @param {String} dir
 * @returns {IterableIterator<String>}
 */
function *walkSync(dir) {
    const files = fs.readdirSync(dir);

    for (const file of files) {
        const pathToFile = path.join(dir, file);
        const isDirectory = fs.statSync(pathToFile).isDirectory();
        if (isDirectory) {
            yield *walkSync(pathToFile);
        } else {
            yield pathToFile;
        }
    }
}

@quantumsheep
Copy link

Hi, here's a modern version:

const fs = require('fs').promises;
const path = require('path');

const walk = async (dir, filelist = []) => {
  const files = await fs.readdir(dir);

  for (file of files) {
    const filepath = path.join(dir, file);
    const stat = await fs.stat(filepath);

    if (stat.isDirectory()) {
      filelist = await walk(filepath, filelist);
    } else {
      filelist.push(file);
    }
  }

  return filelist;
}

@darekmeco
Copy link

Read as hierarchy tree

    const walkSync = (dir, filelist = []) => {
      const files = fs.readdirSync(dir);
      for (const file of files) {
        const dirFile = path.join(dir, file);
        const dirent = fs.statSync(dirFile);
        if (dirent.isDirectory()) {
          console.log('directory', path.join(dir, file));
          var odir = {
            file: dirFile,
            files: []
          }
          odir.files = walkSync(dirFile, dir.files);
          filelist.push(odir);
        } else {
          filelist.push({
            file: dirFile
          });
        }
      }
      return filelist;
    };

@chrisvoo
Copy link

Hi,
this is an excerpt from a project of mine to make a database of all the MP3 I have, using recursive-readdir. It takes around 40 mins for 18641 files. I was wondering how to shorten this time using multiple processes to scan different parts of the file system's portion.

        const promises = [];
        promises.push(readdir(thePath, [function ignoreFiles(file, stats) {
            if (stats.isDirectory()) {
                return false;
            }

            return !isMp3(file);
        }]));

@simlu
Copy link

simlu commented Mar 17, 2019

Throwing another iterative (non recursive) one in here. It won't follow symlinks:

const fs = require('fs');
const path = require('path');

module.exports = (dir) => {
  const result = [];

  const files = [dir];
  do {
    const filepath = files.pop();
    const stat = fs.lstatSync(filepath);
    if (stat.isDirectory()) {
      fs
        .readdirSync(filepath)
        .forEach(f => files.push(path.join(filepath, f)));
    } else if (stat.isFile()) {
      result.push(path.relative(dir, filepath));
    }
  } while (files.length !== 0);

  return result;
};

Shameless self promotion. I now use smart-fs

@SamMaxwell
Copy link

SamMaxwell commented May 6, 2019

an FP approach

const { join } = require('path');
const { readdirSync, statSync } = require('fs');
const {
  concat, difference, filter, map, reduce,
} = require('lodash/fp');

const getFilePaths = (path, encoding) => {
  const entries = readdirSync(path, { encoding });
  const paths = map(entry => join(path, entry), entries);
  const filePaths = filter(entryPath => statSync(entryPath).isFile(), paths);
  const dirPaths = difference(paths, filePaths);
  const dirFiles = reduce((prev, curr) => concat(prev, getFilePaths(curr)), [], dirPaths);
  return [...filePaths, ...dirFiles];
};

module.exports = getFilePaths;

@SirMorfield
Copy link

SirMorfield commented Jun 4, 2019

Async, EJS6, and showing full path.
As few packages as possible.
Tested in Node 10.16.0

const fs = require('fs').promises
const path = require('path')

async function walk(dir, fileList = []) {
  const files = await fs.readdir(dir)
  for (const file of files) {
    const stat = await fs.stat(path.join(dir, file))
    if (stat.isDirectory()) fileList = await walk(path.join(dir, file), fileList)
    else fileList.push(path.join(dir, file))
  }
  return fileList
}

walk('/home/').then((res) => {
  console.log(res)
})

@darioblanco
Copy link

darioblanco commented Jun 6, 2019

@SamMaxwell approach without using lodash

const fs = require('fs');
const path = require('path');

/** Retrieve file paths from a given folder and its subfolders. */
const getFilePaths = (folderPath) => {
  const entryPaths = fs.readdirSync(folderPath).map(entry => path.join(folderPath, entry));
  const filePaths = entryPaths.filter(entryPath => fs.statSync(entryPath).isFile());
  const dirPaths = entryPaths.filter(entryPath => !filePaths.includes(entryPath));
  const dirFiles = dirPaths.reduce((prev, curr) => prev.concat(getFilePaths(curr)), []);
  return [...filePaths, ...dirFiles];
};

module.exports = getFilePaths;

@dmcgrandle
Copy link

dmcgrandle commented Aug 6, 2019

rxjs 6.5, Typescript 3.5, node 10.6, modifying @quantumsheep's solution above, including both files and directories in a single pass, streaming results into an Observable of a custom type:

import { Stats, promises as fs } from 'fs';
import { Observable } from 'rxjs';

const dirsAndFiles = (dir: string): Observable<FileObject> =>
  new Observable(
    (observer): void => {
      // Create an async recursive function for walking the directory tree:
      const walk = async (dir: string): Promise<void> => {
        const files = await fs.readdir(join(__dirname, dir));
        for (const file of files) {
          const filepath = join(dir, file);
          const stat: Stats = await fs.stat(join(__dirname, filepath));
          if (stat.isFile() || stat.isDirectory()) {
            observer.next({
              isFile: stat.isFile(),
              filename: file,
              size: stat.size,
              path: dir
            });
          }
          if (stat.isDirectory()) {
            await walk(filepath);
          }
        }
      };
      // now call that async function and complete the Observable when it resolves
      walk(dir)
        .then((): void => observer.complete())
        .catch((err): void => observer.error(err));
    }
  );

// later in code:
dirsAndFiles(dir).subscribe(
  (file: FileObject) => doSomething(file),
  (err): void => processError(err)
);

@blaasvaer
Copy link

All fine, but how do you create an object than simulates the structure of the actual directory structure?

Like, from this:

Fig 1

LEVEL_1
	LEVEL_2
	|	LEVEL_3_1
	|	|	FILE_3_1_1
	|	|	FILE_3_1_2
	|	LEVEL_3_2
	|	|	FILE_3_2_1
	|	|	FILE_3_2_2
	|	|	LEVEL_4
	|	|	|	FILE_4_1
	|	|	|	FILE_4_2
	|	|	|	... this could go on forever ...
	|	FILE_2_1
	|	FILE_2_2
	FILE_1_1
	FILE_1_2

to this:

Fig 2

{
	LEVEL_2 : {
		LEVEL_3_1 : {
			FILE_3_1_1 : "FILE CONTENT",
			FILE_3_1_2 : "FILE CONTENT"
		},
		LEVEL_3_2 : {
			FILE_3_2_1 : "FILE CONTENT",
			FILE_3_2_2 : "FILE CONTENT"
			LEVEL_4 : {
				FILE_4_1 : "FILE CONTENT",
				FILE_4_2 : "FILE CONTENT"
			}
		},
		FILE_1_1 : "FILE CONTENT",
		FILE_2_1 : "FILE CONTENT"
	}
}

@geenloop
Copy link

good question, same here... anyone with ES5 old school simple solution?

@geenloop
Copy link

geenloop commented Aug 26, 2019

ouuu hell yea... I love google and I love open source comunity... works just like that:

https://www.npmjs.com/package/directory-tree

const dirTree = require("directory-tree");
const tree = dirTree("/Users/admin/Music");

console.log(tree)

thats all... the rest is up to Angular-tree-control xDDD good luck

*edit - solution #2 https://www.npmjs.com/package/dree NOT tested, just another option

@blaasvaer
Copy link

These return the structure alright in some 'weird' way, with all sorts of properties. My own solution to the problem with help from guys over at stackoverflow is more the kind I was looking for: https://stackoverflow.com/a/57626051/8082874

It return a straight object with folders as objects, filenames as keys – and in this case file contents as precompiled Handlebars templates ... like this:

{
	admin: {
		admin_subviews: {
			index: [Function]
		},
		header: {
			[Function: ret] isTop: true,
			_setup: [Function],
			_child: [Function]
		},
		index: {
			[Function: ret] isTop: true,
			_setup: [Function],
			_child: [Function]
		},
		layout: {
			[Function: ret] isTop: true,
			_setup: [Function],
			_child: [Function]
		}
	},
	index:{
		[Function: ret] isTop: true,
		_setup: [Function],
		_child: [Function]
	},
	layout:	{
		[Function: ret] isTop: true,
		_setup: [Function],
		_child: [Function]
	},
	login: {
		index: {
			[Function: ret] isTop: true,
			_setup: [Function],
			_child: [Function]
		},
		layout: {
			[Function: ret] isTop: true,
			_setup: [Function],
			_child: [Function]
		}
	}
}

The root level is the object itself and you can manipulate the file content in any way you want and inject it as the value.

@geenloop
Copy link

I have no idea what you mean but for me it just rendered JSON on server of all files/folders in tree -> send that tree to front-end-> and https://github.com/wix/angular-tree-control just directly render all of that at a glance 👍 but you look like that you need more than that...

@blaasvaer
Copy link

Probably yes. And whenever something contains 'angular' I'm usually out anyway, so ; )

@geenloop
Copy link

yea xD I'm locked in the 2015 year development and ES5 but I'm so grateful for that but depends on everybody's needs ;) good luck anyway

@meowsus
Copy link

meowsus commented Nov 10, 2019

Here's an ES6 refactor. One that applies RegExp filtering:

const fs = require('fs');
const path = require('path');

function findInDir (dir, filter, fileList = []) {
  const files = fs.readdirSync(dir);

  files.forEach((file) => {
    const filePath = path.join(dir, file);
    const fileStat = fs.lstatSync(filePath);

    if (fileStat.isDirectory()) {
      findInDir(filePath, filter, fileList);
    } else if (filter.test(filePath)) {
      fileList.push(filePath);
    }
  });

  return fileList;
}

// Usage
findInDir('./public/audio/', /\.mp3$/);

@MichaelGatesDev
Copy link

@meowsus Your solution works great, thanks.

@konstantin-hatvan
Copy link

Just another implementation

const path = require('path');
const fs = require('fs');

/* Prepend the given path segment */
const prependPathSegment = pathSegment => location => path.join(pathSegment, location);

/* fs.readdir but with relative paths */
const readdirPreserveRelativePath = location => fs.readdirSync(location).map(prependPathSegment(location));

/* Recursive fs.readdir but with relative paths */
const readdirRecursive = location => readdirPreserveRelativePath(location)
    .reduce((result, currentValue) => fs.statSync(currentValue).isDirectory()
        ? result.concat(readdirRecursive(currentValue))
        : result.concat(currentValue), []);

@gemyero
Copy link

gemyero commented Mar 8, 2020

I think the most elegant way to list all files and/or directories recursively is using globs. You can use globby.

const globby = require('globby');

const listAllFilesAndDirs = dir => globby(`${dir}/**/*`);

(async () => {
  const result = await listAllFilesAndDirs(process.cwd());
  console.log(result);
})();

@danilosetubal
Copy link

@gemyero Great solution, thanks!

@SakiiR
Copy link

SakiiR commented Jul 16, 2020

mmh Carefull with globby ;)

Here is my way (with typescript):

import fs from "fs/promises";
import path from "path";

export default async function walk(directory: string) {
  let fileList: string[] = [];

  const files = await fs.readdir(directory);
  for (const file of files) {
    const p = path.join(directory, file);
    if ((await fs.stat(p)).isDirectory()) {
      fileList = [...fileList, ...(await walk(p))];
    } else {
      fileList.push(p);
    }
  }

  return fileList;
}

@ariassd
Copy link

ariassd commented Dec 8, 2020

Alternative solution in Just two lines, Just for fun and learning

const path = ''; // 👈 path to your location
const list = require("child_process").execSync(`cd ${path} && ls -R`).toString().split(`\n`);

⚠️ IMPORTANT
Be careful, this solution use a bash script and it works fine in MACOS, for windows or linux could be different

@kiuKisas
Copy link

kiuKisas commented Feb 18, 2021

What I use is based on @vidul-nikolaev-petrov version, with a callback argument to do something with files:

const walkSync = (dir, callback) =>  fs.lstatSync(dir).isDirectory()
        ? fs.readdirSync(dir).map(f => walkSync(path.join(dir, f), callback))
        : callback(dir);

// Note: call to flat at the end to have one array with every paths
const svgs = walkSync(path, fileToBase64).flat()

@SakiiR
Copy link

SakiiR commented Feb 18, 2021

Alternative solution in Just two lines, Just for fun and learning

const path = ''; // 👈 path to your location
const list = require("child_process").execSync(`cd ${path} && ls -R`).toString().split(`\n`);

⚠️ IMPORTANT
Be careful, this solution use a bash script and it works fine in MACOS, for windows or linux could be different

Please, never do that

@Aleksandar1932
Copy link

@JESUSrosetolife
Copy link

Hi, here's a modern version:

const fs = require('fs').promises;
const path = require('path');

const walk = async (dir, filelist = []) => {
  const files = await fs.readdir(dir);

  for (file of files) {
    const filepath = path.join(dir, file);
    const stat = await fs.stat(filepath);

    if (stat.isDirectory()) {
      filelist = await walk(filepath, filelist);
    } else {
      filelist.push(file);
    }
  }

  return filelist;
}

Here's what I used (needed import)

import * as fs from 'fs/promises';
import path from 'path';

export async function directory_read(dir, filelist = []) {
  const files = await fs.readdir(dir);

  for (let file of files) {
    const filepath = path.join(dir, file);
    const stat = await fs.stat(filepath);

    if (stat.isDirectory()) {
      filelist = await directory_read(filepath, filelist);
    } else {
      filelist.push(file);
    }
  }

  return filelist;
}

@schadocalex
Copy link

schadocalex commented Oct 12, 2023

Node v20.1+

const filelist = await fs.readdir(dir, { recursive: true });

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment