Find files

map-files.ts

import path from "path"

import fs from "fs-extra"
import ignore from "ignore"
import type { Ignore } from "ignore"
import { minimatch } from "minimatch"
import type { SetOptional } from "type-fest"

const ignoreDirectories = [".*", ".git", ".idea", ".venv", ".vscode", "bin", "node_modules", "obj", "vendor"]
const ignoreBuildDirectories = ["build", "dist", "out", "public"]

const ignoreFiles = [".DS_Store", "desktop.ini", "Thumbs.db"]
const ignoreConfigFiles = [".npm*"]
const ignoreLockFiles = [
  "*-lock.json",
  "*-lock.yaml",
  "*-lockfile.json",
  "*-lockfile.yaml",
  "*.lock",
  "*.lockfile",
  "packages.config",
  "pom.xml.sha1",
  "requirements.txt",
]
const ignoreVersionControlNames = ["_darcs*", "_darcs/*", ".bzr*", ".git*", ".hg*", ".svn*"]
const ignoreTestNames = [
  "*_mocks_*",
  "*_test_*",
  "*_tests_*",
  "*.test.*",
  "*.tests.*",
  "*test-*",
  "coverage",
  "e2e",
  "spec",
  "test",
  "tests",
]
const ignoreTestLib = ["cypress", "jest", "chai", "mocha", "vitest"]
const storybookFiles = ["stories", "*.stories.*"]

type MapFilesOptions<C = undefined> = {
  ignorePatterns?: string[]
  includePatterns?: string[]
  callback?: (filePath: string, control: { context: C; stop: () => undefined }) => unknown
  initialContext?: C
  copyContext?: (context: C) => C
  findUp: boolean
}

const defaultOptions = {
  ignorePatterns: [
    ...ignoreDirectories,
    ...ignoreBuildDirectories,
    ...ignoreFiles,
    ...ignoreLockFiles,
    ...ignoreConfigFiles,
    ...ignoreVersionControlNames,
    ...ignoreTestNames,
    ...ignoreTestLib,
    ...storybookFiles,
  ],
  findUp: false,
}

/**
 * Recursively scans a directory and maps file paths to their content, while honoring .gitignore rules.
 * @param directoryPath - The path of the directory to be scanned.
 * @param options - Optional configuration options.
 * @returns A Promise resolving to a Map containing file paths mapped to their content.
 */
export async function mapFiles<C = undefined>(directoryPath: string, options: SetOptional<MapFilesOptions<C>, keyof MapFilesOptions> = {}) {
  const pathList: string[] = []
  const { ignorePatterns, includePatterns, initialContext, callback, copyContext, findUp } = { ...defaultOptions, ...options }

  const createNewIgnore = async (directoryPath: string, _ig?: any) => {
    const gitIgnoreFilePath = path.join(directoryPath, ".gitignore")
    const hasGitIgnoreFile = await fs.exists(gitIgnoreFilePath)
    if (!hasGitIgnoreFile && _ig) return _ig as Ignore

    const ig = ignore({
      ignoreCase: _ig?._ignoreCase ?? true,
      allowRelativePaths: _ig?._allowRelativePaths ?? true,
    })
    // Add ignore paths
    ig.add(_ig ? _ig?._rules?.map?.((rule: any) => rule.origin) : ignorePatterns)

    // Add .gitignore paths
    if (!hasGitIgnoreFile) return ig
    const gitignoreContent = await fs.readFile(gitIgnoreFilePath, "utf-8")
    ig.add(gitignoreContent)
    return ig
  }

  let stop = false
  const control = { context: initialContext, stop: () => void (stop = true) }

  /**
   * Recursively crawls through the directory and populates the fileContentMap.
   * @param currentDir - The current directory to be crawled.
   */
  async function crawlRecursive(currentDir: string, context: C, _ig?: Ignore) {
    if (stop) return
    const [files, ig] = await Promise.all([fs.readdir(currentDir, { withFileTypes: true }), createNewIgnore(currentDir, _ig)])

    files.sort((a, b) => {
      if (a.isDirectory() && b.isFile()) return 1
      if (a.isFile() && b.isDirectory()) return -1
      if ((a.isDirectory() && b.isDirectory()) || (a.isFile() && b.isFile())) return a.name.localeCompare(b.name)
      return 0
    })

    for (const file of files) {
      if (stop) return
      const filePath = path.join(currentDir, file.name)
      if (ig.ignores(path.relative(currentDir, filePath))) continue

      const realPath = await fs.realpath(filePath)
      if (file.isDirectory()) {
        if (findUp) continue
        await crawlRecursive(realPath, copyContext ? copyContext(context) : context, ig)
      } else {
        if (!includePatterns || includePatterns.length < 1 || includePatterns.some((pattern) => minimatch(realPath, pattern))) {
          if (callback) callback(realPath, { ...control, context })
          pathList.push(realPath)
          if (stop) return
        }
      }

      if (findUp) {
        await crawlRecursive(path.resolve(currentDir, ".."), copyContext ? copyContext(context) : context, ig)
      }
    }
  }

  // Start the directory traversal
  await crawlRecursive(directoryPath, initialContext as C)
  return pathList
}

Object.freeze(defaultOptions)

mapFiles.options = defaultOptions