fs.readFileSync() reads hot tiny files much faster than fs.readFile()

micro-benchmark.js

#!/usr/bin/env node
'use strict'

const fs = require('fs')
const util = require('util')

const paths = []
for (let i = 1; i <= 20000; i++) {
  paths.push(`small-files/${i}.txt`)
}

function readFilesSync() {
  const ret = []
  for (let i = 0; i < paths.length; i++) {
    ret.push(fs.readFileSync(paths[i]))
  }
  return ret
}

function readFilesNextTick(callback) {
  const ret = []
  let i = 0

  function step() {
    fs.readFile(paths[i], (err, buf) => {
      if (err) return callback(err)
      ret.push(buf)
      i += 1
      if (i >= paths.length) return callback(null, ret)
      process.nextTick(step)
    })
  }

  step()
}

function readFilesSetImmediate(callback) {
  const ret = []
  let i = 0

  function step() {
    fs.readFile(paths[i], (err, buf) => {
      if (err) return callback(err)
      ret.push(buf)
      i += 1
      if (i >= paths.length) return callback(null, ret)
      setImmediate(step)
    })
  }

  step()
}

async function readFilesPromisify() {
  const readFilePromise = util.promisify(fs.readFile)

  const ret = []
  for (let i = 0; i < paths.length; i++) {
    ret.push(await readFilePromise(paths[i]))
  }

  return ret
}

async function readFilesFsPromises() {
  const ret = []
  for (let i = 0; i < paths.length; i++) {
    ret.push(await fs.promises.readFile(paths[i]))
  }

  return ret
}

const d1 = new Date()
readFilesSync()
const d2 = new Date()
console.log(`sync: ${Math.round(d2 - d1)}ms`)

const d3 = new Date()
readFilesNextTick(err => {
  if (err) throw err
  const d4 = new Date()
  console.log(`async (process.nextTick): ${Math.round(d4 - d3)}ms`)
  console.log(`async (process.nextTick) is ${((d4 - d3) / (d2 - d1)).toFixed(1)}x slower than sync at reading lots of small files`)

  const d5 = new Date()
  readFilesSetImmediate(err => {
    if (err) throw err
    const d6 = new Date()
    console.log(`async (setImmediate): ${Math.round(d6 - d5)}ms`)
    console.log(`async (setImmediate) is ${((d6 - d5) / (d2 - d1)).toFixed(1)}x slower than sync at reading lots of small files`)

    const d7 = new Date()
    readFilesPromisify().then(() => {
      const d8 = new Date()
      console.log(`promise (promisify): ${Math.round(d8 - d7)}ms`)
      console.log(`promise (promisify) is ${((d8 - d7) / (d2 - d1)).toFixed(1)}x slower than sync at reading lots of small files`)
    })
      .then(() => {
        const d9 = new Date()
        readFilesFsPromises().then(() => {
          const d10 = new Date()
          console.log(`promise (fs.promises): ${Math.round(d10 - d9)}ms`)
          console.log(`promise (fs.promises) is ${((d10 - d9) / (d2 - d1)).toFixed(1)}x slower than sync at reading lots of small files`)
        })
      })
  })
})

output

sync: 106ms
async (process.nextTick): 923ms
async (process.nextTick) is 8.7x slower than sync at reading lots of small files
async (setImmediate): 944ms
async (setImmediate) is 8.9x slower than sync at reading lots of small files
promise (promisify): 993ms
promise (promisify) is 9.4x slower than sync at reading lots of small files
promise (fs.promises): 1436ms
promise (fs.promises) is 13.5x slower than sync at reading lots of small files

setup.sh

#!/bin/sh

mkdir -p small-files
for i in `seq 1 20000`; do echo "This is file $i" > small-files/$i.txt; done