Remove duplicate files found inside one directory

clean-dupes.js

#!/usr/bin/env node

const fs = require('fs')
const path = require('path')

function bySizeThenByName(a, b) {
  if (a.size === b.size) {
    return a.path.localeCompare(b)
  }
  return a.size < b.size
}

const mp4s = fs.readdirSync(__dirname)
  .map(x => path.join(__dirname, x))
  .map(x => {
    const stat = fs.statSync(x)
    return { path: x, isFile: stat.isFile(), size: stat.size }
  })
  .filter(x => x.isFile && x.path.endsWith('.mp4'))
  .map(({ path, size }) => ({ size, path }))
  .sort(bySizeThenByName)

// now iterate through all of them and if we find the same size in a row we'll compare the
// names and if they are similar enough we remove the file with the longer name
let original = { size: 0 }
const dupes = []
for (let i = 0; i < mp4s.length; i++) {
  const x = mp4s[i]
  if (x.size !== original.size) {
    original = x
    continue
  }

  const { shorter, longer } = x.path.length < original.path.length
    ? { shorter: x, longer: original }
    : { shorter: original, longer: x }

  const shorterNoExt = shorter.path.slice(0, -4)
  const longerNoExt = longer.path.slice(0, -4)
  if (longerNoExt.startsWith(shorterNoExt)) {
    dupes.push(longer.path)
  }

  original = shorter
}

if (dupes.length === 0) {
  console.log('No dupes found')
} else if (process.argv[2] === '-y') {
  console.log('Removing:\n', dupes)
  dupes.forEach(x => fs.unlinkSync(x))
} else {
  console.log('Would remove:\n', dupes)
  console.log('Rerun with -y flag to remove them.')
}

Note, this only works for .mp4 files for now as that is hardcoded, but could easily be adapted :)