Remove duplicate files by content

duplicates.rb

#!/usr/bin/ruby

require 'digest'
require 'fileutils'

def paths
  directories.map { |d| Dir.glob("#{d}/**/*") }.flatten.select { |p| File.file?(p) }
end

def directories
  []
end

t1 = Time.now

by_checksum = paths.group_by do |file|
  sha512 = Digest::SHA512.file file
  sha512.hexdigest
end

by_checksum.each do |_checksum, files|
  next if files.size == 1

  file = files.first
  more = files.drop(1)

  more.each do |other|
    File.delete(other) if FileUtils.identical?(file, other)
  end
end

t2 = Time.now

puts "Took #{t2 - t1} seconds"