Export files recusively to a common directory with a unique sha

dedup.rb

# frozen_string_literal: true

# Export files recusively to a common directory with a unique sha
# Ignore duplicate files, skip directories

require 'pathname'

INPUT_DIR = '/home/jonathan/duplicate-input'
OUTPUT_DIR = '/home/jonathan/deduped-output'

files = Dir["#{INPUT_DIR}/**/*.*"]
files.each do |f|
  path = Pathname.new(f)
  next if path.directory?

  ext = File.extname(f.shellescape)
  sha_result = `sha256sum #{f.shellescape}`

  sha = sha_result.split(' ').first

  if File.exist?("#{OUTPUT_DIR}/#{sha}#{ext}")
    puts "DUPLICATE: #{f.shellescape}"
  else
    puts "copying #{f}"
    `cp #{f.shellescape} #{OUTPUT_DIR}/#{sha}#{ext.downcase}`
  end
end