De-duplicate a directory of files

dedup.sh

#! /bin/bash
set -eu

DB=~/.dedup
DUPS=dups

if [ ! -e $DB ]; then
  mkdir $DB
fi

if [ ! -e $DUPS ]; then
  mkdir $DUPS
fi

find . -type f | while read F ; do
  printf "%-80s\r" "$F"
  NAME=$(basename "$F")
  SIZE=$(stat -f %z "$F")
  HASH=$(cat "$F" | openssl sha1)
  HASH_FILE="$DB/$HASH"
  if [ -e "$HASH_FILE" ]; then
    EXPECTED=$(cat "$HASH_FILE")
    if [ "$EXPECTED" != "$F" ]; then
      echo "$F duplicates $EXPECTED"
      mv -f "$F" $DUPS/
    fi
  else
    echo "$F" > "$HASH_FILE"
  fi
done