Deduplicate files

deduplicate

#!/bin/bash
# Deduplicates files based on md5 hash
#
# Usage:
#	./deduplicate <directory_name>
#
#
# Output like:
# [DUPLICATE]:  (./Documents/Olympus pics/Stockholm445.JPG)  found in:  (./Documents/P5310445.JPG)
#
# Suggestion:
# Redirect output to file (dups) and alter such that filenames are on individual lines, omitting the filenames which you want to keep.
# Then use the file as to-delete list like this (use IFS to counter the plague in the form of spaces in filenames):
#
# IFS=$(echo -en "\n\b"); for file in `cat dups`; do echo Removing duplicate: "$file"; sudo rm "$file"; done
directory=$1

find ./$directory -type f -exec md5 {} \; \
	| awk -F'=' '{gsub(/ /, "", $2); gsub(/MD5/, "", $1); print $2 "|" $1}' \
	| sort \
	| awk 'BEGIN{ FS="|"; count = 1 } \
		{ curr = $1; \
		 if(curr == prev) { \
			rec = rec " " $2; \
			count = count + 1 \
		} else { \
			if(count>1) { \
				print rec " "; \
				count = 1; \
			}; \
			rec = "[DUPLICATE]: " $2 " found in:" \
		}; \
		prev = curr \
		} \
		END{if(count>1) print rec}'