fdups.py

# http://stackoverflow.com/questions/748675/finding-duplicate-files-and-removing-them/748908#748908

import sys
import os

import mmh3

CHUNK_SIZE = 1024*1024

def check_for_duplicates(paths):
    hashes = {}
    for path in paths:
        for dirpath, _, filenames in os.walk(path):
            for filename in filenames:
                full_path = os.path.join(dirpath, filename)
                h = mmh3.hash(open(full_path, 'rb').read(CHUNK_SIZE))
                file_id = (h, os.path.getsize(full_path))
                duplicate = hashes.get(file_id, None)
                if duplicate:
                    print "Duplicate found: %s and %s" % (full_path, duplicate)
                else:
                    hashes[file_id] = full_path

if sys.argv[1:]:
    check_for_duplicates(sys.argv[1:])
else:
    print "Please pass the paths to check as parameters to the script"