dupefinder.py

# these two functions from github.com/lahwran/dotfiles:bin/summarize
import time
import hashlib

def iterfile(f, bufsize=None):
    if not bufsize:
        bufsize = 8192
    while True:
        buff = f.read(bufsize)
        if not buff:
            break
        yield buff


def hashfile(filename, bufsize=None):
    with open(filename, "rb") as f:
        sha = hashlib.sha256()
        progress = 0
        lastprogress = 0

        for chunk in iterfile(f, bufsize):
            progress += len(chunk)
            if time.time() - lastprogress > 1:
                print "%0.2f%%" % ((float(progress) / float(statsize)) * 100)
                lastprogress = time.time()

            sha.update(chunk)
        return sha.digest()


sizes = ((filename, os.stat(filename).st_size) for filename in <filenames here>)
size_lists = {}
for filename, size in sizes:
    size_lists.setdefault(size, []).append(filename)
possible_dupes = ((size, filenames) for size, filenames in size_lists.iteritems()
                  if len(filenames) > 1)
dupes = {}
for size, filenames in possible_dupes:
    for filename in filenames:
        hash = hashfile(filename)
        dupes.setdefault(hash, []).append(filename)

for hash, filenames in dupes.iteritems():
    print "have hash", hash
    for filename in filenames:
        print "\t", filename
    print