gistfile2.py

In [1]: uniq(load('obr2.dat'))
Out[1]: 2

In [2]: uniq(load('obr3.dat'))
Out[2]: 4

In [3]: uniq(load('set_small.dat'))
Out[3]: 851

uniq.py

#!/usr/bin/python
def insert(line, data):
    hash = line.split(' ')[1:]
    data.setdefault(len(hash), []).append([int(s, 16) for s in hash])

def load(fname):
    data = {}
    fp = file(fname)
    for line in fp.readlines():
        line2 = line.strip()
        insert(line2, data)
    return data

def uniq(data):
    prefixes = []
    unique = 0
    for length in range(max(data.keys()), 0, -1):
        new_prefixes = []
        for hash in data.get(length, []):
            if hash not in prefixes:
                new_prefixes.append(hash[:-1])
                unique = unique + 1
        for prefix in prefixes:
            new_prefixes.append(prefix[:-1])
        prefixes = new_prefixes

    return unique