Answer to stackoverflow question 6089772

data.tsv

process_files.py

# -*- coding: utf-8 -*-

from glob import glob
from os import path
import csv
from collections import namedtuple, defaultdict


def type_fix_value(value):
    _types = (int, float)
    for _type in _types:
        try:
            return _type(value)
        except ValueError:
            continue
    return value

def parse_row(row):
    return [type_fix_value(field) for field in row]


def main(args):
    files = glob( path.join(args.filedir, '*.csv') )
    
    for datafile in files:
        with open(datafile) as srcfile:
            headers = srcfile.next().strip().split('\t')
            datarow = namedtuple('datarow', headers)
            data    = csv.reader(srcfile, delimiter='\t')
        
            ## wrap with row parser and namedtuple
            data = (parse_row(row) for row in data)
            data = (datarow(*row) for row in data)
        
            ## Group by the leading integer columns
            grouped_rows = defaultdict(list)
            for row in data:
                integer_fields = [field for field in row if isinstance(field, int)]
                grouped_rows[tuple(integer_fields)].append(row)
        
            ## DO SOMETHING INTERESTING WITH THE GROUPS
            import pprint
            pprint.pprint(dict(grouped_rows))


if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--file-dir', dest='filedir')
    args = parser.parse_args()
    
    if not path.isdir(args.filedir):
        parser.error('--file-dir should point to a directory')
    
    main(args)