higgs_to_svm.py

#! /bin/env python

"""Convert higgs dataset from GBM-Benchmarks to libsvm format"""

import pandas as pd

with open('HIGGS.csv') as f:
    num_lines = sum(1 for line in f)
print(num_lines)

train_lines = int(num_lines * 0.95)
test_lines = num_lines - train_lines

train_f = open("HIGGS.csv.train", "w")
test_f = open("HIGGS.csv.test", "w")
train_f.write("")
test_f.write("")
train_f = open("HIGGS.csv.train", "a")
test_f = open("HIGGS.csv.test", "a")

with open('HIGGS.csv') as f:
    for li,line in enumerate(f):
        fields = line.split(",")
        output = str(int(float(fields[0])))

        for i,f in enumerate(fields[1:]):
            output += " " + str(i) + ":" + f

        if li % 10000 == 0:
            print(float(li)/num_lines)
        if li < train_lines:
            train_f.write(output)
        else:
            test_f.write(output)