batch_cv.pl

#!/usr/local/bin/perl

use strict;
use warnings;
use 5.008;

main();

sub main {

    my $server_path = "/home/vagrant/git/jubatus/build/src/server/jubaclassifier";
    my $conf_dir    = "/home/vagrant/eval/conf";
    my $client_path = "/home/vagrant/eval/bin/eval_svmdata";

    my @data_result_array = (
        ["/home/vagrant/eval/data/news20.scale", "/home/vagrant/eval/cv/news20"],
        ["/home/vagrant/eval/data/usps", "/home/vagrant/eval/cv/usps"],
        ["/home/vagrant/eval/data/letter.scale", "/home/vagrant/eval/cv/letter"],
        ["/home/vagrant/eval/data/ijcnn1", "/home/vagrant/eval/cv/ijcnn1"],
        ["/home/vagrant/eval/data/w7a", "/home/vagrant/eval/cv/w7a"],
        ["/home/vagrant/eval/data/mnist.scale", "/home/vagrant/eval/cv/mnist"],
    );

    opendir(my $cdh, $conf_dir);
    my @conf_array = grep {$_ =~ /\.json$/} sort readdir $cdh;
    closedir($cdh);

    #splice @conf_array, 2 if @conf_array > 2;

    my $fold_num = 5;

    foreach my $d_r (@data_result_array) {
        my $data       = $d_r->[0];
        my $result_dir = $d_r->[1];

        system("mkdir -p $result_dir") unless -d $result_dir;

        split_for_cv($data, $result_dir, $fold_num);

        foreach my $conf (@conf_array) {
            my $pid = fork();

            if ($pid < 0) {
                die "fork fail!";
            } elsif ($pid == 0) {
                exec("$server_path -f $conf_dir/$conf");
            } else {
                sleep(2);
                my $result_num = 0;
                while ($result_num < $fold_num) {
                    system("rm -f $result_dir/result_$conf");
                    foreach my $i (0..$fold_num-1) {
                        my $cmd = "$client_path $result_dir/train-$i $result_dir/test-$i >> $result_dir/result_$conf";
                        print "$cmd\n";
                        system($cmd);
                    }
                    $result_num = `wc -l $result_dir/result_$conf | cut -d " " -f 1`;
                    chomp $result_num;
                }
                kill(15, $pid);
            }
        }

        system("rm -f $result_dir/train-* $result_dir/test-*");
    }

}

sub split_for_cv {
    my ($data_f, $out_dir, $fold_num) = @_;

    srand(1000);
    open(my $ifh, '<', $data_f) or die $!;

    my @trfh_array;
    my @tefh_array;
    foreach my $i (0..$fold_num-1) {
        open(my $trfh, '>', "$out_dir/train-$i") or die $!;
        open(my $tefh, '>', "$out_dir/test-$i")  or die $!;
        push @trfh_array, $trfh;
        push @tefh_array, $tefh;
    }

    while (my $line = <$ifh>) {
        my $i = int(rand($fold_num));
        print {$tefh_array[$i]} $line;
        foreach my $j (0..$fold_num-1) {
            print {$trfh_array[$j]} $line unless $j == $i;
        }
    }

    close($_) foreach @trfh_array;
    close($_) foreach @tefh_array;

    close($ifh);
}