batch_eval_classifier2.pl

#!/usr/local/bin/perl

use strict;
use warnings;
use 5.008;

use Data::Dumper;

main();

sub main {

    my $server_path = "/home/vagrant/local/bin/jubaclassifier";
    my $conf_dir    = "/home/vagrant/ranking/conf";
    my $script_dir  = "/home/vagrant/ranking/script";
    my $client_path = "/home/vagrant/ranking/bin/eval_classifier2";

    my @data_result_array = (
        ["/home/vagrant/ranking/data/LETOR3.0/OHSUMED/QueryLevelNorm/", "/home/vagrant/ranking/result/classifier2/OHSUMED/", "/home/vagrant/ranking/script/Eval-Score-3.0.pl"],
        ["/home/vagrant/ranking/data/LETOR4.0/MQ2007/", "/home/vagrant/ranking/result/classifier2/MQ2007/", "/home/vagrant/ranking/script/Eval-Score-4.0.pl"],
        ["/home/vagrant/ranking/data/LETOR4.0/MQ2008/", "/home/vagrant/ranking/result/classifier2/MQ2008/", "/home/vagrant/ranking/script/Eval-Score-4.0.pl"],
    );

    my @conf_array = ();
    push @conf_array, "pa.json";
    #push @conf_array, "pa1_04.json";
    #push @conf_array, "pa2_04.json";
    push @conf_array, "cw_07.json";
    push @conf_array, "arow_04.json";
    push @conf_array, "nherd_04.json";

    foreach my $d_r (@data_result_array) {
        my $data_dir    = $d_r->[0];
        my $result_dir  = $d_r->[1];
        my $eval_script = $d_r->[2];

        foreach my $fold (1..5) {
            my $data_fold_dir   = "$data_dir/Fold$fold";
            my $result_fold_dir = "$result_dir/Fold$fold";

            system("mkdir -p $result_fold_dir") unless -d $result_fold_dir;

            my $train_in  = "$data_fold_dir/train.txt";
            my $valid_in  = "$data_fold_dir/vali.txt";
            my $test_in   = "$data_fold_dir/test.txt";
            my $diff_in = "$result_fold_dir/diff.txt";

            my $cmd = "perl $script_dir/make_pairwise.pl $train_in > $diff_in";
            print "$cmd\n"; system($cmd);

            foreach my $conf (@conf_array) {
                my $valid_out = "$result_fold_dir/result_valid_$conf";
                my $test_out  = "$result_fold_dir/result_test_$conf";


                my $pid = fork();

                if ($pid < 0) {
                    die "fork fail!";
                } elsif ($pid == 0) {
                    exec("$server_path -f $conf_dir/$conf");
                } else {
                    sleep(2);
                    my $result_num = 0;
                    system("rm -f $valid_out $test_out");
                    while (! (-f $valid_out && -f $test_out)) {
                        my $cmd = "$client_path $diff_in $valid_in $test_in $valid_out $test_out";
                        print "$cmd\n";
                        system($cmd);

                        sleep(1) if (! (-f $valid_out && -f $test_out));
                    }
                    kill(15, $pid);
                }

                my $valid_score = "$result_fold_dir/score_valid_$conf";
                my $test_score  = "$result_fold_dir/score_test_$conf";

                $cmd = "perl $eval_script $valid_in $valid_out $valid_score 0";
                print "$cmd\n";
                system($cmd);

                $cmd = "perl $eval_script $test_in $test_out $test_score 0";
                print "$cmd\n";
                system($cmd);
            }

            $cmd = "rm -f $diff_in";
            print "$cmd\n"; system($cmd);
        }
    }

}