run_shard_overhead_test

#!/usr/bin/env perl
use LWP;
use HTTP::Request::Common;

my $url = "localhost:9200";
my $protocol = "http";

my $index_name_prefix = "shardtest";
my $counter = 0;
my $indexspec = '{"settings":{"index":{"number_of_shards":"100","number_of_replicas":"0"}}}';

my $baseurl = $protocol . "://" . $url;

my $ua = LWP::UserAgent->new(keep_alive=>1);
$ua->agent('Mozilla/5.0');
$ua->cookie_jar({});

print "timestamp,shard_count,heap_max_in_bytes,heap_used_in_bytes,heap_used_percent\n";

my $test_shards = 0;
my $non_test_shards = 0;

my $delurl = $baseurl . "/" . $index_name_prefix . "*";
my $req = HTTP::Request->new(DELETE => $delurl);
$ua->request($req);

log_error("Starting test.");

while($counter < 10) {
	my $indexname = $index_name_prefix . "-" . $counter;
	my $shardurl = $baseurl . "/" . $indexname;
    my $req = HTTP::Request->new(POST => $shardurl);
    $req->header('content-type' => 'application/json');
    $req->content($indexspec);

    my $resp = $ua->request($req);
    if ($resp->is_success) {
        my $shard_data = get_shard_count($ua, $baseurl);
        bulk_insert_records($ua, $baseurl, $indexname, 10000);
        sleep 1;
        my $jvm_data = get_jvm_data($ua, $baseurl);

        print localtime() . "," . $shard_data . "," . $jvm_data . "\n";        
    }
    else {
        log_error("ERROR creating index. Terminating.");
        exit();
    }

    $counter++;
}

log_error("Test completed.");

sub get_shard_count {
    my ($ua, $baseurl) = @_;
    my $url = $baseurl . '/_cat/shards';
    my $req = $req = HTTP::Request->new(GET => $url);

    while(1) {
        my $resp = $ua->request($req);
        if($resp->is_success) {
        	my $shard_count = 0;
        	my $errors = 0;
            $message = $resp->decoded_content;
            my @lines = split /\n/, $message;
            foreach my $line (@lines) {
                if($line =~ /STARTED/) {
                	$shard_count++;
                } else {
                	$errors++;
                }
            }

            if($errors) {
                sleep 5;
            } else {
            	return $shard_count;
            }
        } else {
       		log_error("ERROR retrieving shard information. Terminating.\n");
            exit();
        }

        $continue = 0;
    }
}

sub get_jvm_data {
	my ($ua) = @_;

    my $url = $baseurl . "/_nodes/stats/jvm";
    my $req = $req = HTTP::Request->new(GET => $url);
    my $resp = $ua->request($req);
    if($resp->is_success) {
    	$jvm_stats = $resp->decoded_content;
        
        my $heap_used_percent;
        if($jvm_stats =~ /\"heap_used_percent\"\s*:\s*(\d+)\D/) {
            $heap_used_percent = $1;
        } else {
        	$heap_used_percent = "unknown";
        }
        
        my $heap_max_in_bytes;
        if($jvm_stats =~ /\"heap_max_in_bytes\"\s*:\s*(\d+)\D/) {
            $heap_max_in_bytes = $1;
        } else {
            $heap_max_in_bytes = "unknown";
        }

        my $heap_used_in_bytes;
        if($jvm_stats =~ /\"heap_used_in_bytes\"\s*:\s*(\d+)\D/) {
            $heap_used_in_bytes = $1;
        } else {
            $heap_used_in_bytes = "unknown";
        }

        return $heap_max_in_bytes . "," . $heap_used_in_bytes . "," . $heap_used_percent;
    } else {
        log_error("ERROR retrieving JVM stats. Terminating.\n");
        exit();
    }
}

sub log_error {
	my ($line) = @_;
	print STDERR localtime() . " " . $line . "\n";
}

sub bulk_insert_records {
    my ($ua, $baseurl, $index, $count) = @_;
    my $body = build_bulk_body($index, $count);

    my $url = $baseurl . "/_bulk";
    my $req = HTTP::Request->new(POST => $url);
    
    $req->content(build_bulk_body($index, $count));
    my $resp = $ua->request($req);
    if(!$resp->is_success) {
    	my $msg = "ERROR bulk indexing records to index " . $index;
    	log_error($msg);
    }
}

sub build_bulk_body {
	my ($index, $count) = @_;
    my $bulk_body = "";

    for($i = 0; $i < $count; $i++) {
        my $header = "{ \"index\" : { \"_index\" : \"" . $index . "\", \"_type\" : \"test\" } }\n";
        my $record = "{ \"sequence\" : " . $i . ", \"text\" : \"This is message " . $count . " for index " . $index . ".\" }\n";
        $bulk_body = $bulk_body . $header . $record;
    }

    return $bulk_body;
}