shards_analysis

#!/usr/bin/env perl

use POSIX;

my $KB = 1024;
my $MB = $KB*1024;
my $GB = $MB*1024;
my $TB = $GB*1024;

my %index_stats;
my %node_stats;

foreach $line (<STDIN>) {
    chomp;

    if($line =~ /^(\S+)\s+\d+\s+(\S)\s+\S+\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s*$/) {
        my ($idx_type, $prim, $docs, $size, $ip, $name) = (extract_index_type($1), $2, $3, string_to_size($4), $5, $6);
        
        if(!$node_stats{$name}) {
            if($prim eq "p") {
                $node_stats{$name}{"primary_shards"} = 1;
    	        $node_stats{$name}{"replica_shards"} = 0;
    	        $node_stats{$name}{"primary_document_count"} = $docs;
    	        $node_stats{$name}{"replica_document_count"} = 0;
    	        $node_stats{$name}{"primary_size"} = $size;
    	        $node_stats{$name}{"replica_size"} = 0;
    	        $node_stats{$name}{"max_primary_shard_size"} = $size;
                $node_stats{$name}{"max_replica_shard_size"} = 0;
            } else {
                $node_stats{$name}{"primary_shards"} = 0;
    	        $node_stats{$name}{"replica_shards"} = 1;
    	        $node_stats{$name}{"primary_document_count"} = 0;
    	        $node_stats{$name}{"replica_document_count"} = $docs;
    	        $node_stats{$name}{"primary_size"} = 0;
    	        $node_stats{$name}{"replica_size"} = $size;
    	        $node_stats{$name}{"max_primary_shard_size"} = 0;
                $node_stats{$name}{"max_replica_shard_size"} = $size;
            }

            $node_stats{$name}{"ip"} = $ip;
            $node_stats{$name}{"max_shard_size"} = $size;
    	    $node_stats{$name}{"min_shard_size"} = $size;

        } else {
            if($prim eq "p") {
                $node_stats{$name}{"primary_shards"} += 1;
    	        $node_stats{$name}{"primary_document_count"} += $docs;
    	        $node_stats{$name}{"primary_size"} += $size;
    	        if($size > $node_stats{$name}{"max_primary_shard_size"}) { $node_stats{$name}{"max_primary_shard_size"} = $size };
            } else {
    	        $node_stats{$name}{"replica_shards"} += 1;
    	        $node_stats{$name}{"replica_document_count"} += $docs;
    	        $node_stats{$name}{"replica_size"} += $size;
    	        if($size > $node_stats{$name}{"max_replica_shard_size"}) { $node_stats{$name}{"max_replica_shard_size"} = $size };
            }
        }

        if(!$index_stats{$idx_type}) {
            if($prim eq "p") {
                $index_stats{$idx_type}{"primary_shards"} = 1;
    	        $index_stats{$idx_type}{"replica_shards"} = 0;
    	        $index_stats{$idx_type}{"primary_document_count"} = $docs;
    	        $index_stats{$idx_type}{"replica_document_count"} = 0;
    	        $index_stats{$idx_type}{"primary_size"} = $size;
    	        $index_stats{$idx_type}{"replica_size"} = 0;
    	        $index_stats{$idx_type}{"max_primary_shard_size"} = $size;
    	        $index_stats{$idx_type}{"max_replica_shard_size"} = 0;
            } else {
                $index_stats{$idx_type}{"primary_shards"} = 0;
    	        $index_stats{$idx_type}{"replica_shards"} = 1;
    	        $index_stats{$idx_type}{"primary_document_count"} = 0;
    	        $index_stats{$idx_type}{"replica_document_count"} = $docs;
    	        $index_stats{$idx_type}{"primary_size"} = 0;
    	        $index_stats{$idx_type}{"replica_size"} = $size;
    	        $index_stats{$idx_type}{"max_primary_shard_size"} = 0;
    	        $index_stats{$idx_type}{"max_replica_shard_size"} = $size;
            }
        } else {
            if($prim eq "p") {
                $index_stats{$idx_type}{"primary_shards"} += 1;
    	        $index_stats{$idx_type}{"primary_document_count"} += $docs;
    	        $index_stats{$idx_type}{"primary_size"} += $size;
    	        if($size > $index_stats{$idx_type}{"max_primary_shard_size"}) { $index_stats{$idx_type}{"max_primary_shard_size"} = $size };
            } else {
    	        $index_stats{$idx_type}{"replica_shards"} += 1;
    	        $index_stats{$idx_type}{"replica_document_count"} += $docs;
    	        $index_stats{$idx_type}{"replica_size"} += $size;
    	        if($size > $index_stats{$idx_type}{"max_replica_shard_size"}) { $index_stats{$idx_type}{"max_replica_shard_size"} = $size };
            }
        }
    }
}

printf("NODE STATISTICS\n\nNode Name, Node IP, Shard Type, Shard Count, Document Dount, Total Shard Size, Average Shard Size, Max Shard Size\n");

foreach my $key (sort keys %node_stats) {
    my $avg_prim_size = ($node_stats{$key}{"primary_shards"} eq 0) ? 0 : ($node_stats{$key}{"primary_size"} / $node_stats{$key}{"primary_shards"});
    my $avg_repl_size = ($node_stats{$key}{"replica_shards"} eq 0) ? 0 : ($node_stats{$key}{"replica_size"} / $node_stats{$key}{"replica_shards"});
    my $avg_size = (($node_stats{$key}{"primary_size"} + $node_stats{$key}{"replica_size"}) / ($node_stats{$key}{"primary_shards"} + $node_stats{$key}{"replica_shards"}));
    my $max_size = max($node_stats{$key}{"max_primary_shard_size"}, $node_stats{$key}{"max_replica_shard_size"});

    printf "%s,%s,PRIMARY,%d,%d,%s,%s,%s\n", 
        $key,
        $node_stats{$key}{"ip"},
        $node_stats{$key}{"primary_shards"},
        $node_stats{$key}{"primary_document_count"},
        size_to_str($node_stats{$key}{"primary_size"}),
        size_to_str($avg_prim_size),
        size_to_str($node_stats{$key}{"max_primary_shard_size"});

    printf "%s,%s,REPLICA,%d,%d,%s,%s,%s\n", 
        $key,
        $node_stats{$key}{"ip"},
        $node_stats{$key}{"replica_shards"},
        $node_stats{$key}{"replica_document_count"},
        size_to_str($node_stats{$key}{"replica_size"}),
        size_to_str($avg_repl_size),
        size_to_str($node_stats{$key}{"max_replica_shard_size"});

    printf "%s,%s,TOTAL,%d,%d,%s,%s,%s\n",
        $key,
        $node_stats{$key}{"ip"},
        ($node_stats{$key}{"primary_shards"} + $node_stats{$key}{"replica_shards"}),
        ($node_stats{$key}{"primary_document_count"} + $node_stats{$key}{"replica_document_count"}),
        size_to_str($node_stats{$key}{"primary_size"} + $node_stats{$key}{"replica_size"}),
        size_to_str($avg_size),
        size_to_str($max_size);
}

printf("\n\nINDEX STATISTICS\n\nIndex Type, Shard Type, Shard Count, Document Dount, Total Shard Size, Average Shard Size, Max Shard Size, Avg Indexed Document Size\n");

foreach my $key (sort keys %index_stats) {
    my $avg_prim_size = ($index_stats{$key}{"primary_shards"} == 0) ? 0 : ($index_stats{$key}{"primary_size"} / $index_stats{$key}{"primary_shards"});
    my $avg_repl_size = ($index_stats{$key}{"replica_shards"} == 0) ? 0 : ($index_stats{$key}{"replica_size"} / $index_stats{$key}{"replica_shards"});
    my $avg_size = (($index_stats{$key}{"primary_size"} + $index_stats{$key}{"replica_size"}) / ($index_stats{$key}{"primary_shards"} + $index_stats{$key}{"replica_shards"}));
    my $max_size = max($index_stats{$key}{"max_primary_shard_size"}, $index_stats{$key}{"max_replica_shard_size"});
    my $avg_doc_size;    
    if($index_stats{$key}{"primary_document_count"} > 0) {
        $avg_doc_size = $index_stats{$key}{"primary_size"} / $index_stats{$key}{"primary_document_count"};
    } else {
        $avg_doc_size = 0;
    }

    printf "%s,PRIMARY,%d,%d,%s,%s,%s,%d\n", 
        $key,
        $index_stats{$key}{"primary_shards"},
        $index_stats{$key}{"primary_document_count"},
        size_to_str($index_stats{$key}{"primary_size"}),
        size_to_str($avg_prim_size),
        size_to_str($index_stats{$key}{"max_primary_shard_size"}),
        $avg_doc_size;

    printf "%s,REPLICA,%d,%d,%s,%s,%s,%d\n", 
        $key,
        $index_stats{$key}{"replica_shards"},
        $index_stats{$key}{"replica_document_count"},
        size_to_str($index_stats{$key}{"replica_size"}),
        size_to_str($avg_repl_size),
        size_to_str($index_stats{$key}{"max_replica_shard_size"}),
        $avg_doc_size;

    printf "%s,TOTAL,%d,%d,%s,%s,%s,%d\n",
        $key,
        ($index_stats{$key}{"replica_shards"} + $index_stats{$key}{"replica_shards"}),
        ($index_stats{$key}{"primary_document_count"} + $index_stats{$key}{"replica_document_count"}),
        size_to_str(($index_stats{$key}{"primary_size"} + $index_stats{$key}{"replica_size"})),
        size_to_str($avg_size),
        size_to_str($max_size),
        $avg_doc_size;
}

printf "\n";

sub extract_index_type {
    my $index = shift;

    if($index =~ /^(\D+?)\-\d+/) {
        return $1;
    } else {
        return $index;
    }
}

sub string_to_size {
    my $str = shift;

    $str =~ /^(.*\d)(\w+)$/;

    my $size = $1;

    if    ($2 =~ /^kb/) { $size *= $KB; }
    elsif ($2 =~ /^mb/) { $size *= $MB; }
    elsif ($2 =~ /^gb/) { $size *= $GB; }
    
    return floor($size);
}

sub size_to_str {
	my $str = "";
	my $size = shift;

    if($size > $TB) {
        $str = sprintf("%0.1ftb", ($size / $TB));
    } elsif($size > $GB) {
        $str = sprintf("%0.1fgb", ($size / $GB));
    } elsif($size > $MB) {
        $str = sprintf("%0.1fmb", ($size / $MB));
    } else {
        $str = sprintf("%0.1fkb", ($size / $KB));
    }

    return $str;
}

sub max {
	my ($a, $b) = shift;

	if($a > $b) {
		return $a;
	}

	return $b;
}