aug-19-script

#!/bin/bash

mins=10

echo "Collecting metrics for $mins minutes..."
dstat_outfile="dstat-`hostname`.csv"
echo "Writing dstat output to $dstat_outfile"
#dstat --output $dstat_outfile -pmgsdr --disk-util -nyct -C $(seq --separator=, 0 `grep -c processor /proc/cpuinfo`),total 1 $mins*60 &
dstat --output "dstat-`hostname`.csv" -pmgsdr --disk-util --disk-avgqu --disk-avgrq --disk-svctm --disk-tps --disk-wait -nyct -C $(seq --separator=, 0 `grep -c processor /proc/cpuinfo`),total 1 $mins*60 &

# Limit the perf sched record to $perf_secs seconds because the data file can grow pretty large.
perf_secs=10
perf_outfile="perf-`hostname`.data"
echo "Recording scheduler events to $perf_outfile for $perf_secs seconds"
# Make sure all the sched tracepoints are available to us.
sudo bash -c 'echo 1 > /proc/sys/kernel/sched_schedstats'
sudo perf sched record -e 'task:*' -o $perf_outfile -a -- sleep 60 &

latency_outfile="jmx-lat-`hostname`.csv"
echo "Writing JMX latency metrics to $latency_outfile"
printf "Local Read Latency (p95),One Min Rate,Local Write Latency (p95),One Min Rate,Coordinator Read Latency (p95),One Min Rate,Coordinator Write Latency (p95), One Min rate\n" >> $latency_outfile
for i in $(seq 1 $mins); do
	# Local table latencies for domain_1300.xml_doc_1300
	local_read_p95=$(nodetool sjk mx -b 'org.apache.cassandra.metrics:type=ColumnFamily,keyspace=domain_1300,scope=xml_doc_1300,name=ReadLatency' -f 95thPercentile -mg | tail -n1)
	local_read_rate=$(nodetool sjk mx -b 'org.apache.cassandra.metrics:type=ColumnFamily,keyspace=domain_1300,scope=xml_doc_1300,name=ReadLatency' -f OneMinuteRate -mg | tail -n1)
	local_write_p95=$(nodetool sjk mx -b 'org.apache.cassandra.metrics:type=ColumnFamily,keyspace=domain_1300,scope=xml_doc_1300,name=WriteLatency' -f 95thPercentile -mg | tail -n1)
	local_write_rate=$(nodetool sjk mx -b 'org.apache.cassandra.metrics:type=ColumnFamily,keyspace=domain_1300,scope=xml_doc_1300,name=WriteLatency' -f OneMinuteRate -mg | tail -n1)
	
	# Coordinator latency
	coord_read_p95=$(nodetool sjk mx -b 'org.apache.cassandra.metrics:type=ClientRequest,scope=Read,name=Latency' -f 95thPercentile -mg | tail -n1)
	coord_read_rate=$(nodetool sjk mx -b 'org.apache.cassandra.metrics:type=ClientRequest,scope=Read,name=Latency' -f OneMinuteRate -mg | tail -n1)
	coord_write_p95=$(nodetool sjk mx -b 'org.apache.cassandra.metrics:type=ClientRequest,scope=Write,name=Latency' -f 95thPercentile -mg | tail -n1)
	coord_write_rate=$(nodetool sjk mx -b 'org.apache.cassandra.metrics:type=ClientRequest,scope=Write,name=Latency' -f OneMinuteRate -mg | tail -n1)

	printf "%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f,%.2f\n" $local_read_p95 $local_read_rate $local_write_p95 $local_write_rate $coord_read_p95 $coord_read_rate $coord_write_p95 $coord_write_rate >> $latency_outfile
	sleep 60
done