Test Script to run JMeter with multiple concurrent users

apim_start.sh

#!/bin/bash
export JAVA_HOME=/usr/lib/jvm/jdk1.8.0_121
if pgrep -f "carbon" > /dev/null; then
    echo "Shutting down APIM"
    /home/ubuntu/apim-product/wso2am-2.1.0/bin/wso2server.sh stop
    sleep 30
fi

log_files=(/home/ubuntu/apim-product/wso2am-2.1.0/repository/logs/*)
if [ ${#log_files[@]} -gt 1 ]; then
    echo "Log files exists. Moving to /tmp"
    mv /home/ubuntu/apim-product/wso2am-2.1.0/repository/logs/* /tmp/;
fi

# if [[ -f /home/ubuntu/apim-product/wso2am-2.1.0/repository/logs/gc.log ]]; then
#     mv /home/ubuntu/apim-product/wso2am-2.1.0/repository/logs/gc.log /tmp/
# fi

echo "Starting APIM"
/home/ubuntu/apim-product/wso2am-2.1.0/bin/wso2server.sh start
sleep 120

config

Host apim1
     HostName 172.30.2.105
     User ubuntu
     IdentityFile ~/keys/api.pem

Host apim2
     HostName 172.30.2.26
     User ubuntu
     IdentityFile ~/keys/api.pem

Host apim3
     HostName 172.30.2.101
     User ubuntu
     IdentityFile ~/keys/api.pem

Host apim4
     HostName 172.30.2.145
     User ubuntu
     IdentityFile ~/keys/api.pem

netty_start.sh

#!/bin/bash
if pgrep -f "netty" > /dev/null; then
    echo "Shutting down Netty"
    killall java
    sleep 2
fi

if [[ -f /home/ubuntu/nettygc.log ]]; then
    echo "GC Log exists. Moving to /tmp"
    mv /home/ubuntu/nettygc.log /tmp/
fi

echo "Starting Netty"
nohup java -Xms2g -Xmx2g -XX:+PrintGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/home/ubuntu/nettygc.log -jar netty-echo-service-1.0.0-SNAPSHOT.jar --port 8080 --worker-threads 3000 --sleep-time $1 </dev/null >netty.out 2>&1 & 
sleep 2

run_all.sh

#!/bin/bash
if [[ -d results ]]; then
    echo "Results directory already exists"
    exit 1
fi

concurrent_users=(3000 2000 1000 300 200 100)

api_host=172.30.2.26
api_path=/correctApi/v1/service

api_ssh_host=apim2
backend_ssh_host=apim3

mkdir -p results/gclogs
cp $0 results

run_jmeter() {
    report_location=results/$1/$2
    export JVM_ARGS="-Xms4g -Xmx4g -XX:+PrintGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$PWD/results/gclogs/jmeter_gc_$1_$2.log"
    echo "# Running JMeter. Concurrent Users: $1 Mode: $2. Duration: $3 JVM Args: $JVM_ARGS"
    jmeter.sh -n -t apim-test.jmx -Jusers=$1 -Jduration=$3 -Jhost=$api_host -Jpath=$api_path -Jbody="{\"greetings\":\"HelloWorld\"}" -l $report_location.jtl -e -o $report_location
    ss -s > results/$1/jmeter_ss_$2.txt
    ssh $api_ssh_host "ss -s" > results/$1/apim_ss_$2.txt
    ssh $backend_ssh_host "ss -s" > results/$1/tomcat_ss_$2.txt
}


run_tests() {
    for u in ${concurrent_users[@]}
    do
        mkdir -p results/$u
        ssh $api_ssh_host "./apim_start.sh"
        ssh $backend_ssh_host "./tomcat_start.sh"
        run_jmeter $u warmup 300
        # Increased due to errors
        sleep 180
        run_jmeter $u measurement 600
        scp $api_ssh_host:/home/ubuntu/apim-product/wso2am-2.1.0/repository/logs/gc.log $PWD/results/gclogs/apim_gc_$u.log
        scp $backend_ssh_host:/home/ubuntu/apache-tomcat-8.5.13/bin/gc.log $PWD/results/gclogs/tomcat_gc_$u.log
        sar -q > results/$u/jmeter_loadavg.txt
        ssh $api_ssh_host "sar -q" > results/$u/apim_loadavg.txt
        ssh $api_ssh_host "top -bn 1" > results/$u/apim_top.txt
        ssh $api_ssh_host "ps u -p \`pgrep -f carbon\`" > results/$u/apim_ps.txt
        ssh $backend_ssh_host "sar -q" > results/$u/tomcat_loadavg.txt
        ssh $backend_ssh_host "top -bn 1" > results/$u/tomcat_top.txt
        ssh $backend_ssh_host "ps u -p \`pgrep -f tomcat\`" > results/$u/tomcat_ps.txt
        # Increased due to errors
        sleep 240
    done
}

run_tests

echo "Completed"

run_all_netty.sh

#!/bin/bash
if [[ -d results ]]; then
    echo "Results directory already exists"
    exit 1
fi
threadsleep=$1
concurrent_users=(3000 2000 1000 300 200 100)

api_host=172.30.2.26
api_path=/customerapi/v1/service

api_ssh_host=apim2
backend_ssh_host=apim3

mkdir -p results/gclogs
cp $0 results

run_jmeter() {
    report_location=results/$1/$2
    export JVM_ARGS="-Xms4g -Xmx4g -XX:+PrintGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$PWD/results/gclogs/jmeter_gc_$1_$2.log"
    echo "# Running JMeter. Concurrent Users: $1 Mode: $2. Duration: $3 JVM Args: $JVM_ARGS"
    jmeter.sh -n -t apim-think-time-test.jmx -Jusers=$1 -Jduration=$3 -Jhost=$api_host -Jpath=$api_path -Jbody="{\"greetings\":\"HelloWorld\"}" -l $report_location.jtl -e -o $report_location
    ss -s > results/$1/jmeter_ss_$2.txt
    ssh $api_ssh_host "ss -s" > results/$1/apim_ss_$2.txt
    ssh $backend_ssh_host "ss -s" > results/$1/netty_ss_$2.txt
}


run_tests() {
    for u in ${concurrent_users[@]}
    do
        mkdir -p results/$u
        ssh $api_ssh_host "./apim_start.sh"
        ssh $backend_ssh_host "./netty_start.sh $threadsleep"
        run_jmeter $u warmup 300
        # Increased due to errors
        sleep 180
        run_jmeter $u measurement 600
        scp $api_ssh_host:/home/ubuntu/apim-product/wso2am-2.1.0/repository/logs/gc.log $PWD/results/gclogs/apim_gc_$u.log
        scp $backend_ssh_host:/home/ubuntu/nettygc.log $PWD/results/gclogs/netty_gc_$u.log
        sar -q > results/$u/jmeter_loadavg.txt
        ssh $api_ssh_host "sar -q" > results/$u/apim_loadavg.txt
        ssh $api_ssh_host "top -bn 1" > results/$u/apim_top.txt
        ssh $api_ssh_host "ps u -p \`pgrep -f carbon\`" > results/$u/apim_ps.txt
        ssh $backend_ssh_host "sar -q" > results/$u/netty_loadavg.txt
        ssh $backend_ssh_host "top -bn 1" > results/$u/netty_top.txt
        ssh $backend_ssh_host "ps u -p \`pgrep -f netty\`" > results/$u/netty_ps.txt
        # Increased due to errors
        sleep 240
    done
}

run_tests

echo "Completed"

tomcat_start.sh

#!/bin/bash
if pgrep -f "tomcat" > /dev/null; then
    echo "Shutting down Tomcat"
    /home/ubuntu/apache-tomcat-8.5.13/bin/shutdown.sh
    sleep 5
fi

if [[ -f /home/ubuntu/apache-tomcat-8.5.13/bin/gc.log ]]; then
    echo "GC Log exists. Moving to /tmp"
    mv /home/ubuntu/apache-tomcat-8.5.13/bin/gc.log /tmp/
fi

echo "Starting Tomcat"
/home/ubuntu/apache-tomcat-8.5.13/bin/startup.sh
sleep 5