Accounts operations setup and test

accounts_tests.sh

#!/bin/bash

USERS=$1
GERRIT_WAR=$2
JAVAMELODY_PLUGIN=$3
LIB_MODULE=$4

#parameters
REPO=test_repo
CHANGES=200
MIN_REVIEWERS=5
MAX_REVIEWERS=10
QUERY_CHANGES_TIMES=2000

if [ -z "$USERS" ]; then
  USERS=1024
  echo "### Using default no of accounts [${USERS}]"
else
  echo "### Using provided no of accounts [${USERS}]"
fi

# prepare test dir
TEST_SITE=$(mktemp -d || $(echo >&2 "Could not create temp dir" && exit 1))
echo "### Setting up the Gerrit site [${TEST_SITE}]..."
cd "$TEST_SITE"

# download Gerrit
if [ -z "$GERRIT_WAR" ]; then
  echo "### Downloadnig release: gerrit-3.2.12.war..."
  wget https://gerrit-releases.storage.googleapis.com/gerrit-3.2.12.war
else
  echo "### Using Gerrit release from [${GERRIT_WAR}]"
  cp "$GERRIT_WAR" gerrit.war
fi

GERRIT_WAR_FILENAME=$(find . -name 'gerrit*')
echo "### Running tests against the Gerrit: $(java -jar ${GERRIT_WAR_FILENAME} version) version"

# setup site
GERRIT_PORT=8091
GERRIT_SSH=29418
GERRIT_DIR="gerrit"
GERRIT_ETC_DIR="${GERRIT_DIR}/etc"

mkdir -p ${GERRIT_ETC_DIR}
mkdir -p ${GERRIT_DIR}/plugins

if [ -z "$JAVAMELODY_PLUGIN" ]; then
  echo "### Downloadnig javamelody plugin from stable-3.2 plugins..."
  wget https://gerrit-ci.gerritforge.com/view/Plugins-stable-3.2/job/plugin-javamelody-bazel-stable-3.2/lastSuccessfulBuild/artifact/bazel-bin/plugins/javamelody/javamelody.jar \
    -O "${GERRIT_DIR}/plugins/javamelody.jar"
else
  echo "### Using javamelody plugin from [${JAVAMELODY_PLUGIN}]"
  cp "$JAVAMELODY_PLUGIN" "${GERRIT_DIR}/plugins/javamelody.jar"
fi

if [ ! -z "$LIB_MODULE" ]; then
  echo "### Installing the lib module from [${LIB_MODULE}]"
  #cp "$LIB_MODULE" "${GERRIT_DIR}/plugins/"
  mkdir -p "${GERRIT_DIR}/lib"
  cp "$LIB_MODULE" "${GERRIT_DIR}/lib/"
  git config --file ${GERRIT_ETC_DIR}/gerrit.config gerrit.installDbModule "com.googlesource.gerrit.plugins.gerritcachedrefdb.LibDbModule"
  git config --file ${GERRIT_ETC_DIR}/gerrit.config gerrit.installModule "com.googlesource.gerrit.plugins.gerritcachedrefdb.LibSysModule"
  #git config --file ${GERRIT_ETC_DIR}/gerrit.config cache."gerrit-cached-refdb.ref_by_name".memoryLimit $(( ${USERS} + ( ${CHANGES} * 4) + 700 ))
  git config --file ${GERRIT_ETC_DIR}/gerrit.config cache.ref_by_name.memoryLimit $(( ( ${USERS} * 4 ) + ( ${CHANGES} * 4) ))
fi

# configure Gerrit
git config --file ${GERRIT_ETC_DIR}/gerrit.config gerrit.basePath git
git config --file ${GERRIT_ETC_DIR}/gerrit.config gerrit.canonicalWebUrl "http://localhost:${GERRIT_PORT}"
git config --file ${GERRIT_ETC_DIR}/gerrit.config auth.type DEVELOPMENT_BECOME_ANY_ACCOUNT
git config --file ${GERRIT_ETC_DIR}/gerrit.config auth.httpPasswordSettingsEnabled true
git config --file ${GERRIT_ETC_DIR}/gerrit.config container.user $(whoami)
git config --file ${GERRIT_ETC_DIR}/gerrit.config container.javaOptions "-agentlib:jdwp=transport=dt_socket,server=y,address=6667,suspend=n"
git config --file ${GERRIT_ETC_DIR}/gerrit.config sshd.listenAddress "*:${GERRIT_SSH}"
git config --file ${GERRIT_ETC_DIR}/gerrit.config httpd.listenUrl "http://*:${GERRIT_PORT}/"
git config --file ${GERRIT_ETC_DIR}/gerrit.config sendemail.enable false
git config --file ${GERRIT_ETC_DIR}/gerrit.config cache.accounts.memoryLimit $(( ${USERS} + 1 ))
git config --file ${GERRIT_ETC_DIR}/gerrit.config cache.web_sessions.memoryLimit $(( ${USERS} + 1 ))

# init site and start
echo "### Calling Gerrit init and start..."
java -jar ${GERRIT_WAR_FILENAME} init -d "$GERRIT_DIR" --install-all-plugins --batch
${GERRIT_DIR}/bin/gerrit.sh start

# prepare site for testing
export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"
export GIT_REDIRECT_STDERR='2>&1'
SSH_OPTIONS="-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no"

# create repo and upload changes for review
echo "### Creating $REPO and pushing $CHANGES changes for review..."
ssh $SSH_OPTIONS -p $GERRIT_SSH admin@localhost gerrit create-project test_repo --empty-commit
git clone "ssh://admin@localhost:${GERRIT_SSH}/${REPO}" && scp $SSH_OPTIONS -p -P $GERRIT_SSH admin@localhost:hooks/commit-msg "${REPO}/.git/hooks/"
cd ${REPO}
git config user.name Admin
git config user.email admin@email
time for i in $(seq 1 ${CHANGES}); do
  echo "some content ${i}" >> README.md
  git add README.md
  git commit -qm "update content ${i}"
  git push -q origin HEAD:refs/for/master
done
cd -
echo "### $CHANGES changes pushed for review"
ssh $SSH_OPTIONS -p $GERRIT_SSH admin@localhost gerrit show-caches -w 90

# create users
echo "### Creating ${USERS} users..."
time for i in $(seq 1 ${USERS}); do
  printf -v paddedId "%04g" $i
  ssh -q $SSH_OPTIONS -p $GERRIT_SSH admin@localhost gerrit create-account --full-name test_${paddedId} --email test_${paddedId}@example.com test_${paddedId}
  # get gerrit cookies
  GERRIT_ACCOUNT=$(curl -s -o /dev/null -D - http://localhost:${GERRIT_PORT}/login/%2Fq%2Fstatus%3Aopen%2B-is%3Awip\?account_id\=100${paddedId} | grep -Fi Set-Cookie | cut -d" " -f2)
  XSRF_TOKEN=$(curl -H "Cookie: ${GERRIT_ACCOUNT}" -s -o /dev/null -D - http://localhost:${GERRIT_PORT}/q/status:open+-is:wip | grep -Fi Set-Cookie | cut -d" " -f2)
  GERRIT_COOKIES="Cookie: ${GERRIT_ACCOUNT} ${XSRF_TOKEN}"
  COOKIES[${i}]="$GERRIT_COOKIES"
done
echo "### Creating ${USERS} users done"
ssh $SSH_OPTIONS -p $GERRIT_SSH admin@localhost gerrit show-caches -w 90

# add reviewers
echo "### Adding ${MIN_REVIEWERS} - ${MAX_REVIEWERS} random reviewers to $CHANGES changes..."
time for i in $(seq 1 ${CHANGES}); do
  unset REVIEWERS
  USER_INDEX=$(( $RANDOM % (${USERS} - ${MAX_REVIEWERS}) + 1 ))
  for j in $(seq 1 $(( ($RANDOM % (${MAX_REVIEWERS} - ${MIN_REVIEWERS})) + $MIN_REVIEWERS )) ); do
    printf -v paddedId "%04g" $USER_INDEX
    REVIEWERS="${REVIEWERS} -a 100$paddedId"
    USER_INDEX=$(( $USER_INDEX + 1 ))
  done
  ssh -q $SSH_OPTIONS -p $GERRIT_SSH admin@localhost gerrit set-reviewers -p $REPO $REVIEWERS -- $i
done
echo "### Adding reviewers done"
ssh $SSH_OPTIONS -p $GERRIT_SSH admin@localhost gerrit show-caches -w 90

# query changes
echo '%{time_total}\n' > .curl-total-inline-format.txt
echo 'time_total(s)' > query_change.log
echo "### Getting change details ${QUERY_CHANGES_TIMES} times for random 1 to ${CHANGES} changes, by random 1 to ${USERS} users..."
time for i in $(seq 1 ${QUERY_CHANGES_TIMES}); do
  curl -w "@./.curl-total-inline-format.txt" -H "${COOKIES[($RANDOM % ${USERS}) + 1]}" \
    -o /dev/null -s http://localhost:${GERRIT_PORT}/a/changes/${REPO}\~$(( ($RANDOM % ${CHANGES}) + 1 ))/detail\?O\=916314 >> query_change.log
done
echo "### Getting changes done"

# preparing script to call get changes in parallel
PATH_FILE="${TEST_SITE}/path.lua"
echo "local changes = ${CHANGES}" > ${PATH_FILE}
cat >> ${PATH_FILE} << EOM

function init(args)
    local r = {}
    local path = "/a/changes/test_repo~%d/detail?O=916314"
    for i = 1, 90 do
      r[i] = wrk.format(nil, path:format(math.random(changes - 1) + 1))
    end
    req = table.concat(r)
end

function request()
    return req
end
EOM
echo "Script to call get change details in parallel"
echo "${TEST_SITE}/gerrit/bin/gerrit.sh start"
echo "GERRIT_ACCOUNT=\$(curl -s -o /dev/null -D - http://localhost:${GERRIT_PORT}/login/%2Fq%2Fstatus%3Aopen%2B-is%3Awip\?account_id\=1000000 | grep -Fi Set-Cookie | cut -d\" \" -f2)"
echo "XSRF_TOKEN=\$(curl -H \"Cookie: \${GERRIT_ACCOUNT}\" -s -o /dev/null -D - http://localhost:${GERRIT_PORT}/q/status:open+-is:wip | grep -Fi Set-Cookie | cut -d\" \" -f2)"
echo "wrk -t8 -c8 -d60s -H \"Cookie: \${GERRIT_ACCOUNT} \${XSRF_TOKEN}\" -s \"${PATH_FILE}\" 'http://localhost:${GERRIT_PORT}'"

echo "### Stopping Gerrit..."
ssh $SSH_OPTIONS -p $GERRIT_SSH admin@localhost gerrit show-caches -w 90
${GERRIT_DIR}/bin/gerrit.sh stop

echo "### Test directory [${TEST_SITE}]"

TLDR;

Gerrit 3.1.16 performs account-heavy operations 2 (4 when trustFolderStat = false) times faster than vanilla Gerrit 3.2.12.
Gerrit 3.2.12 with RFC: Cache All-Users object ids by ref performs account-heavy operations 2-4 (7-8 when trustFolderStat = false) times faster than vanilla Gerrit 3.2.12. (unfortunately this change will never get accepted as a core solution)
Gerrit 3.2.14 with RefDB backed by Gerrit cache loaded though lib module performs account/change operations 2 (9 when trustFolderStat = false ) times faster then vanilla 3.2.14. (this change, due to its minimal impact on Gerrit core, has a chance to get accepted as a core solution for Gerrit from 3.2 to 3.5 versions)

Explanation

The accounts_test.sh script gathers stats from a few typical operations that require interaction with account_cache.

Calling the accounts_test.sh script

When called without any parameters it assumes that 1024 accounts will be created, downloads gerrit-3.2.12.war and the corresponding javamelody.jar plugin. It can be also called with three positional parameters:

1. number of accounts to be created during the test (by default 1024)
2. path to Gerrit war file (any version can be used, note that getting change detail REST API was used with version 3.2 and as such might require some modifications if different version is not compatible with)
3. path to javamelody.jar plugin (any version matching with Gerrit war file can be used).

The testing procedure

1. Gerrit site gets initiated under random test directory (configuration details are in # configure Gerrit section) and started
2. Repository test_repo gets created and cloned by Admin (over SSH)
3. 200 changes get created and pushed for review sequentially by Admin (over SSH)
4. 1024 accounts get created sequentially by Admin (over SSH), each account has username, full name and email; note that Gerrit cookies are obtained for each account (2 HTTP GET calls)
5. For each od 200 changes 5 to 10 reviewers get added sequentially by Admin (single SSH call to add varying number of reviewers per change)
6. Call 2000 times (sequentially) to get random change details by random user (over HTTP) - curl's time_total benchmark is captured for each call and stored in query_change.log file
7. Generate input (path.lua) and sequence of commands to call wrk benchmarking tool to test getting random change details in parallel. The path.lua is a wrk input script that generates random sequence of 90 changes to be queried by single thread (each thread gets its own queue). The output sequence of commands look like the following:

[path_to_temp_dir]gerrit/bin/gerrit.sh start

GERRIT_ACCOUNT=$(curl -s -o /dev/null -D - http://localhost:8091/login/%2Fq%2Fstatus%3Aopen%2B-is%3Awip\?account_id\=1000000 | grep -Fi Set-Cookie | cut -d" " -f2)
XSRF_TOKEN=$(curl -H "Cookie: ${GERRIT_ACCOUNT}" -s -o /dev/null -D - http://localhost:8091/q/status:open+-is:wip | grep -Fi Set-Cookie | cut -d" " -f2)

wrk -t8 -c8 -d300s -H "Cookie: ${GERRIT_ACCOUNT} ${XSRF_TOKEN}" -s "[path_to_temp_dir]/path.lua" 'http://localhost:8091'

and means that 8 threads in parallel is going to gat as often as possible random change details during 300s period of time.

Results

Hardware:

Model Name: MacBook Pro
Model Identifier: MacBookPro16,1
Processor Name: 6-Core Intel Core i7
Processor Speed: 2,6 GHz
Number of Processors: 1
Total Number of Cores: 6
L2 Cache (per Core): 256 KB
L3 Cache: 12 MB
Hyper-Threading Technology: Enabled (results in 12 HT Cores)
Memory: 16 GB
HDD: local SSD drive

The test script and wrk was called against:

1. vanilla Gerrit 3.1.16 version (marked as stable-3.1 in the results)
2. vanilla Gerrit 3.2.12 version (marked as stable-3.2 in the results)
3. Gerrit 3.2.12 with RFC: Cache All-Users object ids by ref change applied (marked as stable-3.2-cache in the results)
4. Gerrit 3.2.14 with libCache module loaded (marked as stable-3.2-libCache in the results)

Steps 3 - 6 totals:

release	create 200 changes(s)	create 1024 users(s)	Add 5-10 reviewers(s)	Get random changes 2000 times(s)
stable-3.1	59,165	106,714	50,639	34,169
stable-3.2	71,552	122,036	79,73	56,141
stable-3.2-cache	66,492	115,476	46,874	32,132
stable-3.2-libCache	58,829	124,426	37,952	47,425
stable-3.1 vs stable-3.2	-20,94%	-14,36%	-57,45%	-64,30%
stable-3.2-cache vs stable-3.2	-7,61%	-5,68%	-70,09%	-74,72%
stable-3.2-cache vs stable-3.1	12,38%	8,21%	-8,03%	-6,34%
stable-3.2-libCache vs stable-3.2	-21,63%	1,96%	-110,08%	-18,38%
stable-3.2-libCache vs stable-3.1	-0,57%	16,60%	-33,43%	38,80%

Lib-cached version with regards to totals performs sometimes better a bit than stable-3.1 (changes creation and adding reviewers) and a bit slower then stable-3.2 (for accounts creation).
Notes:

• so far no concurrency is involved
• chart (and all following) is with the given order stable-3.2 (typically the slowest), stable-3.1, stable-3.2-cache, stable-3.2-libCache.

Get random change details 2000 times step details (unless specified values are given in ms):

release	min	median	avg	perc60	perc70	perc80	perc90	perc99	max
stable-3.1	8,193	10,702	10,993	11,006	11,286	11,588	12,200	15,889	114,492
stable-3.2	16,009	21,307	21,603	22,239	22,880	23,703	24,628	29,797	171,432
stable-3.2-cache	7,269	8,710	8,947	8,915	9,160	9,495	10,243	12,717	35,006
stable-3.2-libCache	13,045	16,600	17,001	16,899	17,206	17,674	18,899	23,334	67,856
stable-3.1 vs stable-3.2	-95,40%	-99,10%	-96,52%	-102,06%	-102,73%	-104,54%	-101,87%	-87,53%	-49,73%
stable-3.2-cache vs stable-3.2	-120,24%	-144,63%	-141,45%	-149,46%	-149,77%	-149,62%	-140,43%	-134,31%	-389,72%
stable-3.2-cache vs stable-3.1	-12,71%	-22,86%	-22,86%	-23,46%	-23,20%	-22,04%	-19,11%	-24,94%	-227,06%
stable-3.2-libCache vs stable-3.2	-22,72%	-28,36%	-27,07%	-31,60%	-32,97%	-34,11%	-30,31%	-27,70%	-152,64%
stable-3.2-libCache vs stable-3.1	59,22%	55,12%	54,65%	53,54%	52,46%	52,51%	54,91%	46,85%	-68,73%

Note that the all following pictures have log(2) scale.

The stable-3.1 version is typically ~2 times more performant in all aspects but max where it is closer to ~1.5 whereas libCached version is typically ~1/4 faster than stable-3.2 and seems to be better in responding to load as max is ~2 times faster than stable-3.1.

Adding concurrency to the picture with wrk benchmarking tool

The following wrk command was used

wrk -t8 -c8 -d300s -H "Cookie: ${GERRIT_ACCOUNT} ${XSRF_TOKEN}" -s "[path_to_temp_dir]/path.lua" 'http://192.168.1.115:8091'

Note that in order to avoid resources cannibalisation by client the wrk command was called from the different machine in LAN.

The following table shows the summary of the wrk command run where

• TRS is Reqs/Sec for each Thread

release	TRS Avg	TRS Std Dev	TRS Max	Total Reqs/sec	Transfer/sec(MB)
stable-3.1	109,09	11,96	141	868,29	8,26
stable-3.2	43,79	6,63	70	349,68	3,41
stable-3.2-cache	166,19	23,38	282	1323,78	13,14
stable-3.2-libCache	106,28	10,36	181	844,88	8,51
stable-3.1 vs stable-3.2	149,12%	80,39%	101,43%	148,31%	142,23%
stable-3.2-cache vs stable-3.2	279,52%	252,64%	302,86%	278,57%	285,34%
stable-3.2-cache vs stable-3.1	52,34%	95,48%	100,00%	52,46%	59,08%
stable-3.2-libCache vs stable-3.2	142,70%	56,26%	158,57%	141,62%	149,56%
stable-3.2-libCache vs stable-3.1	-2,64%	-15,44%	28,37%	-2,77%	3,03%

The average Regs/Sec for each Thread for libCached version is 106 and for stable-3.1 is 109 which is ~2 correspondingly times higher than stable-3.2 Gerrit.

Adding NFS-like behaviour by setting trustFolderStat = false

The same wrk command as previously was called but this time trustFolderStat was disabled (and Gerrit restarted to pickup the change).

release	TRS Avg	TRS Std Dev	TRS Max	Total Reqs/sec	Transfer/sec(MB)
stable-3.1	57,33	8,26	80	456,95	4,34
stable-3.2	13,87	4,92	20	110,18	1,07
stable-3.2-cache	92,21	13,75	191	733	7,28
stable-3.2-libCache	105,27	14,55	150	834,88	8,41
stable-3.1 vs stable-3.2	313,34%	67,89%	300,00%	314,73%	305,61%
stable-3.2-cache vs stable-3.2	564,82%	179,47%	855,00%	565,28%	580,37%
stable-3.2-cache vs stable-3.1	60,84%	66,46%	138,75%	60,41%	67,74%
stable-3.2-libCache vs stable-3.2	658,98%	195,73%	650,00%	657,74%	685,98%
stable-3.2-libCache vs stable-3.1	83,62%	76,15%	87,50%	82,71%	93,78%

In this test both stable-3.1 gets ~2 times slower in comparison to when trustFolderStat = true and stable-3.2-libCached version maintains the performance however stable-3.2 gets substantially worse:

• it is ~4 times slower then stable-3.1
• and ~9 times slower then stable-3.2-libCache
stable-3.2-libCache is visibly faster then stable-3.1 (by factor ~2).

Resource consumption during the whole testing

CPU consumption:

• stable-3.1

* stable-3.2

* stable-3.2-cache

With the same resource consumption stable-3.1 version offers easily 2 (4 when trustFolderStat = false) times better performance for account-heavy operations. The stable-3.2-libCache performs comparable to stable-3.1 being even better (9 times faster) when trustFolderStat = false.